#!/usr/bin/env python3 import argparse import json import jinja2 import re import requests import shutil import tempfile import pathlib import subprocess as sp from base64 import b32encode, b32decode, b64encode, b16decode from bs4 import BeautifulSoup env = jinja2.Environment() all_versions_template = env.from_string("""{ buildECTesterStandalone }: { {% for version in pkg_versions %} {{ version }} {% endfor %} }""") def get_source_hash(url, unpack=False): digest_type = "sha256" cmd = ["nix-prefetch-url"] if unpack: cmd.append("--unpack") cmd.extend(["--type", digest_type, url]) digest_nixbase32 = sp.check_output(cmd, stderr=sp.DEVNULL).strip() digest_sri = sp.check_output(["nix", "hash", "to-sri", "--type", digest_type, digest_nixbase32.decode()], stderr=sp.DEVNULL).strip().decode() return digest_sri def fetch_botan(): pkg = "botan" # NOTE: this way omits the older releases at https://botan.randombit.net/releases/old release_list = "https://botan.randombit.net/releases/" download_url = "https://botan.randombit.net/releases/{version}" resp = requests.get(release_list) soup = BeautifulSoup(resp.content, 'html.parser') single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; source_extension="{{ ext }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for link in soup.find_all("a"): if link.text.startswith("Botan") and not link.text.endswith('.asc'): download_link = download_url.format(version=link['href']) match = re.match(r"Botan-(?P\d+)\.(?P\d+)\.(?P\d+)\.(?P.*)", link.text) version = f"{match['major']}.{match['minor']}.{match['patch']}" ext = f"{match['ext']}" digest = get_source_hash(download_link) # NOTE: use underscore to separate the versions? flat_version = f"v{match['major']}{match['minor']}{match['patch']}" print(f"{version}:{digest}") rendered = single_version_template.render(pkg=pkg, digest=digest, ext=ext, flat_version=flat_version, version=version).strip() renders.append(rendered) versions[flat_version] = { "version": version, "source_extension": ext, "hash": digest } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_cryptopp(): pkg = "cryptopp" owner = "weidai11" repo = "cryptopp" release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" resp = requests.get(release_url) single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for release in resp.json(): if not release['draft'] and not release['prerelease']: _, *version_values = release['tag_name'].split('_') underscored_version = '_'.join(version_values) flat_version = "v" + "".join(version_values) download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz" digest = get_source_hash(download_url, unpack=True) print(f"{underscored_version}:{digest}") rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=underscored_version).strip() renders.append(rendered) versions[flat_version] = { "version": underscored_version, "hash": digest } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_openssl(): pkg = "openssl" owner = "openssl" repo = "openssl" release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" resp = requests.get(release_url) single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for release in resp.json(): if not release['draft'] and not release['prerelease']: try: _, dotted_version = release['tag_name'].split('-') except ValueError: continue flat_version = "v" + "".join(dotted_version.split('.')) download_url = f"https://www.openssl.org/source/openssl-{dotted_version}.tar.gz" digest = get_source_hash(download_url) print(f"{dotted_version}:{digest}") versions[flat_version] = { "version": dotted_version, "hash": digest } rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=dotted_version).strip() renders.append(rendered) all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_tomcrypt(): # fetch libtomcrypt pass def fetch_gcrypt(): pkg = "gcrypt" release_list = "https://gnupg.org/ftp/gcrypt/libgcrypt/" download_url = "https://gnupg.org/ftp/gcrypt/libgcrypt/{version}" resp = requests.get(release_list) soup = BeautifulSoup(resp.content, 'html.parser') single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for link in soup.find_all("a"): if link.text.startswith("libgcrypt") and link.text.endswith("tar.bz2"): download_link = download_url.format(version=link['href']) match = re.match(r"libgcrypt-(?P\d+)\.(?P\d+)\.(?P\d+)(?P_do_not_use)?\.(?P.*)", link.text) version = f"{match['major']}.{match['minor']}.{match['patch']}" digest = get_source_hash(download_link) print(f"{version}:{digest}") flat_version = f"v{match['major']}{match['minor']}{match['patch']}" if match['dont']: flat_version += "_do_not_use" rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() renders.append(rendered) versions[flat_version] = { "version": version, "hash": digest } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open("./nix/gcrypt_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_boringssl(): pkg = "boringssl" single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { rev="{{ rev }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} with tempfile.TemporaryDirectory() as repodir, tempfile.TemporaryDirectory() as gitdir: repodir = pathlib.Path(repodir) gitdir = pathlib.Path(gitdir) sp.run(["git", "clone", "https://boringssl.googlesource.com/boringssl", repodir]) # NOTE: we need to get rid of the .git so that it is not included in the derivation hash shutil.move(repodir / ".git", gitdir) output = sp.check_output(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "log", "--pretty=format:%H"]) refs = output.decode().split('\n') for i, rev in enumerate(refs[:100]): sp.run(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "checkout", rev]) digest = sp.check_output(["nix", "hash", "path", str(repodir)]).decode().strip() print(f"{i + 1: 4d}:{rev}:{digest}") abbrev_commit = str(rev[:8]) rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=f"r{abbrev_commit}", rev=rev).strip() renders.append(rendered) versions[f"r{abbrev_commit}"] = { "rev": rev, "hash": digest, } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_mbedtls(): # Mbed-TLS/mbedtls pkg = "mbedtls" owner = "Mbed-TLS" repo = "mbedtls" release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" resp = requests.get(release_url) single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for release in resp.json(): if not release['draft'] and not release['prerelease']: version = release['tag_name'] flat_version = version.replace('.', '') download_url = f"https://github.com/{owner}/{repo}/archive/{version}.tar.gz" digest = get_source_hash(download_url, unpack=True) print(f"{version}:{digest}") rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() renders.append(rendered) versions[flat_version] = { "version": version, "hash": digest, } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_ippcp(): # https://api.github.com/repos/intel/ipp-crypto/releases pkg = "ippcp" owner = "intel" repo = "ipp-crypto" release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" resp = requests.get(release_url) single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for release in resp.json(): if not release['draft'] and not release['prerelease']: version = release['tag_name'].split('_')[1] flat_version = "v" + version.replace('.', '_') download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz" digest = get_source_hash(download_url, unpack=True) print(f"{version}:{digest}") rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() renders.append(rendered) versions[flat_version] = { "version": version, "hash": digest, } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_nettle(): # https://api.github.com/repos/intel/ipp-crypto/releases pkg = "nettle" owner = "gnutls" repo = "nettle" release_url = f"https://api.github.com/repos/{owner}/{repo}/tags" resp = requests.get(release_url) single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; tag="{{ tag }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for tag in resp.json(): if tag['name'] == 'release_nettle_0.2.20010617': continue version = tag['name'].split('_')[1] # NOTE skip release candidates if re.search(r'\drc\d', version): continue flat_version = "v" + version.replace('.', '_') # download_url = f"https://github.com/{owner}/{repo}/archive/{tag['name']}.tar.gz" download_url = f"mirror://gnu/nettle/nettle-{version}.tar.gz" digest = get_source_hash(download_url, unpack=False) print(f"{version}:{digest}") rendered = single_version_template.render( pkg=pkg, digest=digest, flat_version=flat_version, tag=tag['name'], version=version).strip() renders.append(rendered) versions[flat_version] = { "version": version, "tag": tag['name'], "hash": digest, } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def fetch_libressl(): pkg = "libressl" release_list = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/" download_url = "mirror://openbsd/LibreSSL/libressl-{version}.tar.gz" resp = requests.get(release_list) soup = BeautifulSoup(resp.content, 'html.parser') single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; };""") renders = [] versions = {} for link in soup.find_all("a"): if link.text.startswith("libressl") and link.text.endswith('.tar.gz'): match = re.match(r"libressl-(?P\d+)\.(?P\d+)\.(?P\d+)\.tar.gz", link.text) version = f"{match['major']}.{match['minor']}.{match['patch']}" download_link = download_url.format(version=version) digest = get_source_hash(download_link) print(f"{version}:{digest}") # NOTE: use underscore to separate the versions? flat_version = f"v{match['major']}{match['minor']}{match['patch']}" rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() renders.append(rendered) versions[flat_version] = { "version": version, "hash": digest, } all_versions = all_versions_template.render(pkg_versions=renders).strip() with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: handle.write(all_versions) with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: json.dump(versions, handle, indent=4) def main(): parser = argparse.ArgumentParser() parser.add_argument("lib") args = parser.parse_args() print(f"Fetching versions and source hashes for: {args.lib}") match args.lib: case "botan": fetch_botan() case "cryptopp": fetch_cryptopp() case "openssl": fetch_openssl() case "boringssl": fetch_boringssl() case "gcrypt": fetch_gcrypt() case "mbedtls": fetch_mbedtls() case "ippcp": fetch_ippcp() case "nettle": fetch_nettle() case "libressl": fetch_libressl() case "all": fetch_botan() fetch_cryptopp() fetch_openssl() fetch_boringssl() fetch_gcrypt() fetch_mbedtls() fetch_ippcp() fetch_nettle() fetch_libressl() case _: print("Unknown library") if __name__ == '__main__': main()