From 8347e1dba7804f2b0a380d18dc4e9fd850250ea7 Mon Sep 17 00:00:00 2001 From: J08nY Date: Sat, 17 Aug 2024 19:32:02 +0200 Subject: Move nix-related Python scripts to nix directory. --- fetchReleases.py | 432 -------------------------------------------------- flake.nix | 6 +- nix/build_all.py | 100 ++++++++++++ nix/fetch_releases.py | 432 ++++++++++++++++++++++++++++++++++++++++++++++++++ nix/plot_versions.py | 81 ++++++++++ plot_versions.py | 81 ---------- test_building_all.py | 100 ------------ 7 files changed, 616 insertions(+), 616 deletions(-) delete mode 100644 fetchReleases.py create mode 100644 nix/build_all.py create mode 100644 nix/fetch_releases.py create mode 100644 nix/plot_versions.py delete mode 100644 plot_versions.py delete mode 100644 test_building_all.py diff --git a/fetchReleases.py b/fetchReleases.py deleted file mode 100644 index e6f03b5..0000000 --- a/fetchReleases.py +++ /dev/null @@ -1,432 +0,0 @@ -#!/usr/bin/env python3 - -import argparse - -import json -import jinja2 -import re -import requests -import shutil -import tempfile - -import pathlib -import subprocess as sp - -from base64 import b32encode, b32decode, b64encode, b16decode -from bs4 import BeautifulSoup -from packaging.version import parse as parse_version, Version - -env = jinja2.Environment() - -all_versions_template = env.from_string("""{ - buildECTesterStandalone -}: -{ {% for version in pkg_versions %} - {{ version }} {% endfor %} -}""") - -def get_source_hash(url, unpack=False): - digest_type = "sha256" - - cmd = ["nix-prefetch-url"] - if unpack: - cmd.append("--unpack") - cmd.extend(["--type", digest_type, url]) - - digest_nixbase32 = sp.check_output(cmd, stderr=sp.DEVNULL).strip() - digest_sri = sp.check_output(["nix", "hash", "to-sri", "--type", digest_type, digest_nixbase32.decode()], stderr=sp.DEVNULL).strip().decode() - return digest_sri - -def serialize_versions(pkg, renders, versions): - sorted_versions = {k: {kk: vv for kk, vv in v.items() if kk != "sort"} for k, v in sorted(versions.items(), key=lambda item: item[1]["sort"], reverse=True)} - - # all_versions = all_versions_template.render(pkg_versions=renders).strip() - # with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: - # handle.write(all_versions) - - with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: - json.dump(sorted_versions, handle, indent=4) - -def fetch_botan(): - pkg = "botan" - # NOTE: this way omits the older releases at https://botan.randombit.net/releases/old - release_list = "https://botan.randombit.net/releases/" - download_url = "https://botan.randombit.net/releases/{version}" - resp = requests.get(release_list) - soup = BeautifulSoup(resp.content, 'html.parser') - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; source_extension="{{ ext }}"; hash="{{ digest }}"; }; - };""") - - renders = [] - versions = {} - for link in soup.find_all("a"): - if link.text.startswith("Botan") and not link.text.endswith('.asc'): - download_link = download_url.format(version=link['href']) - - match = re.match(r"Botan-(?P\d+)\.(?P\d+)\.(?P\d+)\.(?P.*)", link.text) - if match['major'] == "3": - # TODO: Handle Botan-3 - print(f"Skipping Botan-3 {match}") - continue - version = f"{match['major']}.{match['minor']}.{match['patch']}" - ext = f"{match['ext']}" - - digest = get_source_hash(download_link) - # NOTE: use underscore to separate the versions? - flat_version = f"v{match['major']}{match['minor']}{match['patch']}" - print(f"{version}:{digest}") - - rendered = single_version_template.render(pkg=pkg, digest=digest, ext=ext, flat_version=flat_version, version=version).strip() - renders.append(rendered) - versions[flat_version] = { - "version": version, - "source_extension": ext, - "hash": digest, - "sort": parse_version(version) - } - serialize_versions(pkg, renders, versions) - -def fetch_cryptopp(): - pkg = "cryptopp" - owner = "weidai11" - repo = "cryptopp" - release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" - resp = requests.get(release_url) - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; - };""") - renders = [] - versions = {} - for release in resp.json(): - if not release['draft'] and not release['prerelease']: - _, *version_values = release['tag_name'].split('_') - underscored_version = '_'.join(version_values) - flat_version = "v" + "".join(version_values) - download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz" - digest = get_source_hash(download_url, unpack=True) - print(f"{underscored_version}:{digest}") - - rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=underscored_version).strip() - renders.append(rendered) - versions[flat_version] = { - "version": underscored_version, - "hash": digest, - "sort": parse_version(underscored_version.replace("_", ".")) - } - serialize_versions(pkg, renders, versions) - -def fetch_openssl(): - pkg = "openssl" - owner = "openssl" - repo = "openssl" - release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" - resp_releases = requests.get(release_url) - tags_url = f"https://api.github.com/repos/{owner}/{repo}/git/matching-refs/tags" - resp_tags = requests.get(tags_url) - - tags = [release["tag_name"] for release in resp_releases.json() if not release["draft"] and not release["prerelease"]] - tags += [tag_ref["ref"].split("/")[-1] for tag_ref in resp_tags.json() if tag_ref["ref"].startswith("refs/tags/openssl-") or tag_ref["ref"].startswith("refs/tags/OpenSSL_")] - tags = list(filter(lambda tag: "FIPS" not in tag and "reformat" not in tag and "alpha" not in tag and "beta" not in tag and "pre" not in tag, tags)) - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; - };""") - renders = [] - versions = {} - for tag in tags: - if tag.startswith("OpenSSL_"): - match = re.match(r"OpenSSL_(?P\d+)_(?P\d+)_(?P\d+)(?P.*)", tag) - sort_version = f"{match['major']}.{match['minor']}.{match['patch']}{'+' + match['ext'] if match['ext'] else ''}" - dotted_version = f"{match['major']}.{match['minor']}.{match['patch']}{ match['ext'] if match['ext'] else ''}" - else: - try: - _, dotted_version = tag.split('-') - sort_version = dotted_version - except ValueError: - continue - flat_version = "v" + "".join(dotted_version.split('.')) - download_url = f"https://www.openssl.org/source/openssl-{dotted_version}.tar.gz" - old_url = f"https://www.openssl.org/source/old/openssl-{dotted_version}.tar.gz" - try: - digest = get_source_hash(download_url) - except Exception: - try: - digest = get_source_hash(old_url) - except Exception: - print(f"Skipping {dotted_version} (unavailable)") - continue - print(f"{dotted_version}:{digest}") - versions[flat_version] = { - "version": dotted_version, - "hash": digest, - "sort": parse_version(sort_version) - } - - rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=dotted_version).strip() - renders.append(rendered) - serialize_versions(pkg, renders, versions) - - -def fetch_tomcrypt(): - # fetch libtomcrypt - pass - -def fetch_gcrypt(): - pkg = "gcrypt" - release_list = "https://gnupg.org/ftp/gcrypt/libgcrypt/" - download_url = "https://gnupg.org/ftp/gcrypt/libgcrypt/{version}" - resp = requests.get(release_list) - soup = BeautifulSoup(resp.content, 'html.parser') - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; - };""") - - renders = [] - versions = {} - for link in soup.find_all("a"): - if link.text.startswith("libgcrypt") and link.text.endswith("tar.bz2"): - download_link = download_url.format(version=link['href']) - - match = re.match(r"libgcrypt-(?P\d+)\.(?P\d+)\.(?P\d+)(?P_do_not_use)?\.(?P.*)", link.text) - version = f"{match['major']}.{match['minor']}.{match['patch']}" - - digest = get_source_hash(download_link) - print(f"{version}:{digest}") - - flat_version = f"v{match['major']}{match['minor']}{match['patch']}" - if match['dont']: - flat_version += "_do_not_use" - - rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() - renders.append(rendered) - versions[flat_version] = { - "version": version, - "hash": digest, - "sort": parse_version(version) - } - serialize_versions(pkg, renders, versions) - -def fetch_boringssl(): - pkg = "boringssl" - upto = "76bb1411acf5cf6935586182a3a037d372ed1636" - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { rev="{{ rev }}"; hash="{{ digest }}"; }; - };""") - renders = [] - versions = {} - with tempfile.TemporaryDirectory() as repodir, tempfile.TemporaryDirectory() as gitdir: - repodir = pathlib.Path(repodir) - gitdir = pathlib.Path(gitdir) - sp.run(["git", "clone", "https://boringssl.googlesource.com/boringssl", repodir]) - # NOTE: we need to get rid of the .git so that it is not included in the derivation hash - shutil.move(repodir / ".git", gitdir) - - output = sp.check_output(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "log", "--pretty=format:%H"]) - refs = output.decode().split('\n') - - upto_index = refs.index(upto) - - # pick roughly 100 commits evenly spaced from the "upto" commit - for i, rev in enumerate(refs[upto_index:0:-40]): - sp.run(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "checkout", rev]) - digest = sp.check_output(["nix", "hash", "path", str(repodir)]).decode().strip() - print(f"{i + 1: 4d}:{rev}:{digest}") - abbrev_commit = str(rev[:8]) - - rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=f"r{abbrev_commit}", rev=rev).strip() - renders.append(rendered) - versions[f"r{abbrev_commit}"] = { - "rev": rev, - "hash": digest, - "sort": i - } - serialize_versions(pkg, renders, versions) - -def fetch_mbedtls(): - # Mbed-TLS/mbedtls - pkg = "mbedtls" - owner = "Mbed-TLS" - repo = "mbedtls" - release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" - resp = requests.get(release_url) - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; - };""") - renders = [] - versions = {} - for release in resp.json(): - if not release['draft'] and not release['prerelease']: - tag = release["tag_name"] - version = tag.replace("mbedtls-", "v") - flat_version = version.replace('.', '') - download_url = f"https://github.com/{owner}/{repo}/archive/{tag}.tar.gz" - if version == "v3.6.0": - # TODO: Special case for the time being - digest = "sha256-tCwAKoTvY8VCjcTPNwS3DeitflhpKHLr6ygHZDbR6wQ=" - else: - digest = get_source_hash(download_url, unpack=True) - - print(f"{version}:{digest}") - - rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() - renders.append(rendered) - versions[flat_version] = { - "version": version, - "hash": digest, - "tag": tag, - "sort": parse_version(version) - } - serialize_versions(pkg, renders, versions) - -def fetch_ippcp(): - # https://api.github.com/repos/intel/ipp-crypto/releases - pkg = "ippcp" - owner = "intel" - repo = "ipp-crypto" - release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" - resp = requests.get(release_url) - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; - };""") - renders = [] - versions = {} - for release in resp.json(): - if not release['draft'] and not release['prerelease']: - version = release['tag_name'].split('_')[1] - flat_version = "v" + version.replace('.', '_') - download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz" - digest = get_source_hash(download_url, unpack=True) - print(f"{version}:{digest}") - - rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() - renders.append(rendered) - versions[flat_version] = { - "version": version, - "hash": digest, - "sort": parse_version(version.replace("u", "+u")) - } - serialize_versions(pkg, renders, versions) - -def fetch_nettle(): - # https://api.github.com/repos/intel/ipp-crypto/releases - pkg = "nettle" - owner = "gnutls" - repo = "nettle" - release_url = f"https://api.github.com/repos/{owner}/{repo}/tags" - resp = requests.get(release_url) - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; tag="{{ tag }}"; hash="{{ digest }}"; }; - };""") - renders = [] - versions = {} - for tag in resp.json(): - if tag['name'] == 'release_nettle_0.2.20010617': - continue - if tag['name'] == 'nettle_3.5_release_20190626': - # broken upstream! https://git.lysator.liu.se/nettle/nettle/-/commit/ee5d62898cf070f08beedc410a8d7c418588bd95 - continue - version = tag['name'].split('_')[1] - # NOTE skip release candidates - if re.search(r'\drc\d', version): - continue - flat_version = "v" + version.replace('.', '_') - # download_url = f"https://github.com/{owner}/{repo}/archive/{tag['name']}.tar.gz" - download_url = f"mirror://gnu/nettle/nettle-{version}.tar.gz" - digest = get_source_hash(download_url, unpack=False) - print(f"{version}:{digest}") - - rendered = single_version_template.render( - pkg=pkg, digest=digest, flat_version=flat_version, tag=tag['name'], version=version).strip() - renders.append(rendered) - versions[flat_version] = { - "version": version, - "tag": tag['name'], - "hash": digest, - "sort": parse_version(version) - } - serialize_versions(pkg, renders, versions) - - -def fetch_libressl(): - pkg = "libressl" - release_list = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/" - download_url = "mirror://openbsd/LibreSSL/libressl-{version}.tar.gz" - resp = requests.get(release_list) - soup = BeautifulSoup(resp.content, 'html.parser') - - single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { - {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; - };""") - - renders = [] - versions = {} - for link in soup.find_all("a"): - if link.text.startswith("libressl") and link.text.endswith('.tar.gz'): - match = re.match(r"libressl-(?P\d+)\.(?P\d+)\.(?P\d+)\.tar.gz", link.text) - version = f"{match['major']}.{match['minor']}.{match['patch']}" - download_link = download_url.format(version=version) - digest = get_source_hash(download_link) - print(f"{version}:{digest}") - # NOTE: use underscore to separate the versions? - flat_version = f"v{match['major']}{match['minor']}{match['patch']}" - - rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() - renders.append(rendered) - versions[flat_version] = { - "version": version, - "hash": digest, - "sort": parse_version(version) - } - serialize_versions(pkg, renders, versions) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("lib") - args = parser.parse_args() - - print(f"Fetching versions and source hashes for: {args.lib}") - - match args.lib: - case "botan": - fetch_botan() - case "cryptopp": - fetch_cryptopp() - case "openssl": - fetch_openssl() - case "boringssl": - fetch_boringssl() - case "gcrypt": - fetch_gcrypt() - case "mbedtls": - fetch_mbedtls() - case "ippcp": - fetch_ippcp() - case "nettle": - fetch_nettle() - case "libressl": - fetch_libressl() - case "all": - fetch_botan() - fetch_cryptopp() - fetch_openssl() - fetch_boringssl() - fetch_gcrypt() - fetch_mbedtls() - fetch_ippcp() - fetch_nettle() - fetch_libressl() - case _: - print("Unknown library") - - -if __name__ == '__main__': - main() diff --git a/flake.nix b/flake.nix index 33ac4f5..236e5eb 100644 --- a/flake.nix +++ b/flake.nix @@ -870,7 +870,7 @@ packaging ]; - src = ./fetchReleases.py; + src = ./nix/fetch_releases.py; dontUnpack = true; installPhase = '' install -Dm755 $src $out/bin/$pname @@ -889,7 +889,7 @@ pudb ]; - src = ./test_building_all.py; + src = ./nix/build_all.py; dontUnpack = true; installPhase = '' install -Dm755 $src $out/bin/$pname @@ -908,7 +908,7 @@ jinja2 ]; - src = ./plot_versions.py; + src = ./nix/plot_versions.py; dontUnpack = true; installPhase = '' install -Dm755 $src $out/bin/$pname diff --git a/nix/build_all.py b/nix/build_all.py new file mode 100644 index 0000000..782f8b2 --- /dev/null +++ b/nix/build_all.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +import argparse +import json +import time + +from pathlib import Path + +import subprocess as sp + +def get_all_versions(library): + with open(f"./nix/{library}_pkg_versions.json", "r") as handle: + versions = json.load(handle) + + return versions + +def attempt_build(library, version, variant): + cmd = ["nix", "build", f".#{variant}.{library}.{version}"] + start = time.time() + + result = {} + try: + sp.check_output(cmd, stderr=sp.STDOUT) + success = True + stderr = "" + except sp.CalledProcessError as e: + stderr = e.output.decode() + success = False + + result['build_time'] = time.time() - start + result['success'] = success + result['stderr'] = stderr.split('\n') if stderr else [] + + return result + +def valid_build_type(value): + value = value.strip() + valid_types = ["shim", "lib"] + if value not in valid_types: + raise argparse.ArgumentTypeError(f"'{value}' not from expected {', '.join(valid_types)}.") + return value + +def save_build_result(library, variant, version, result): + resdir = Path(f"build_all/{variant}") + resdir.mkdir(parents=True, exist_ok=True) + try: + # Update previous results + with open(resdir / f"{library}.json", "r") as handle: + prev_results = json.load(handle) + # NOTE this is not ideal as the JSON decoding problem can be other than just an empty file + except (FileNotFoundError, json.JSONDecodeError): + prev_results = {} + + prev_results[version] = result + with open(resdir / f"{library}.json", "w") as handle: + json.dump(prev_results, handle, indent=4) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-l", "--library") + parser.add_argument("-v", "--variant", default="shim", type=valid_build_type) + args = parser.parse_args() + library = args.library + variant = args.variant + + libraries = [ + "botan", + "cryptopp", + "openssl", + "boringssl", + "gcrypt", + "mbedtls", + "ippcp", + "nettle", + "libressl", + ] + + match library: + case None: + print("Building all libraries") + # Build all libraries by default + for lib in libraries: + print(f"Library: {lib}") + for version in get_all_versions(lib): + result = attempt_build(lib, version, variant) + save_build_result(lib, variant, version, result) + print(f"{version}: {result['success']}") + case lib if lib in libraries: + print(f"Library: {library}") + for version in get_all_versions(library): + result = attempt_build(lib, version, variant) + save_build_result(lib, variant, version, result) + print(f"{version}: {result['success']}") + case _: + print(f"Unrecognized library '{library}'. Try one of: {', '.join(libraries)}.") + + +if __name__ == '__main__': + main() diff --git a/nix/fetch_releases.py b/nix/fetch_releases.py new file mode 100644 index 0000000..e6f03b5 --- /dev/null +++ b/nix/fetch_releases.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 + +import argparse + +import json +import jinja2 +import re +import requests +import shutil +import tempfile + +import pathlib +import subprocess as sp + +from base64 import b32encode, b32decode, b64encode, b16decode +from bs4 import BeautifulSoup +from packaging.version import parse as parse_version, Version + +env = jinja2.Environment() + +all_versions_template = env.from_string("""{ + buildECTesterStandalone +}: +{ {% for version in pkg_versions %} + {{ version }} {% endfor %} +}""") + +def get_source_hash(url, unpack=False): + digest_type = "sha256" + + cmd = ["nix-prefetch-url"] + if unpack: + cmd.append("--unpack") + cmd.extend(["--type", digest_type, url]) + + digest_nixbase32 = sp.check_output(cmd, stderr=sp.DEVNULL).strip() + digest_sri = sp.check_output(["nix", "hash", "to-sri", "--type", digest_type, digest_nixbase32.decode()], stderr=sp.DEVNULL).strip().decode() + return digest_sri + +def serialize_versions(pkg, renders, versions): + sorted_versions = {k: {kk: vv for kk, vv in v.items() if kk != "sort"} for k, v in sorted(versions.items(), key=lambda item: item[1]["sort"], reverse=True)} + + # all_versions = all_versions_template.render(pkg_versions=renders).strip() + # with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle: + # handle.write(all_versions) + + with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle: + json.dump(sorted_versions, handle, indent=4) + +def fetch_botan(): + pkg = "botan" + # NOTE: this way omits the older releases at https://botan.randombit.net/releases/old + release_list = "https://botan.randombit.net/releases/" + download_url = "https://botan.randombit.net/releases/{version}" + resp = requests.get(release_list) + soup = BeautifulSoup(resp.content, 'html.parser') + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; source_extension="{{ ext }}"; hash="{{ digest }}"; }; + };""") + + renders = [] + versions = {} + for link in soup.find_all("a"): + if link.text.startswith("Botan") and not link.text.endswith('.asc'): + download_link = download_url.format(version=link['href']) + + match = re.match(r"Botan-(?P\d+)\.(?P\d+)\.(?P\d+)\.(?P.*)", link.text) + if match['major'] == "3": + # TODO: Handle Botan-3 + print(f"Skipping Botan-3 {match}") + continue + version = f"{match['major']}.{match['minor']}.{match['patch']}" + ext = f"{match['ext']}" + + digest = get_source_hash(download_link) + # NOTE: use underscore to separate the versions? + flat_version = f"v{match['major']}{match['minor']}{match['patch']}" + print(f"{version}:{digest}") + + rendered = single_version_template.render(pkg=pkg, digest=digest, ext=ext, flat_version=flat_version, version=version).strip() + renders.append(rendered) + versions[flat_version] = { + "version": version, + "source_extension": ext, + "hash": digest, + "sort": parse_version(version) + } + serialize_versions(pkg, renders, versions) + +def fetch_cryptopp(): + pkg = "cryptopp" + owner = "weidai11" + repo = "cryptopp" + release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" + resp = requests.get(release_url) + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; + };""") + renders = [] + versions = {} + for release in resp.json(): + if not release['draft'] and not release['prerelease']: + _, *version_values = release['tag_name'].split('_') + underscored_version = '_'.join(version_values) + flat_version = "v" + "".join(version_values) + download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz" + digest = get_source_hash(download_url, unpack=True) + print(f"{underscored_version}:{digest}") + + rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=underscored_version).strip() + renders.append(rendered) + versions[flat_version] = { + "version": underscored_version, + "hash": digest, + "sort": parse_version(underscored_version.replace("_", ".")) + } + serialize_versions(pkg, renders, versions) + +def fetch_openssl(): + pkg = "openssl" + owner = "openssl" + repo = "openssl" + release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" + resp_releases = requests.get(release_url) + tags_url = f"https://api.github.com/repos/{owner}/{repo}/git/matching-refs/tags" + resp_tags = requests.get(tags_url) + + tags = [release["tag_name"] for release in resp_releases.json() if not release["draft"] and not release["prerelease"]] + tags += [tag_ref["ref"].split("/")[-1] for tag_ref in resp_tags.json() if tag_ref["ref"].startswith("refs/tags/openssl-") or tag_ref["ref"].startswith("refs/tags/OpenSSL_")] + tags = list(filter(lambda tag: "FIPS" not in tag and "reformat" not in tag and "alpha" not in tag and "beta" not in tag and "pre" not in tag, tags)) + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; + };""") + renders = [] + versions = {} + for tag in tags: + if tag.startswith("OpenSSL_"): + match = re.match(r"OpenSSL_(?P\d+)_(?P\d+)_(?P\d+)(?P.*)", tag) + sort_version = f"{match['major']}.{match['minor']}.{match['patch']}{'+' + match['ext'] if match['ext'] else ''}" + dotted_version = f"{match['major']}.{match['minor']}.{match['patch']}{ match['ext'] if match['ext'] else ''}" + else: + try: + _, dotted_version = tag.split('-') + sort_version = dotted_version + except ValueError: + continue + flat_version = "v" + "".join(dotted_version.split('.')) + download_url = f"https://www.openssl.org/source/openssl-{dotted_version}.tar.gz" + old_url = f"https://www.openssl.org/source/old/openssl-{dotted_version}.tar.gz" + try: + digest = get_source_hash(download_url) + except Exception: + try: + digest = get_source_hash(old_url) + except Exception: + print(f"Skipping {dotted_version} (unavailable)") + continue + print(f"{dotted_version}:{digest}") + versions[flat_version] = { + "version": dotted_version, + "hash": digest, + "sort": parse_version(sort_version) + } + + rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=dotted_version).strip() + renders.append(rendered) + serialize_versions(pkg, renders, versions) + + +def fetch_tomcrypt(): + # fetch libtomcrypt + pass + +def fetch_gcrypt(): + pkg = "gcrypt" + release_list = "https://gnupg.org/ftp/gcrypt/libgcrypt/" + download_url = "https://gnupg.org/ftp/gcrypt/libgcrypt/{version}" + resp = requests.get(release_list) + soup = BeautifulSoup(resp.content, 'html.parser') + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; + };""") + + renders = [] + versions = {} + for link in soup.find_all("a"): + if link.text.startswith("libgcrypt") and link.text.endswith("tar.bz2"): + download_link = download_url.format(version=link['href']) + + match = re.match(r"libgcrypt-(?P\d+)\.(?P\d+)\.(?P\d+)(?P_do_not_use)?\.(?P.*)", link.text) + version = f"{match['major']}.{match['minor']}.{match['patch']}" + + digest = get_source_hash(download_link) + print(f"{version}:{digest}") + + flat_version = f"v{match['major']}{match['minor']}{match['patch']}" + if match['dont']: + flat_version += "_do_not_use" + + rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() + renders.append(rendered) + versions[flat_version] = { + "version": version, + "hash": digest, + "sort": parse_version(version) + } + serialize_versions(pkg, renders, versions) + +def fetch_boringssl(): + pkg = "boringssl" + upto = "76bb1411acf5cf6935586182a3a037d372ed1636" + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { rev="{{ rev }}"; hash="{{ digest }}"; }; + };""") + renders = [] + versions = {} + with tempfile.TemporaryDirectory() as repodir, tempfile.TemporaryDirectory() as gitdir: + repodir = pathlib.Path(repodir) + gitdir = pathlib.Path(gitdir) + sp.run(["git", "clone", "https://boringssl.googlesource.com/boringssl", repodir]) + # NOTE: we need to get rid of the .git so that it is not included in the derivation hash + shutil.move(repodir / ".git", gitdir) + + output = sp.check_output(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "log", "--pretty=format:%H"]) + refs = output.decode().split('\n') + + upto_index = refs.index(upto) + + # pick roughly 100 commits evenly spaced from the "upto" commit + for i, rev in enumerate(refs[upto_index:0:-40]): + sp.run(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "checkout", rev]) + digest = sp.check_output(["nix", "hash", "path", str(repodir)]).decode().strip() + print(f"{i + 1: 4d}:{rev}:{digest}") + abbrev_commit = str(rev[:8]) + + rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=f"r{abbrev_commit}", rev=rev).strip() + renders.append(rendered) + versions[f"r{abbrev_commit}"] = { + "rev": rev, + "hash": digest, + "sort": i + } + serialize_versions(pkg, renders, versions) + +def fetch_mbedtls(): + # Mbed-TLS/mbedtls + pkg = "mbedtls" + owner = "Mbed-TLS" + repo = "mbedtls" + release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" + resp = requests.get(release_url) + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; + };""") + renders = [] + versions = {} + for release in resp.json(): + if not release['draft'] and not release['prerelease']: + tag = release["tag_name"] + version = tag.replace("mbedtls-", "v") + flat_version = version.replace('.', '') + download_url = f"https://github.com/{owner}/{repo}/archive/{tag}.tar.gz" + if version == "v3.6.0": + # TODO: Special case for the time being + digest = "sha256-tCwAKoTvY8VCjcTPNwS3DeitflhpKHLr6ygHZDbR6wQ=" + else: + digest = get_source_hash(download_url, unpack=True) + + print(f"{version}:{digest}") + + rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() + renders.append(rendered) + versions[flat_version] = { + "version": version, + "hash": digest, + "tag": tag, + "sort": parse_version(version) + } + serialize_versions(pkg, renders, versions) + +def fetch_ippcp(): + # https://api.github.com/repos/intel/ipp-crypto/releases + pkg = "ippcp" + owner = "intel" + repo = "ipp-crypto" + release_url = f"https://api.github.com/repos/{owner}/{repo}/releases" + resp = requests.get(release_url) + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; + };""") + renders = [] + versions = {} + for release in resp.json(): + if not release['draft'] and not release['prerelease']: + version = release['tag_name'].split('_')[1] + flat_version = "v" + version.replace('.', '_') + download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz" + digest = get_source_hash(download_url, unpack=True) + print(f"{version}:{digest}") + + rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() + renders.append(rendered) + versions[flat_version] = { + "version": version, + "hash": digest, + "sort": parse_version(version.replace("u", "+u")) + } + serialize_versions(pkg, renders, versions) + +def fetch_nettle(): + # https://api.github.com/repos/intel/ipp-crypto/releases + pkg = "nettle" + owner = "gnutls" + repo = "nettle" + release_url = f"https://api.github.com/repos/{owner}/{repo}/tags" + resp = requests.get(release_url) + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; tag="{{ tag }}"; hash="{{ digest }}"; }; + };""") + renders = [] + versions = {} + for tag in resp.json(): + if tag['name'] == 'release_nettle_0.2.20010617': + continue + if tag['name'] == 'nettle_3.5_release_20190626': + # broken upstream! https://git.lysator.liu.se/nettle/nettle/-/commit/ee5d62898cf070f08beedc410a8d7c418588bd95 + continue + version = tag['name'].split('_')[1] + # NOTE skip release candidates + if re.search(r'\drc\d', version): + continue + flat_version = "v" + version.replace('.', '_') + # download_url = f"https://github.com/{owner}/{repo}/archive/{tag['name']}.tar.gz" + download_url = f"mirror://gnu/nettle/nettle-{version}.tar.gz" + digest = get_source_hash(download_url, unpack=False) + print(f"{version}:{digest}") + + rendered = single_version_template.render( + pkg=pkg, digest=digest, flat_version=flat_version, tag=tag['name'], version=version).strip() + renders.append(rendered) + versions[flat_version] = { + "version": version, + "tag": tag['name'], + "hash": digest, + "sort": parse_version(version) + } + serialize_versions(pkg, renders, versions) + + +def fetch_libressl(): + pkg = "libressl" + release_list = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/" + download_url = "mirror://openbsd/LibreSSL/libressl-{version}.tar.gz" + resp = requests.get(release_list) + soup = BeautifulSoup(resp.content, 'html.parser') + + single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone { + {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; }; + };""") + + renders = [] + versions = {} + for link in soup.find_all("a"): + if link.text.startswith("libressl") and link.text.endswith('.tar.gz'): + match = re.match(r"libressl-(?P\d+)\.(?P\d+)\.(?P\d+)\.tar.gz", link.text) + version = f"{match['major']}.{match['minor']}.{match['patch']}" + download_link = download_url.format(version=version) + digest = get_source_hash(download_link) + print(f"{version}:{digest}") + # NOTE: use underscore to separate the versions? + flat_version = f"v{match['major']}{match['minor']}{match['patch']}" + + rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip() + renders.append(rendered) + versions[flat_version] = { + "version": version, + "hash": digest, + "sort": parse_version(version) + } + serialize_versions(pkg, renders, versions) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("lib") + args = parser.parse_args() + + print(f"Fetching versions and source hashes for: {args.lib}") + + match args.lib: + case "botan": + fetch_botan() + case "cryptopp": + fetch_cryptopp() + case "openssl": + fetch_openssl() + case "boringssl": + fetch_boringssl() + case "gcrypt": + fetch_gcrypt() + case "mbedtls": + fetch_mbedtls() + case "ippcp": + fetch_ippcp() + case "nettle": + fetch_nettle() + case "libressl": + fetch_libressl() + case "all": + fetch_botan() + fetch_cryptopp() + fetch_openssl() + fetch_boringssl() + fetch_gcrypt() + fetch_mbedtls() + fetch_ippcp() + fetch_nettle() + fetch_libressl() + case _: + print("Unknown library") + + +if __name__ == '__main__': + main() diff --git a/nix/plot_versions.py b/nix/plot_versions.py new file mode 100644 index 0000000..012ec3f --- /dev/null +++ b/nix/plot_versions.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import argparse +import json + +from collections import defaultdict +from pathlib import Path + +import pandas as pd + +def get_all_versions(library): + with open(f"./nix/{library}_pkg_versions.json", "r") as handle: + versions = json.load(handle) + return versions + +def build_results_to_latex(library): + versions = get_all_versions(library) + lib_results = get_results(library, "lib") + lib_rows = [r"{\color{blue}\cmark}" if lib_results[ver]["success"] else r"{\color{red}\xmark}" for ver in versions.keys()] + + shim_results = get_results(library, "shim") + shim_rows = [r"{\color{blue}\cmark}" if shim_results[ver]["success"] else r"{\color{red}\xmark}" for ver in versions.keys()] + # shim_rows = [shim_results[ver] for ver in versions.keys()] + + cleaned_versions = [v.replace('_', r"{\_}") for v in versions.keys()] + df = pd.DataFrame(dict(Versions=cleaned_versions, Library=lib_rows, Shim=shim_rows)) + # FIXME there should be a translation from `openssl` -> `OpenSSL` etc. + tabledir = Path(f"./build_all/tables") + tabledir.mkdir(parents=True, exist_ok=True) + with open(tabledir / f"{library}.tex", "w") as handle: + handle.write(df.to_latex(index=False, caption=library, label=f"{library}-lib-and-shim-builds")) + +def get_results(library, variant): + with open(f"./build_all/{variant}/{library}.json", "r") as handle: + return json.load(handle) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-l", "--library") + # parser.add_argument("-v", "--variant", default="shim", type=valid_build_type) + args = parser.parse_args() + library = args.library + # variant = args.variant + + libraries = [ + "botan", + "cryptopp", + "openssl", + "boringssl", + "gcrypt", + "mbedtls", + "ippcp", + "nettle", + "libressl", + ] + + match library: + case None: + # print("Building all libraries") + # # Build all libraries by default + for lib in libraries: + build_results_to_latex(lib) + # print(f"Library: {lib}") + # for version in get_all_versions(lib): + # result = attempt_build(lib, version, variant) + # save_build_result(lib, variant, version, result) + # print(f"{version}: {result['success']}") + case lib if lib in libraries: + build_results_to_latex(lib) + # print(f"Library: {library}") + # for version in get_all_versions(library): + # result = attempt_build(lib, version, variant) + # save_build_result(lib, variant, version, result) + # print(f"{version}: {result['success']}") + case _: + pass + print(f"Unrecognized library '{library}'. Try one of: {', '.join(libraries)}.") + + +if __name__ == '__main__': + main() diff --git a/plot_versions.py b/plot_versions.py deleted file mode 100644 index 012ec3f..0000000 --- a/plot_versions.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json - -from collections import defaultdict -from pathlib import Path - -import pandas as pd - -def get_all_versions(library): - with open(f"./nix/{library}_pkg_versions.json", "r") as handle: - versions = json.load(handle) - return versions - -def build_results_to_latex(library): - versions = get_all_versions(library) - lib_results = get_results(library, "lib") - lib_rows = [r"{\color{blue}\cmark}" if lib_results[ver]["success"] else r"{\color{red}\xmark}" for ver in versions.keys()] - - shim_results = get_results(library, "shim") - shim_rows = [r"{\color{blue}\cmark}" if shim_results[ver]["success"] else r"{\color{red}\xmark}" for ver in versions.keys()] - # shim_rows = [shim_results[ver] for ver in versions.keys()] - - cleaned_versions = [v.replace('_', r"{\_}") for v in versions.keys()] - df = pd.DataFrame(dict(Versions=cleaned_versions, Library=lib_rows, Shim=shim_rows)) - # FIXME there should be a translation from `openssl` -> `OpenSSL` etc. - tabledir = Path(f"./build_all/tables") - tabledir.mkdir(parents=True, exist_ok=True) - with open(tabledir / f"{library}.tex", "w") as handle: - handle.write(df.to_latex(index=False, caption=library, label=f"{library}-lib-and-shim-builds")) - -def get_results(library, variant): - with open(f"./build_all/{variant}/{library}.json", "r") as handle: - return json.load(handle) - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-l", "--library") - # parser.add_argument("-v", "--variant", default="shim", type=valid_build_type) - args = parser.parse_args() - library = args.library - # variant = args.variant - - libraries = [ - "botan", - "cryptopp", - "openssl", - "boringssl", - "gcrypt", - "mbedtls", - "ippcp", - "nettle", - "libressl", - ] - - match library: - case None: - # print("Building all libraries") - # # Build all libraries by default - for lib in libraries: - build_results_to_latex(lib) - # print(f"Library: {lib}") - # for version in get_all_versions(lib): - # result = attempt_build(lib, version, variant) - # save_build_result(lib, variant, version, result) - # print(f"{version}: {result['success']}") - case lib if lib in libraries: - build_results_to_latex(lib) - # print(f"Library: {library}") - # for version in get_all_versions(library): - # result = attempt_build(lib, version, variant) - # save_build_result(lib, variant, version, result) - # print(f"{version}: {result['success']}") - case _: - pass - print(f"Unrecognized library '{library}'. Try one of: {', '.join(libraries)}.") - - -if __name__ == '__main__': - main() diff --git a/test_building_all.py b/test_building_all.py deleted file mode 100644 index 782f8b2..0000000 --- a/test_building_all.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python - -import argparse -import json -import time - -from pathlib import Path - -import subprocess as sp - -def get_all_versions(library): - with open(f"./nix/{library}_pkg_versions.json", "r") as handle: - versions = json.load(handle) - - return versions - -def attempt_build(library, version, variant): - cmd = ["nix", "build", f".#{variant}.{library}.{version}"] - start = time.time() - - result = {} - try: - sp.check_output(cmd, stderr=sp.STDOUT) - success = True - stderr = "" - except sp.CalledProcessError as e: - stderr = e.output.decode() - success = False - - result['build_time'] = time.time() - start - result['success'] = success - result['stderr'] = stderr.split('\n') if stderr else [] - - return result - -def valid_build_type(value): - value = value.strip() - valid_types = ["shim", "lib"] - if value not in valid_types: - raise argparse.ArgumentTypeError(f"'{value}' not from expected {', '.join(valid_types)}.") - return value - -def save_build_result(library, variant, version, result): - resdir = Path(f"build_all/{variant}") - resdir.mkdir(parents=True, exist_ok=True) - try: - # Update previous results - with open(resdir / f"{library}.json", "r") as handle: - prev_results = json.load(handle) - # NOTE this is not ideal as the JSON decoding problem can be other than just an empty file - except (FileNotFoundError, json.JSONDecodeError): - prev_results = {} - - prev_results[version] = result - with open(resdir / f"{library}.json", "w") as handle: - json.dump(prev_results, handle, indent=4) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-l", "--library") - parser.add_argument("-v", "--variant", default="shim", type=valid_build_type) - args = parser.parse_args() - library = args.library - variant = args.variant - - libraries = [ - "botan", - "cryptopp", - "openssl", - "boringssl", - "gcrypt", - "mbedtls", - "ippcp", - "nettle", - "libressl", - ] - - match library: - case None: - print("Building all libraries") - # Build all libraries by default - for lib in libraries: - print(f"Library: {lib}") - for version in get_all_versions(lib): - result = attempt_build(lib, version, variant) - save_build_result(lib, variant, version, result) - print(f"{version}: {result['success']}") - case lib if lib in libraries: - print(f"Library: {library}") - for version in get_all_versions(library): - result = attempt_build(lib, version, variant) - save_build_result(lib, variant, version, result) - print(f"{version}: {result['success']}") - case _: - print(f"Unrecognized library '{library}'. Try one of: {', '.join(libraries)}.") - - -if __name__ == '__main__': - main() -- cgit v1.3.1