3 files changed, 613 insertions, 0 deletions

diff --git a/nix/build_all.py b/nix/build_all.py
new file mode 100644
index 0000000..782f8b2
--- /dev/null
+++ b/nix/build_all.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import time
+
+from pathlib import Path
+
+import subprocess as sp
+
+def get_all_versions(library):
+    with open(f"./nix/{library}_pkg_versions.json", "r") as handle:
+        versions = json.load(handle)
+
+    return versions
+
+def attempt_build(library, version, variant):
+    cmd = ["nix", "build", f".#{variant}.{library}.{version}"]
+    start = time.time()
+
+    result = {}
+    try:
+        sp.check_output(cmd, stderr=sp.STDOUT)
+        success = True
+        stderr = ""
+    except sp.CalledProcessError as e:
+        stderr = e.output.decode()
+        success = False
+
+    result['build_time'] = time.time() - start
+    result['success'] = success
+    result['stderr'] = stderr.split('\n') if stderr else []
+
+    return result
+
+def valid_build_type(value):
+    value = value.strip()
+    valid_types = ["shim", "lib"]
+    if value not in valid_types:
+        raise argparse.ArgumentTypeError(f"'{value}' not from expected {', '.join(valid_types)}.")
+    return value
+
+def save_build_result(library, variant, version, result):
+    resdir = Path(f"build_all/{variant}")
+    resdir.mkdir(parents=True, exist_ok=True)
+    try:
+        # Update previous results
+        with open(resdir / f"{library}.json", "r") as handle:
+            prev_results = json.load(handle)
+    # NOTE this is not ideal as the JSON decoding problem can be other than just an empty file
+    except (FileNotFoundError, json.JSONDecodeError):
+        prev_results = {}
+
+    prev_results[version] = result
+    with open(resdir / f"{library}.json", "w") as handle:
+        json.dump(prev_results, handle, indent=4)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-l", "--library")
+    parser.add_argument("-v", "--variant", default="shim", type=valid_build_type)
+    args = parser.parse_args()
+    library = args.library
+    variant = args.variant
+
+    libraries = [
+         "botan", 
+         "cryptopp", 
+         "openssl", 
+         "boringssl", 
+         "gcrypt", 
+         "mbedtls", 
+         "ippcp", 
+         "nettle", 
+         "libressl", 
+    ]
+
+    match library:
+        case None:
+            print("Building all libraries")
+            # Build all libraries by default
+            for lib in libraries:
+                print(f"Library: {lib}")
+                for version in get_all_versions(lib):
+                    result = attempt_build(lib, version, variant)
+                    save_build_result(lib, variant, version, result)
+                    print(f"{version}: {result['success']}")
+        case lib if lib in libraries:
+            print(f"Library: {library}")
+            for version in get_all_versions(library):
+                result = attempt_build(lib, version, variant)
+                save_build_result(lib, variant, version, result)
+                print(f"{version}: {result['success']}")
+        case _:
+            print(f"Unrecognized library '{library}'. Try one of: {', '.join(libraries)}.")
+
+
+if __name__ == '__main__':
+    main()
diff --git a/nix/fetch_releases.py b/nix/fetch_releases.py
new file mode 100644
index 0000000..e6f03b5
--- /dev/null
+++ b/nix/fetch_releases.py
@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+
+import argparse
+
+import json
+import jinja2
+import re
+import requests
+import shutil
+import tempfile
+
+import pathlib
+import subprocess as sp
+
+from base64 import b32encode, b32decode, b64encode, b16decode
+from bs4 import BeautifulSoup
+from packaging.version import parse as parse_version, Version
+
+env = jinja2.Environment()
+
+all_versions_template = env.from_string("""{
+  buildECTesterStandalone
+}:
+{ {% for version in pkg_versions %}
+  {{ version }} {% endfor %}
+}""")
+
+def get_source_hash(url, unpack=False):
+    digest_type = "sha256"
+
+    cmd = ["nix-prefetch-url"]
+    if unpack:
+        cmd.append("--unpack")
+    cmd.extend(["--type", digest_type, url])
+
+    digest_nixbase32 = sp.check_output(cmd, stderr=sp.DEVNULL).strip()
+    digest_sri = sp.check_output(["nix", "hash", "to-sri", "--type", digest_type, digest_nixbase32.decode()], stderr=sp.DEVNULL).strip().decode()
+    return digest_sri
+
+def serialize_versions(pkg, renders, versions):
+    sorted_versions = {k: {kk: vv for kk, vv in v.items() if kk != "sort"} for k, v in sorted(versions.items(), key=lambda item: item[1]["sort"], reverse=True)}
+
+    # all_versions = all_versions_template.render(pkg_versions=renders).strip()
+    # with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle:
+    #     handle.write(all_versions)
+
+    with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle:
+        json.dump(sorted_versions, handle, indent=4)
+
+def fetch_botan():
+    pkg = "botan"
+    # NOTE: this way omits the older releases at https://botan.randombit.net/releases/old
+    release_list = "https://botan.randombit.net/releases/"
+    download_url = "https://botan.randombit.net/releases/{version}"
+    resp = requests.get(release_list)
+    soup = BeautifulSoup(resp.content, 'html.parser')
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}"; source_extension="{{ ext }}"; hash="{{ digest }}"; };
+  };""")
+
+    renders = []
+    versions = {}
+    for link in soup.find_all("a"):
+        if link.text.startswith("Botan") and not link.text.endswith('.asc'):
+            download_link = download_url.format(version=link['href'])
+
+            match = re.match(r"Botan-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\.(?P<ext>.*)", link.text)
+            if match['major'] == "3":
+                # TODO: Handle Botan-3
+                print(f"Skipping Botan-3 {match}")
+                continue
+            version = f"{match['major']}.{match['minor']}.{match['patch']}"
+            ext = f"{match['ext']}"
+
+            digest = get_source_hash(download_link)
+            # NOTE: use underscore to separate the versions?
+            flat_version = f"v{match['major']}{match['minor']}{match['patch']}"
+            print(f"{version}:{digest}")
+
+            rendered = single_version_template.render(pkg=pkg, digest=digest, ext=ext, flat_version=flat_version, version=version).strip()
+            renders.append(rendered)
+            versions[flat_version] = {
+                "version": version,
+                "source_extension": ext,
+                "hash": digest,
+                "sort": parse_version(version)
+            }
+    serialize_versions(pkg, renders, versions)
+
+def fetch_cryptopp():
+    pkg = "cryptopp"
+    owner = "weidai11"
+    repo = "cryptopp"
+    release_url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+    resp = requests.get(release_url)
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
+  };""")
+    renders = []
+    versions = {}
+    for release in resp.json():
+        if not release['draft'] and not release['prerelease']:
+            _, *version_values = release['tag_name'].split('_')
+            underscored_version = '_'.join(version_values)
+            flat_version = "v" + "".join(version_values)
+            download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz"
+            digest = get_source_hash(download_url, unpack=True)
+            print(f"{underscored_version}:{digest}")
+
+            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=underscored_version).strip()
+            renders.append(rendered)
+            versions[flat_version] = {
+                "version": underscored_version,
+                "hash": digest,
+                "sort": parse_version(underscored_version.replace("_", "."))
+            }
+    serialize_versions(pkg, renders, versions)
+
+def fetch_openssl():
+    pkg = "openssl"
+    owner = "openssl"
+    repo = "openssl"
+    release_url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+    resp_releases = requests.get(release_url)
+    tags_url = f"https://api.github.com/repos/{owner}/{repo}/git/matching-refs/tags"
+    resp_tags = requests.get(tags_url)
+
+    tags = [release["tag_name"] for release in resp_releases.json() if not release["draft"] and not release["prerelease"]]
+    tags += [tag_ref["ref"].split("/")[-1] for tag_ref in resp_tags.json() if tag_ref["ref"].startswith("refs/tags/openssl-") or tag_ref["ref"].startswith("refs/tags/OpenSSL_")]
+    tags = list(filter(lambda tag: "FIPS" not in tag and "reformat" not in tag and "alpha" not in tag and "beta" not in tag and "pre" not in tag, tags))    
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
+  };""")
+    renders = []
+    versions = {}
+    for tag in tags:
+        if tag.startswith("OpenSSL_"):
+            match = re.match(r"OpenSSL_(?P<major>\d+)_(?P<minor>\d+)_(?P<patch>\d+)(?P<ext>.*)", tag)
+            sort_version = f"{match['major']}.{match['minor']}.{match['patch']}{'+' + match['ext'] if match['ext'] else ''}"
+            dotted_version = f"{match['major']}.{match['minor']}.{match['patch']}{ match['ext'] if match['ext'] else ''}"
+        else:
+            try: 
+                _, dotted_version = tag.split('-')
+                sort_version = dotted_version
+            except ValueError:
+                continue
+        flat_version = "v" + "".join(dotted_version.split('.'))
+        download_url = f"https://www.openssl.org/source/openssl-{dotted_version}.tar.gz"
+        old_url = f"https://www.openssl.org/source/old/openssl-{dotted_version}.tar.gz"
+        try:
+            digest = get_source_hash(download_url)
+        except Exception:
+            try:
+                digest = get_source_hash(old_url)
+            except Exception:
+                print(f"Skipping {dotted_version} (unavailable)")
+                continue
+        print(f"{dotted_version}:{digest}")
+        versions[flat_version] = {
+            "version": dotted_version,
+            "hash": digest,
+            "sort": parse_version(sort_version)
+        }
+
+        rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=dotted_version).strip()
+        renders.append(rendered)
+    serialize_versions(pkg, renders, versions)
+
+
+def fetch_tomcrypt():
+    # fetch libtomcrypt
+    pass
+
+def fetch_gcrypt():
+    pkg = "gcrypt"
+    release_list = "https://gnupg.org/ftp/gcrypt/libgcrypt/"
+    download_url = "https://gnupg.org/ftp/gcrypt/libgcrypt/{version}"
+    resp = requests.get(release_list)
+    soup = BeautifulSoup(resp.content, 'html.parser')
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}";  hash="{{ digest }}"; };
+  };""")
+
+    renders = []
+    versions = {}
+    for link in soup.find_all("a"):
+        if link.text.startswith("libgcrypt") and link.text.endswith("tar.bz2"):
+            download_link = download_url.format(version=link['href'])
+
+            match = re.match(r"libgcrypt-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<dont>_do_not_use)?\.(?P<ext>.*)", link.text)
+            version = f"{match['major']}.{match['minor']}.{match['patch']}"
+
+            digest = get_source_hash(download_link)
+            print(f"{version}:{digest}")
+
+            flat_version = f"v{match['major']}{match['minor']}{match['patch']}"
+            if match['dont']:
+                flat_version += "_do_not_use"
+
+            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            renders.append(rendered)
+            versions[flat_version] = {
+                "version": version,
+                "hash": digest,
+                "sort": parse_version(version)
+            }
+    serialize_versions(pkg, renders, versions)
+
+def fetch_boringssl():
+    pkg = "boringssl"
+    upto = "76bb1411acf5cf6935586182a3a037d372ed1636"
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { rev="{{ rev }}"; hash="{{ digest }}"; };
+  };""")
+    renders = []
+    versions = {}
+    with tempfile.TemporaryDirectory() as repodir, tempfile.TemporaryDirectory() as gitdir:
+        repodir = pathlib.Path(repodir)
+        gitdir = pathlib.Path(gitdir)
+        sp.run(["git", "clone", "https://boringssl.googlesource.com/boringssl", repodir])
+        # NOTE: we need to get rid of the .git so that it is not included in the derivation hash
+        shutil.move(repodir / ".git", gitdir)
+
+        output = sp.check_output(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "log", "--pretty=format:%H"])
+        refs = output.decode().split('\n')
+
+        upto_index = refs.index(upto)
+
+        # pick roughly 100 commits evenly spaced from the "upto" commit
+        for i, rev in enumerate(refs[upto_index:0:-40]):
+            sp.run(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "checkout", rev])
+            digest = sp.check_output(["nix", "hash", "path", str(repodir)]).decode().strip()
+            print(f"{i + 1: 4d}:{rev}:{digest}")
+            abbrev_commit = str(rev[:8])
+
+            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=f"r{abbrev_commit}", rev=rev).strip()
+            renders.append(rendered)
+            versions[f"r{abbrev_commit}"] = {
+                "rev": rev,
+                "hash": digest,
+                "sort": i
+            }
+    serialize_versions(pkg, renders, versions)
+
+def fetch_mbedtls():
+    # Mbed-TLS/mbedtls
+    pkg = "mbedtls"
+    owner = "Mbed-TLS"
+    repo = "mbedtls"
+    release_url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+    resp = requests.get(release_url)
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
+  };""")
+    renders = []
+    versions = {}
+    for release in resp.json():
+        if not release['draft'] and not release['prerelease']:
+            tag = release["tag_name"]
+            version = tag.replace("mbedtls-", "v")
+            flat_version = version.replace('.', '')
+            download_url = f"https://github.com/{owner}/{repo}/archive/{tag}.tar.gz"
+            if version == "v3.6.0":
+                # TODO: Special case for the time being
+                digest = "sha256-tCwAKoTvY8VCjcTPNwS3DeitflhpKHLr6ygHZDbR6wQ="
+            else:
+                digest = get_source_hash(download_url, unpack=True)
+
+            print(f"{version}:{digest}")
+
+            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            renders.append(rendered)
+            versions[flat_version] = {
+                "version": version,
+                "hash": digest,
+                "tag": tag,
+                "sort": parse_version(version)
+            }
+    serialize_versions(pkg, renders, versions)
+
+def fetch_ippcp():
+    # https://api.github.com/repos/intel/ipp-crypto/releases
+    pkg = "ippcp"
+    owner = "intel"
+    repo = "ipp-crypto"
+    release_url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+    resp = requests.get(release_url)
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
+  };""")
+    renders = []
+    versions = {}
+    for release in resp.json():
+        if not release['draft'] and not release['prerelease']:
+            version = release['tag_name'].split('_')[1]
+            flat_version = "v" + version.replace('.', '_')
+            download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz"
+            digest = get_source_hash(download_url, unpack=True)
+            print(f"{version}:{digest}")
+
+            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            renders.append(rendered)
+            versions[flat_version] = {
+                "version": version,
+                "hash": digest,
+                "sort": parse_version(version.replace("u", "+u"))
+            }
+    serialize_versions(pkg, renders, versions)
+
+def fetch_nettle():
+    # https://api.github.com/repos/intel/ipp-crypto/releases
+    pkg = "nettle"
+    owner = "gnutls"
+    repo = "nettle"
+    release_url = f"https://api.github.com/repos/{owner}/{repo}/tags"
+    resp = requests.get(release_url)
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}"; tag="{{ tag }}"; hash="{{ digest }}"; };
+  };""")
+    renders = []
+    versions = {}
+    for tag in resp.json():
+        if tag['name'] == 'release_nettle_0.2.20010617':
+            continue
+        if tag['name'] == 'nettle_3.5_release_20190626':
+            # broken upstream! https://git.lysator.liu.se/nettle/nettle/-/commit/ee5d62898cf070f08beedc410a8d7c418588bd95
+            continue
+        version = tag['name'].split('_')[1]
+        # NOTE skip release candidates
+        if re.search(r'\drc\d', version):
+            continue
+        flat_version = "v" + version.replace('.', '_')
+        # download_url = f"https://github.com/{owner}/{repo}/archive/{tag['name']}.tar.gz"
+        download_url = f"mirror://gnu/nettle/nettle-{version}.tar.gz"
+        digest = get_source_hash(download_url, unpack=False)
+        print(f"{version}:{digest}")
+
+        rendered = single_version_template.render(
+                pkg=pkg, digest=digest, flat_version=flat_version, tag=tag['name'], version=version).strip()
+        renders.append(rendered)
+        versions[flat_version] = {
+            "version": version,
+            "tag": tag['name'],
+            "hash": digest,
+            "sort": parse_version(version)
+        }
+    serialize_versions(pkg, renders, versions)
+
+
+def fetch_libressl():
+    pkg = "libressl"
+    release_list = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/"
+    download_url = "mirror://openbsd/LibreSSL/libressl-{version}.tar.gz"
+    resp = requests.get(release_list)
+    soup = BeautifulSoup(resp.content, 'html.parser')
+
+    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
+  };""")
+
+    renders = []
+    versions = {}
+    for link in soup.find_all("a"):
+        if link.text.startswith("libressl") and link.text.endswith('.tar.gz'):
+            match = re.match(r"libressl-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\.tar.gz", link.text)
+            version = f"{match['major']}.{match['minor']}.{match['patch']}"
+            download_link = download_url.format(version=version)
+            digest = get_source_hash(download_link)
+            print(f"{version}:{digest}")
+            # NOTE: use underscore to separate the versions?
+            flat_version = f"v{match['major']}{match['minor']}{match['patch']}"
+
+            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            renders.append(rendered)
+            versions[flat_version] = {
+                "version": version,
+                "hash": digest,
+                "sort": parse_version(version)
+            }
+    serialize_versions(pkg, renders, versions)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("lib")
+    args = parser.parse_args()
+
+    print(f"Fetching versions and source hashes for: {args.lib}")
+
+    match args.lib:
+        case "botan":
+            fetch_botan()
+        case "cryptopp":
+            fetch_cryptopp()
+        case "openssl":
+            fetch_openssl()
+        case "boringssl":
+            fetch_boringssl()
+        case "gcrypt":
+            fetch_gcrypt()
+        case "mbedtls":
+            fetch_mbedtls()
+        case "ippcp":
+            fetch_ippcp()
+        case "nettle":
+            fetch_nettle()
+        case "libressl":
+            fetch_libressl()
+        case "all":
+            fetch_botan()
+            fetch_cryptopp()
+            fetch_openssl()
+            fetch_boringssl()
+            fetch_gcrypt()
+            fetch_mbedtls()
+            fetch_ippcp()
+            fetch_nettle()
+            fetch_libressl()
+        case _:
+            print("Unknown library")
+
+
+if __name__ == '__main__':
+    main()
diff --git a/nix/plot_versions.py b/nix/plot_versions.py
new file mode 100644
index 0000000..012ec3f
--- /dev/null
+++ b/nix/plot_versions.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+
+from collections import defaultdict
+from pathlib import Path
+
+import pandas as pd
+
+def get_all_versions(library):
+    with open(f"./nix/{library}_pkg_versions.json", "r") as handle:
+        versions = json.load(handle)
+    return versions
+
+def build_results_to_latex(library):
+    versions = get_all_versions(library)
+    lib_results = get_results(library, "lib")
+    lib_rows = [r"{\color{blue}\cmark}" if lib_results[ver]["success"] else r"{\color{red}\xmark}" for ver in versions.keys()]
+
+    shim_results = get_results(library, "shim")
+    shim_rows = [r"{\color{blue}\cmark}" if shim_results[ver]["success"] else r"{\color{red}\xmark}" for ver in versions.keys()]
+    # shim_rows = [shim_results[ver] for ver in versions.keys()]
+
+    cleaned_versions = [v.replace('_', r"{\_}") for v in versions.keys()]
+    df = pd.DataFrame(dict(Versions=cleaned_versions, Library=lib_rows, Shim=shim_rows))
+    # FIXME there should be a translation from `openssl` -> `OpenSSL` etc.
+    tabledir = Path(f"./build_all/tables")
+    tabledir.mkdir(parents=True, exist_ok=True)
+    with open(tabledir / f"{library}.tex", "w") as handle:
+        handle.write(df.to_latex(index=False, caption=library, label=f"{library}-lib-and-shim-builds"))
+
+def get_results(library, variant):
+    with open(f"./build_all/{variant}/{library}.json", "r") as handle:
+        return json.load(handle)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-l", "--library")
+    # parser.add_argument("-v", "--variant", default="shim", type=valid_build_type)
+    args = parser.parse_args()
+    library = args.library
+    # variant = args.variant
+
+    libraries = [
+         "botan",
+         "cryptopp",
+         "openssl",
+         "boringssl",
+         "gcrypt",
+         "mbedtls",
+         "ippcp",
+         "nettle",
+         "libressl",
+    ]
+
+    match library:
+        case None:
+            # print("Building all libraries")
+            # # Build all libraries by default
+            for lib in libraries:
+                build_results_to_latex(lib)
+            #     print(f"Library: {lib}")
+            #     for version in get_all_versions(lib):
+            #         result = attempt_build(lib, version, variant)
+            #         save_build_result(lib, variant, version, result)
+            #         print(f"{version}: {result['success']}")
+        case lib if lib in libraries:
+            build_results_to_latex(lib)
+            # print(f"Library: {library}")
+            # for version in get_all_versions(library):
+            #     result = attempt_build(lib, version, variant)
+            #     save_build_result(lib, variant, version, result)
+            #     print(f"{version}: {result['success']}")
+        case _:
+            pass
+            print(f"Unrecognized library '{library}'. Try one of: {', '.join(libraries)}.")
+
+
+if __name__ == '__main__':
+    main()