1 files changed, 208 insertions, 86 deletions

diff --git a/nix/fetch_releases.py b/nix/fetch_releases.py
index 8cd8044..d8e5df5 100644
--- a/nix/fetch_releases.py
+++ b/nix/fetch_releases.py
@@ -18,12 +18,15 @@ from packaging.version import parse as parse_version, Version
 
 env = jinja2.Environment()
 
-all_versions_template = env.from_string("""{
+all_versions_template = env.from_string(
+    """{
   buildECTesterStandalone
 }:
 { {% for version in pkg_versions %}
   {{ version }} {% endfor %}
-}""")
+}"""
+)
+
 
 def get_source_hash(url, unpack=False):
     digest_type = "sha256"
@@ -34,11 +37,24 @@ def get_source_hash(url, unpack=False):
     cmd.extend(["--type", digest_type, url])
 
     digest_nixbase32 = sp.check_output(cmd, stderr=sp.DEVNULL).strip()
-    digest_sri = sp.check_output(["nix", "hash", "to-sri", "--type", digest_type, digest_nixbase32.decode()], stderr=sp.DEVNULL).strip().decode()
+    digest_sri = (
+        sp.check_output(
+            ["nix", "hash", "to-sri", "--type", digest_type, digest_nixbase32.decode()],
+            stderr=sp.DEVNULL,
+        )
+        .strip()
+        .decode()
+    )
     return digest_sri
 
+
 def serialize_versions(pkg, renders, versions):
-    sorted_versions = {k: {kk: vv for kk, vv in v.items() if kk != "sort"} for k, v in sorted(versions.items(), key=lambda item: item[1]["sort"], reverse=True)}
+    sorted_versions = {
+        k: {kk: vv for kk, vv in v.items() if kk != "sort"}
+        for k, v in sorted(
+            versions.items(), key=lambda item: item[1]["sort"], reverse=True
+        )
+    }
 
     # all_versions = all_versions_template.render(pkg_versions=renders).strip()
     # with open(f"./nix/{pkg}_pkg_versions.nix", "w") as handle:
@@ -47,26 +63,32 @@ def serialize_versions(pkg, renders, versions):
     with open(f"./nix/{pkg}_pkg_versions.json", "w") as handle:
         json.dump(sorted_versions, handle, indent=4)
 
+
 def fetch_botan():
     pkg = "botan"
     # NOTE: this way omits the older releases at https://botan.randombit.net/releases/old
     release_list = "https://botan.randombit.net/releases/"
     download_url = "https://botan.randombit.net/releases/{version}"
     resp = requests.get(release_list)
-    soup = BeautifulSoup(resp.content, 'html.parser')
+    soup = BeautifulSoup(resp.content, "html.parser")
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}"; source_extension="{{ ext }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
 
     renders = []
     versions = {}
     for link in soup.find_all("a"):
-        if link.text.startswith("Botan") and not link.text.endswith('.asc'):
-            download_link = download_url.format(version=link['href'])
+        if link.text.startswith("Botan") and not link.text.endswith(".asc"):
+            download_link = download_url.format(version=link["href"])
 
-            match = re.match(r"Botan-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\.(?P<ext>.*)", link.text)
-            if match['major'] == "3":
+            match = re.match(
+                r"Botan-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\.(?P<ext>.*)",
+                link.text,
+            )
+            if match["major"] == "3":
                 # TODO: Handle Botan-3
                 print(f"Skipping Botan-3 {match}")
                 continue
@@ -78,16 +100,23 @@ def fetch_botan():
             flat_version = f"v{match['major']}{match['minor']}{match['patch']}"
             print(f"{version}:{digest}")
 
-            rendered = single_version_template.render(pkg=pkg, digest=digest, ext=ext, flat_version=flat_version, version=version).strip()
+            rendered = single_version_template.render(
+                pkg=pkg,
+                digest=digest,
+                ext=ext,
+                flat_version=flat_version,
+                version=version,
+            ).strip()
             renders.append(rendered)
             versions[flat_version] = {
                 "version": version,
                 "source_extension": ext,
                 "hash": digest,
-                "sort": parse_version(version)
+                "sort": parse_version(version),
             }
     serialize_versions(pkg, renders, versions)
 
+
 def fetch_cryptopp():
     pkg = "cryptopp"
     owner = "weidai11"
@@ -95,29 +124,37 @@ def fetch_cryptopp():
     release_url = f"https://api.github.com/repos/{owner}/{repo}/releases"
     resp = requests.get(release_url)
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
     renders = []
     versions = {}
     for release in resp.json():
-        if not release['draft'] and not release['prerelease']:
-            _, *version_values = release['tag_name'].split('_')
-            underscored_version = '_'.join(version_values)
+        if not release["draft"] and not release["prerelease"]:
+            _, *version_values = release["tag_name"].split("_")
+            underscored_version = "_".join(version_values)
             flat_version = "v" + "".join(version_values)
             download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz"
             digest = get_source_hash(download_url, unpack=True)
             print(f"{underscored_version}:{digest}")
 
-            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=underscored_version).strip()
+            rendered = single_version_template.render(
+                pkg=pkg,
+                digest=digest,
+                flat_version=flat_version,
+                version=underscored_version,
+            ).strip()
             renders.append(rendered)
             versions[flat_version] = {
                 "version": underscored_version,
                 "hash": digest,
-                "sort": parse_version(underscored_version.replace("_", "."))
+                "sort": parse_version(underscored_version.replace("_", ".")),
             }
     serialize_versions(pkg, renders, versions)
 
+
 def fetch_openssl():
     pkg = "openssl"
     owner = "openssl"
@@ -127,27 +164,49 @@ def fetch_openssl():
     tags_url = f"https://api.github.com/repos/{owner}/{repo}/git/matching-refs/tags"
     resp_tags = requests.get(tags_url)
 
-    tags = [release["tag_name"] for release in resp_releases.json() if not release["draft"] and not release["prerelease"]]
-    tags += [tag_ref["ref"].split("/")[-1] for tag_ref in resp_tags.json() if tag_ref["ref"].startswith("refs/tags/openssl-") or tag_ref["ref"].startswith("refs/tags/OpenSSL_")]
-    tags = list(filter(lambda tag: "FIPS" not in tag and "reformat" not in tag and "alpha" not in tag and "beta" not in tag and "pre" not in tag, tags))    
+    tags = [
+        release["tag_name"]
+        for release in resp_releases.json()
+        if not release["draft"] and not release["prerelease"]
+    ]
+    tags += [
+        tag_ref["ref"].split("/")[-1]
+        for tag_ref in resp_tags.json()
+        if tag_ref["ref"].startswith("refs/tags/openssl-")
+        or tag_ref["ref"].startswith("refs/tags/OpenSSL_")
+    ]
+    tags = list(
+        filter(
+            lambda tag: "FIPS" not in tag
+            and "reformat" not in tag
+            and "alpha" not in tag
+            and "beta" not in tag
+            and "pre" not in tag,
+            tags,
+        )
+    )
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
     renders = []
     versions = {}
     for tag in tags:
         if tag.startswith("OpenSSL_"):
-            match = re.match(r"OpenSSL_(?P<major>\d+)_(?P<minor>\d+)_(?P<patch>\d+)(?P<ext>.*)", tag)
+            match = re.match(
+                r"OpenSSL_(?P<major>\d+)_(?P<minor>\d+)_(?P<patch>\d+)(?P<ext>.*)", tag
+            )
             sort_version = f"{match['major']}.{match['minor']}.{match['patch']}{'+' + match['ext'] if match['ext'] else ''}"
             dotted_version = f"{match['major']}.{match['minor']}.{match['patch']}{ match['ext'] if match['ext'] else ''}"
         else:
-            try: 
-                _, dotted_version = tag.split('-')
+            try:
+                _, dotted_version = tag.split("-")
                 sort_version = dotted_version
             except ValueError:
                 continue
-        flat_version = "v" + "".join(dotted_version.split('.'))
+        flat_version = "v" + "".join(dotted_version.split("."))
         download_url = f"https://www.openssl.org/source/openssl-{dotted_version}.tar.gz"
         old_url = f"https://www.openssl.org/source/old/openssl-{dotted_version}.tar.gz"
         try:
@@ -162,10 +221,12 @@ def fetch_openssl():
         versions[flat_version] = {
             "version": dotted_version,
             "hash": digest,
-            "sort": parse_version(sort_version)
+            "sort": parse_version(sort_version),
         }
 
-        rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=dotted_version).strip()
+        rendered = single_version_template.render(
+            pkg=pkg, digest=digest, flat_version=flat_version, version=dotted_version
+        ).strip()
         renders.append(rendered)
     serialize_versions(pkg, renders, versions)
 
@@ -174,60 +235,86 @@ def fetch_tomcrypt():
     # fetch libtomcrypt
     pass
 
+
 def fetch_gcrypt():
     pkg = "gcrypt"
     release_list = "https://gnupg.org/ftp/gcrypt/libgcrypt/"
     download_url = "https://gnupg.org/ftp/gcrypt/libgcrypt/{version}"
     resp = requests.get(release_list)
-    soup = BeautifulSoup(resp.content, 'html.parser')
+    soup = BeautifulSoup(resp.content, "html.parser")
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}";  hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
 
     renders = []
     versions = {}
     for link in soup.find_all("a"):
         if link.text.startswith("libgcrypt") and link.text.endswith("tar.bz2"):
-            download_link = download_url.format(version=link['href'])
+            download_link = download_url.format(version=link["href"])
 
-            match = re.match(r"libgcrypt-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<dont>_do_not_use)?\.(?P<ext>.*)", link.text)
+            match = re.match(
+                r"libgcrypt-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<dont>_do_not_use)?\.(?P<ext>.*)",
+                link.text,
+            )
             version = f"{match['major']}.{match['minor']}.{match['patch']}"
 
             digest = get_source_hash(download_link)
             print(f"{version}:{digest}")
 
             flat_version = f"v{match['major']}{match['minor']}{match['patch']}"
-            if match['dont']:
+            if match["dont"]:
                 flat_version += "_do_not_use"
 
-            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            rendered = single_version_template.render(
+                pkg=pkg, digest=digest, flat_version=flat_version, version=version
+            ).strip()
             renders.append(rendered)
             versions[flat_version] = {
                 "version": version,
                 "hash": digest,
-                "sort": parse_version(version)
+                "sort": parse_version(version),
             }
     serialize_versions(pkg, renders, versions)
 
+
 def fetch_boringssl():
     pkg = "boringssl"
-    upto = "76bb1411acf5cf6935586182a3a037d372ed1636"   
+    upto = "76bb1411acf5cf6935586182a3a037d372ed1636"
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { rev="{{ rev }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
     renders = []
     versions = {}
-    with tempfile.TemporaryDirectory() as repodir, tempfile.TemporaryDirectory() as gitdir:
+    with (
+        tempfile.TemporaryDirectory() as repodir,
+        tempfile.TemporaryDirectory() as gitdir,
+    ):
         repodir = pathlib.Path(repodir)
         gitdir = pathlib.Path(gitdir)
-        sp.run(["git", "clone", "https://boringssl.googlesource.com/boringssl", repodir])
+        sp.run(
+            ["git", "clone", "https://boringssl.googlesource.com/boringssl", repodir]
+        )
         # NOTE: we need to get rid of the .git so that it is not included in the derivation hash
         shutil.move(repodir / ".git", gitdir)
 
-        output = sp.check_output(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "log", "--pretty=format:%H"])
-        refs = output.decode().split('\n')
+        output = sp.check_output(
+            [
+                "git",
+                "-C",
+                str(repodir),
+                "--git-dir",
+                str(gitdir / ".git"),
+                "log",
+                "--pretty=format:%H",
+            ]
+        )
+        refs = output.decode().split("\n")
 
         upto_index = refs.index(upto)
 
@@ -235,20 +322,31 @@ def fetch_boringssl():
         # this picks out roughly 100 commits as of January 2025, but will grow over time
 
         for i, rev in enumerate(refs[upto_index:0:-40]):
-            sp.run(["git", "-C", str(repodir), "--git-dir", str(gitdir / ".git"), "checkout", rev])
-            digest = sp.check_output(["nix", "hash", "path", str(repodir)]).decode().strip()
+            sp.run(
+                [
+                    "git",
+                    "-C",
+                    str(repodir),
+                    "--git-dir",
+                    str(gitdir / ".git"),
+                    "checkout",
+                    rev,
+                ]
+            )
+            digest = (
+                sp.check_output(["nix", "hash", "path", str(repodir)]).decode().strip()
+            )
             print(f"{i + 1: 4d}:{rev}:{digest}")
             abbrev_commit = str(rev[:8])
 
-            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=f"r{abbrev_commit}", rev=rev).strip()
+            rendered = single_version_template.render(
+                pkg=pkg, digest=digest, flat_version=f"r{abbrev_commit}", rev=rev
+            ).strip()
             renders.append(rendered)
-            versions[f"r{abbrev_commit}"] = {
-                "rev": rev,
-                "hash": digest,
-                "sort": i
-            }
+            versions[f"r{abbrev_commit}"] = {"rev": rev, "hash": digest, "sort": i}
     serialize_versions(pkg, renders, versions)
 
+
 def fetch_mbedtls():
     # Mbed-TLS/mbedtls
     pkg = "mbedtls"
@@ -257,16 +355,18 @@ def fetch_mbedtls():
     release_url = f"https://api.github.com/repos/{owner}/{repo}/releases"
     resp = requests.get(release_url)
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
     renders = []
     versions = {}
     for release in resp.json():
-        if not release['draft'] and not release['prerelease']:
+        if not release["draft"] and not release["prerelease"]:
             tag = release["tag_name"]
             version = tag.replace("mbedtls-", "v")
-            flat_version = version.replace('.', '')
+            flat_version = version.replace(".", "")
             download_url = f"https://github.com/{owner}/{repo}/archive/{tag}.tar.gz"
             # TODO: Special case for the time being, not sure what is broken.
             if version == "v3.6.0":
@@ -280,16 +380,19 @@ def fetch_mbedtls():
 
             print(f"{version}:{digest}")
 
-            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            rendered = single_version_template.render(
+                pkg=pkg, digest=digest, flat_version=flat_version, version=version
+            ).strip()
             renders.append(rendered)
             versions[flat_version] = {
                 "version": version,
                 "hash": digest,
                 "tag": tag,
-                "sort": parse_version(version)
+                "sort": parse_version(version),
             }
     serialize_versions(pkg, renders, versions)
 
+
 def fetch_ippcp():
     # https://api.github.com/repos/intel/ipp-crypto/releases
     pkg = "ippcp"
@@ -298,34 +401,39 @@ def fetch_ippcp():
     release_url = f"https://api.github.com/repos/{owner}/{repo}/releases"
     resp = requests.get(release_url)
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
     renders = []
     versions = {}
     for release in resp.json():
-        if not release['draft'] and not release['prerelease']:
+        if not release["draft"] and not release["prerelease"]:
             if "_" in release["tag_name"]:
-                version = release['tag_name'].split('_')[1]
-                flat_version = "v" + version.replace('.', '_')
+                version = release["tag_name"].split("_")[1]
+                flat_version = "v" + version.replace(".", "_")
             else:
                 version = release["tag_name"]
-                flat_version = version.replace('.', '_')
+                flat_version = version.replace(".", "_")
             parsed = parse_version(version.replace("u", "+u"))
 
             download_url = f"https://github.com/{owner}/{repo}/archive/{release['tag_name']}.tar.gz"
             digest = get_source_hash(download_url, unpack=True)
             print(f"{version}:{digest}")
 
-            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            rendered = single_version_template.render(
+                pkg=pkg, digest=digest, flat_version=flat_version, version=version
+            ).strip()
             renders.append(rendered)
             versions[flat_version] = {
                 "version": version,
                 "hash": digest,
-                "sort": (1, parsed) if parsed.major < 2000 else (0, parsed)
+                "sort": (1, parsed) if parsed.major < 2000 else (0, parsed),
             }
     serialize_versions(pkg, renders, versions)
 
+
 def fetch_nettle():
     # https://api.github.com/repos/intel/ipp-crypto/releases
     pkg = "nettle"
@@ -334,35 +442,42 @@ def fetch_nettle():
     release_url = f"https://api.github.com/repos/{owner}/{repo}/tags"
     resp = requests.get(release_url)
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}"; tag="{{ tag }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
     renders = []
     versions = {}
     for tag in resp.json():
-        if tag['name'] == 'release_nettle_0.2.20010617':
+        if tag["name"] == "release_nettle_0.2.20010617":
             continue
-        if tag['name'] == 'nettle_3.5_release_20190626':
+        if tag["name"] == "nettle_3.5_release_20190626":
             # broken upstream! https://git.lysator.liu.se/nettle/nettle/-/commit/ee5d62898cf070f08beedc410a8d7c418588bd95
             continue
-        version = tag['name'].split('_')[1]
+        version = tag["name"].split("_")[1]
         # NOTE skip release candidates
-        if re.search(r'\drc\d', version):
+        if re.search(r"\drc\d", version):
             continue
-        flat_version = "v" + version.replace('.', '_')
+        flat_version = "v" + version.replace(".", "_")
         # download_url = f"https://github.com/{owner}/{repo}/archive/{tag['name']}.tar.gz"
         download_url = f"mirror://gnu/nettle/nettle-{version}.tar.gz"
         digest = get_source_hash(download_url, unpack=False)
         print(f"{version}:{digest}")
 
         rendered = single_version_template.render(
-                pkg=pkg, digest=digest, flat_version=flat_version, tag=tag['name'], version=version).strip()
+            pkg=pkg,
+            digest=digest,
+            flat_version=flat_version,
+            tag=tag["name"],
+            version=version,
+        ).strip()
         renders.append(rendered)
         versions[flat_version] = {
             "version": version,
-            "tag": tag['name'],
+            "tag": tag["name"],
             "hash": digest,
-            "sort": parse_version(version)
+            "sort": parse_version(version),
         }
     serialize_versions(pkg, renders, versions)
 
@@ -372,17 +487,22 @@ def fetch_libressl():
     release_list = "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/"
     download_url = "mirror://openbsd/LibreSSL/libressl-{version}.tar.gz"
     resp = requests.get(release_list)
-    soup = BeautifulSoup(resp.content, 'html.parser')
+    soup = BeautifulSoup(resp.content, "html.parser")
 
-    single_version_template = env.from_string("""{{ flat_version }} = buildECTesterStandalone {
+    single_version_template = env.from_string(
+        """{{ flat_version }} = buildECTesterStandalone {
     {{ pkg }} = { version="{{ version }}"; hash="{{ digest }}"; };
-  };""")
+  };"""
+    )
 
     renders = []
     versions = {}
     for link in soup.find_all("a"):
-        if link.text.startswith("libressl") and link.text.endswith('.tar.gz'):
-            match = re.match(r"libressl-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\.tar.gz", link.text)
+        if link.text.startswith("libressl") and link.text.endswith(".tar.gz"):
+            match = re.match(
+                r"libressl-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\.tar.gz",
+                link.text,
+            )
             version = f"{match['major']}.{match['minor']}.{match['patch']}"
             download_link = download_url.format(version=version)
             digest = get_source_hash(download_link)
@@ -390,12 +510,14 @@ def fetch_libressl():
             # NOTE: use underscore to separate the versions?
             flat_version = f"v{match['major']}{match['minor']}{match['patch']}"
 
-            rendered = single_version_template.render(pkg=pkg, digest=digest, flat_version=flat_version, version=version).strip()
+            rendered = single_version_template.render(
+                pkg=pkg, digest=digest, flat_version=flat_version, version=version
+            ).strip()
             renders.append(rendered)
             versions[flat_version] = {
                 "version": version,
                 "hash": digest,
-                "sort": parse_version(version)
+                "sort": parse_version(version),
             }
     serialize_versions(pkg, renders, versions)
 
@@ -440,5 +562,5 @@ def main():
             print("Unknown library")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()