Merge pull request #322 from crocs-muni/reference-analysis

Reference analysis

81 files changed, 7512 insertions, 2256 deletions

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index d3ac2afe..038c9eeb 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -6,14 +6,16 @@ on:
     types: [published]
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+      - name: apt-get update
+        run: sudo apt-get update
       - name: Install external dependencies
         run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
       - name: Install sec-certs and deps
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index 3524329a..6855dc63 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -4,12 +4,14 @@ on:
   workflow_dispatch:
 jobs:
   pre-commit:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+      - name: apt-get update
+        run: sudo apt-get update
       - name: Install dependencies
         run: |
           sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 70fcc839..1ab6081f 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -6,7 +6,7 @@ on:
 jobs:
   pypi_release:
     name: Release on PyPi
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: github.repository == 'crocs-muni/sec-certs'
     environment:
       name: pypi
@@ -14,13 +14,15 @@ jobs:
     permissions:
       id-token: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+      - name: apt-get update
+        run: sudo apt-get update
       - name: Install build dependencies
         run: python -m pip install build
       - name: Build distributions
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index ccda7c3f..51b13b90 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -9,9 +9,11 @@ jobs:
   test:
     runs-on: ubuntu-22.04
     steps:
+      - name: apt-get update
+        run: sudo apt-get update
       - name: Install Poppler
         run: sudo apt-get install -y build-essential libpoppler-cpp-dev pkg-config python3-dev
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup python
         uses: actions/setup-python@v4
         with:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e9db8430..c4c31049 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,3 +14,4 @@ repos:
           - "types-PyYAML"
           - "types-python-dateutil"
           - "types-requests"
+          - "datasets"
diff --git a/data/readme.md b/data/readme.md
deleted file mode 100644
index f1580e7e..00000000
--- a/data/readme.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# Data
-
-The file [label_studio_interface.txt](label_studio_interface.txt) contains XML-like specification of the labeling interface
-for CPE matching. As such, it was used in the Label studio tool.
-
-## Certificate ID evaluation
-
-The directory `./cert_id_eval` contains data on a manual evaluation of certificate ID assignment.
-`missing_ids.csv` contains an evaluation of certificates to which the sec-certs tool was not able to
-find a certificate ID (to analyze why that happened and whether we could fix that).
-`duplicate_ids.csv` contains an evaluation of certificates to which the sec-certs tool assigned a duplicate
-ID (to analyze why that happened and whether we could fix that). These files are used by the `cert_id_eval.ipynb`
-Jupyter notebook which evaluates a dataset with respect to the manually labeled ground truth in them.
-
-
-### Reference coding
-
-#### Component used
-The referenced certificate is for a component used in the product (e.g., IC used by a smartcard).
-
-#### Re-certification of
-The referenced certificate is of a previous version of the product that is now being re-certified.
-
-#### Evaluation reused
-The evaluation results of the referenced certificate were used for evaluation of the product, likely due to a shared component.
-
-#### Basis for
-The current certificate is a basis for the referenced certificate, likely during a simultaneous certification.
-
-#### Previous version
-The referenced certificate is for a previous version of the product, but re-certification is not explicitly mentioned.
-
-
-## CPE Matching
-
-The directory `./cpe_eval` contains digests of 100 randomly sampled certificates, together with predicted and ground-truth labels.
-The file `random.csv` summarizes the data above, while `manual_cpe_labels.json` is a JSON-min export from label studio instance.
-
-These files can be utilized from [cpe_eval notebook](../notebooks/cc/cpe_eval.ipynb) to see the performance of the classifier.
-
-Folder `./old_manual_cpe_labels` contains some old incomplete labeling that was obtained highly unoptimized classifier.
diff --git a/notebooks/cc/reference_annotations/hyperparameter_search.py b/notebooks/cc/reference_annotations/hyperparameter_search.py
new file mode 100644
index 00000000..a7286aa0
--- /dev/null
+++ b/notebooks/cc/reference_annotations/hyperparameter_search.py
@@ -0,0 +1,220 @@
+"""
+Simple script to perform hyperparameter search over various parameters of SentenceTransformer trained for reference
+meaning classification.
+"""
+
+from __future__ import annotations
+
+import os
+from functools import partial
+from pathlib import Path
+
+import click
+import optuna
+import pandas as pd
+import torch
+from rapidfuzz import fuzz
+from sklearn.metrics import f1_score
+
+from sec_certs.dataset import CCDataset
+from sec_certs.model.references.annotator_trainer import ReferenceAnnotatorTrainer
+from sec_certs.model.references.segment_extractor import ReferenceSegmentExtractor
+from sec_certs.utils.helpers import compute_heuristics_version
+from sec_certs.utils.nlp import prec_recall_metric
+
+
+def replace_all(text: str, to_replce: set[str]) -> str:
+    for i in to_replce:
+        text = text.replace(i, "")
+    return text
+
+
+def load_dataset(dataset_path: Path, annotations_dir: Path) -> tuple[CCDataset, pd.DataFrame]:
+    """
+    Load dataset from dataset_path and annotations_dir
+    :param dataset_path: path to dataset
+    :param annotations_dir: path to annotations
+    :return: pd.DataFrame
+    """
+    train_annotations = pd.read_csv(annotations_dir / "train.csv")
+    valid_annotations = pd.read_csv(annotations_dir / "valid.csv")
+    all_annotations = pd.concat([train_annotations, valid_annotations])
+    all_annotations = all_annotations[all_annotations.label != "None"].assign(label=lambda df: df.label.str.upper())
+
+    dset = CCDataset.from_json(dataset_path)
+    all_certs = {x.dgst: x for x in dset.certs.values()}
+    dset.certs = {x.dgst: x for x in dset.certs.values() if x.dgst in all_annotations.dgst.unique()}
+
+    cert_id_to_name_mapping = {x.heuristics.cert_id: x.name for x in all_certs.values()}
+    all_annotations["referenced_cert_name"] = all_annotations["referenced_cert_id"].map(cert_id_to_name_mapping)
+    all_annotations["cert_name"] = all_annotations["dgst"].map(lambda x: dset[x].name)
+    all_annotations["cert_versions"] = all_annotations["cert_name"].map(compute_heuristics_version)
+    all_annotations = all_annotations.loc[all_annotations["referenced_cert_name"].notnull()].copy()
+    all_annotations["referenced_cert_versions"] = all_annotations["referenced_cert_name"].map(
+        compute_heuristics_version
+    )
+    all_annotations["cert_name_stripped_version"] = all_annotations.apply(
+        lambda x: replace_all(x["cert_name"], x["cert_versions"]), axis=1
+    )
+    all_annotations["referenced_cert_name_stripped_version"] = all_annotations.apply(
+        lambda x: replace_all(x["referenced_cert_name"], x["referenced_cert_versions"]), axis=1
+    )
+    all_annotations["name_similarity"] = all_annotations.apply(
+        lambda x: fuzz.token_set_ratio(x["cert_name"], x["referenced_cert_name"]), axis=1
+    )
+    all_annotations["name_similarity_stripped_version"] = all_annotations.apply(
+        lambda x: fuzz.token_set_ratio(x["cert_name_stripped_version"], x["referenced_cert_name_stripped_version"]),
+        axis=1,
+    )
+    all_annotations["name_len_diff"] = all_annotations.apply(
+        lambda x: abs(len(x["cert_name_stripped_version"]) - len(x["referenced_cert_name_stripped_version"])), axis=1
+    )
+
+    return dset, all_annotations
+
+
+def preprocess_data(dset: CCDataset, df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Preprocess data
+    :param df: pd.DataFrame
+    :return: pd.DataFrame
+    """
+
+    def process_segment(segment: str, referenced_cert_id: str) -> str:
+        segment = segment.replace(referenced_cert_id, "the referenced product")
+        return segment
+
+    new_df = ReferenceSegmentExtractor()(dset.certs.values())
+    new_df = new_df.loc[new_df.label.notnull()].copy()
+    new_df = new_df.merge(
+        df.loc[
+            :,
+            [
+                "dgst",
+                "referenced_cert_id",
+                "name_similarity_stripped_version",
+                "name_len_diff",
+                "cert_name",
+                "referenced_cert_name",
+            ],
+        ],
+        on=["dgst", "referenced_cert_id"],
+    )
+
+    new_df.segments = new_df.apply(
+        lambda row: [process_segment(x, row.referenced_cert_id) for x in row.segments], axis=1
+    )
+
+    return new_df
+
+
+def define_trainer(trial: optuna.trial.Trial, df: pd.DataFrame) -> ReferenceAnnotatorTrainer:
+    use_analytical_rule_name_similarity = trial.suggest_categorical(
+        "use_analytical_rule_name_similarity", [True, False]
+    )
+    n_iterations = trial.suggest_int("n_iterations", 1, 50)
+    n_epochs = trial.suggest_int("n_epochs", 1, 5)
+    batch_size = trial.suggest_int("batch_size", 8, 32)
+    segmenter_metric = trial.suggest_categorical("segmenter_metric", ["accuracy", "f1"])
+    ensemble_soft_voting_power = trial.suggest_int("ensemble_soft_voting_power", 1, 5)
+
+    return ReferenceAnnotatorTrainer.from_df(
+        df,
+        prec_recall_metric,
+        mode="training",
+        use_analytical_rule_name_similarity=use_analytical_rule_name_similarity,
+        n_iterations=n_iterations,
+        n_epochs=n_epochs,
+        batch_size=batch_size,
+        segmenter_metric=segmenter_metric,
+        ensemble_soft_voting_power=ensemble_soft_voting_power,
+    )
+
+
+def objective(trial: optuna.trial.Trial, df: pd.DataFrame):
+    trainer = define_trainer(trial, df)
+    trainer.train()
+
+    annotator = trainer.clf
+    df_predicted = annotator.predict_df(df)
+
+    return f1_score(
+        df_predicted.loc[df_predicted.split == "valid", ["y_pred"]],
+        df_predicted.loc[df_predicted.split == "valid", ["label"]],
+        zero_division="warn",
+        average="weighted",
+    )
+
+
+@click.command()
+@click.option("-n", "--trials", "trials", type=int, required=True, help="Number of optimization trials to run.")
+@click.option(
+    "-d",
+    "--dataset",
+    "dataset_path",
+    type=click.Path(exists=True, dir_okay=False, file_okay=True, readable=True),
+    required=True,
+    help="Path to CCDataset json.",
+)
+@click.option(
+    "-a",
+    "--annotations",
+    "annotations_dir",
+    type=click.Path(exists=True, dir_okay=True, file_okay=False, readable=True),
+    required=True,
+    help="Path to annotations directory.",
+)
+@click.option(
+    "-o",
+    "--output",
+    "output_dir",
+    type=click.Path(exists=True, dir_okay=True, file_okay=False, readable=True),
+    required=True,
+    help="Path to output directory.",
+)
+@click.option("-t", "--timeout", "timeout", type=int, default=24, help="Timeout in hours", show_default=True)
+def main(trials: int, dataset_path: Path, annotations_dir: Path, output_dir: Path, timeout: int):
+    if not torch.cuda.is_available():
+        print("GPU is not available, exiting. Did you set `CUDA_VISIBLE_DEVICES` environment variable properly?")
+        return -1
+
+    if os.environ.get("TOKENIZERS_PARALLELISM", True) != "FALSE":
+        print(
+            "Tokenizers parallelism not disabled for spacy, exiting. Did you set `TOKENIZERS_PARALLELISM` environment variable to `FALSE`?"
+        )
+        return -1
+
+    # os.environ["CUDA_VISIBLE_DEVICES"] = "MIG-56c53afb-6f08-5e5b-83fa-32fc6f09eeb0"
+    # os.environ["TOKENIZERS_PARALLELISM"] = "FALSE"
+
+    dataset_path = Path(dataset_path)
+    annotations_dir = Path(annotations_dir)
+    output_dir = Path(output_dir)
+
+    print("Loading dataset...")
+    cc_dset, df = load_dataset(dataset_path, annotations_dir)
+
+    print("Preprocessing data...")
+    df_processed = preprocess_data(cc_dset, df)
+    partial_objective = partial(objective, df=df_processed)
+
+    print("Starting hyperparameter search...")
+    study = optuna.create_study(direction="maximize")
+    study.optimize(partial_objective, n_trials=trials, timeout=60 * 60 * timeout)
+
+    study.trials_dataframe().to_csv(output_dir / "hyperparameter_search.csv")
+
+    ax = optuna.visualization.matplotlib.plot_optimization_history(study)
+    ax.figure.savefig(output_dir / "optimization_history.pdf", bbox_inches="tight")
+
+    ax = optuna.visualization.matplotlib.plot_param_importances(study)
+    ax.figure.savefig(output_dir / "param_importances.pdf", bbox_inches="tight")
+
+    ax = optuna.visualization.matplotlib.plot_timeline(study)
+    ax.figure.savefig(output_dir / "timeline.pdf", bbox_inches="tight")
+
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb b/notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb
new file mode 100644
index 00000000..2f8f0b78
--- /dev/null
+++ b/notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb
@@ -0,0 +1,105 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Results on 5 classes:\n",
+      "\t- Cohen's Kappa: 0.7101271765978729\n",
+      "\t- Percentage agreement: 0.8225\n",
+      "Results on simplified 2 classes:\n",
+      "\t- Cohen's Kappa: 0.8424203759140207\n",
+      "\t- Percentage agreement: 0.9437869822485208\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "import pandas as pd\n",
+    "from sklearn.metrics import cohen_kappa_score\n",
+    "\n",
+    "label_mapping = {\n",
+    "    \"COMPONENT_USED\": \"COMPONENT_USED\",\n",
+    "    \"RE-EVALUATION\": \"PREVIOUS_VERSION\",\n",
+    "    \"EVALUATION_REUSED\": \"COMPONENT_USED\",\n",
+    "    \"PREVIOUS_VERSION\": \"PREVIOUS_VERSION\",\n",
+    "    \"COMPONENT_SHARED\": \"COMPONENT_USED\",\n",
+    "}\n",
+    "\n",
+    "\n",
+    "def load_all_dataframes(base_folder: Path) -> pd.DataFrame:\n",
+    "    splits = [\"train\", \"valid\", \"test\"]\n",
+    "\n",
+    "    df_train, df_valid, df_test = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()\n",
+    "    for split in splits:\n",
+    "        df = pd.read_csv(base_folder / f\"{split}.csv\")\n",
+    "        if split == \"train\":\n",
+    "            df_train = df\n",
+    "        elif split == \"valid\":\n",
+    "            df_valid = df\n",
+    "        else:\n",
+    "            df_test = df\n",
+    "\n",
+    "    df_to_return = pd.concat([df_train, df_valid, df_test])\n",
+    "    return df_to_return.assign(label=lambda df_: df_.label.fillna(\"unknown\")).assign(\n",
+    "        label=lambda df_: df_.label.str.upper(),\n",
+    "        simplified_label=lambda df_: df_.label.map(label_mapping),\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "REPO_ROOT = Path()\n",
+    "\n",
+    "\n",
+    "adam_df = load_all_dataframes(REPO_ROOT / \"src/sec_certs/data/reference_annotations/adam\")\n",
+    "jano_df = load_all_dataframes(REPO_ROOT / \"src/sec_certs/data/reference_annotations/jano\")\n",
+    "agreement_series = adam_df.label == jano_df.label\n",
+    "\n",
+    "print(\"Results on 5 classes:\")\n",
+    "print(f\"\\t- Cohen's Kappa: {cohen_kappa_score(adam_df.label, jano_df.label)}\")\n",
+    "print(f\"\\t- Percentage agreement: {agreement_series.loc[agreement_series == True].count() / agreement_series.count()}\")\n",
+    "\n",
+    "indices_to_drop = set(adam_df.loc[adam_df.simplified_label.isnull()].index.tolist()) | set(\n",
+    "    jano_df.loc[jano_df.simplified_label.isnull()].index.tolist()\n",
+    ")\n",
+    "adam_df_simplified = adam_df.drop(indices_to_drop)\n",
+    "jano_df_simplified = jano_df.drop(indices_to_drop)\n",
+    "agreement_series = adam_df_simplified.simplified_label == jano_df_simplified.simplified_label\n",
+    "\n",
+    "\n",
+    "print(\"Results on simplified 2 classes:\")\n",
+    "print(\n",
+    "    f\"\\t- Cohen's Kappa: {cohen_kappa_score(adam_df_simplified.simplified_label, jano_df_simplified.simplified_label)}\"\n",
+    ")\n",
+    "print(f\"\\t- Percentage agreement: {agreement_series.loc[agreement_series == True].count() / agreement_series.count()}\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/cc/reference_annotations/prediction.ipynb b/notebooks/cc/reference_annotations/prediction.ipynb
new file mode 100644
index 00000000..4eec5da8
--- /dev/null
+++ b/notebooks/cc/reference_annotations/prediction.ipynb
@@ -0,0 +1,264 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import logging\n",
+    "import sys\n",
+    "from ast import literal_eval\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import pandas as pd\n",
+    "import torch\n",
+    "from catboost import Pool\n",
+    "\n",
+    "from sec_certs.dataset import CCDataset\n",
+    "from sec_certs.model.references_nlp.evaluation import (\n",
+    "    evaluate_model,\n",
+    ")\n",
+    "from sec_certs.model.references_nlp.feature_extraction import (\n",
+    "    build_embeddings,\n",
+    "    dataframe_to_training_arrays,\n",
+    "    extract_geometrical_features,\n",
+    "    extract_language_features,\n",
+    "    extract_prediction_features,\n",
+    "    extract_segments,\n",
+    "    perform_dimensionality_reduction,\n",
+    ")\n",
+    "from sec_certs.model.references_nlp.training import train_model\n",
+    "\n",
+    "REPO_ROOT = Path().resolve()\n",
+    "DATASET_PATH = REPO_ROOT / \"dataset/cc_november_23/dataset.json\"\n",
+    "TENSORBOARD_DATA_DIR = REPO_ROOT / \"dataset/tensorboard_visualisation/\"\n",
+    "TRAINED_MODEL_PATH = REPO_ROOT / \"dataset/reference_prediction/final_model\"\n",
+    "\n",
+    "print(f\"GPU available: {torch.cuda.is_available()}\")\n",
+    "\n",
+    "logger = logging.getLogger(__name__)\n",
+    "logging.getLogger(\"setfit\").setLevel(logging.CRITICAL)\n",
+    "logging.getLogger(\"sentence_transformers\").setLevel(logging.CRITICAL)\n",
+    "file_handler = logging.StreamHandler(sys.stderr)\n",
+    "file_handler.setFormatter(logging.Formatter(\"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"))\n",
+    "logging.basicConfig(level=logging.INFO, handlers=[file_handler])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mode = \"production\"\n",
+    "cc_dset = CCDataset.from_json(DATASET_PATH)\n",
+    "\n",
+    "# df = extract_segments(cc_dset, mode=mode)\n",
+    "# df.to_csv(REPO_ROOT / \"dataset/reference_prediction/dataset.csv\", index=False)\n",
+    "\n",
+    "df = (\n",
+    "    pd.read_csv(REPO_ROOT / \"dataset/reference_prediction/dataset.csv\")\n",
+    "    .assign(\n",
+    "        segments=lambda df_: df_.segments.apply(literal_eval),\n",
+    "        actual_reference_keywords=lambda df_: df_.actual_reference_keywords.apply(literal_eval),\n",
+    "    )\n",
+    "    .loc[lambda df_: (df_.label != \"IRRELEVANT\")]\n",
+    ")\n",
+    "\n",
+    "label_mapping = {\n",
+    "    \"COMPONENT_USED\": \"COMPONENT_USED\",\n",
+    "    \"RE-EVALUATION\": \"PREVIOUS_VERSION\",\n",
+    "    \"EVALUATION_REUSED\": \"COMPONENT_USED\",\n",
+    "    \"PREVIOUS_VERSION\": \"PREVIOUS_VERSION\",\n",
+    "    \"COMPONENT_SHARED\": \"COMPONENT_USED\",\n",
+    "}\n",
+    "df.label = df.label.map(label_mapping)\n",
+    "\n",
+    "df, annotator = build_embeddings(\n",
+    "    df,\n",
+    "    mode=mode,\n",
+    "    method=\"transformer\",\n",
+    "    model_path=\"/var/tmp/xjanovsk/certs/sec-certs/dataset/reference_prediction/final_model\",\n",
+    ")\n",
+    "df = perform_dimensionality_reduction(\n",
+    "    df,\n",
+    "    mode,\n",
+    ")\n",
+    "df = extract_language_features(df, cc_dset)\n",
+    "df = extract_prediction_features(df, annotator._model)\n",
+    "df = extract_geometrical_features(df)\n",
+    "\n",
+    "# Obtained from running the feature selection algorithm below\n",
+    "features_to_use = [\n",
+    "    \"pca_mean_x\",\n",
+    "    \"pca_mean_y\",\n",
+    "    \"pca_var_y\",\n",
+    "    \"pca_cov_xy\",\n",
+    "    \"pca_median_x\",\n",
+    "    \"pca_median_y\",\n",
+    "    \"pca_std_distance_to_centroid\",\n",
+    "    \"pca_point_density\",\n",
+    "    \"umap_mean_x\",\n",
+    "    \"umap_mean_y\",\n",
+    "    \"umap_skew_y\",\n",
+    "    \"umap_cov_xy\",\n",
+    "    \"umap_median_x\",\n",
+    "    \"umap_median_y\",\n",
+    "    \"umap_max_distance_to_centroid\",\n",
+    "    \"umap_aspect_ratio\",\n",
+    "    \"lang_partial_ratio\",\n",
+    "    \"lang_token_sort_ratio\",\n",
+    "    \"lang_n_segments\",\n",
+    "    \"lang_matches_recertification\",\n",
+    "    \"lang_n_intersection_versions\",\n",
+    "    \"lang_common_words\",\n",
+    "    \"lang_bigram_overlap\",\n",
+    "    \"lang_common_suffix_len\",\n",
+    "    \"lang_character_trigram_overlap\",\n",
+    "    \"lang_len_difference\",\n",
+    "    \"pred_0\",\n",
+    "    \"pred_2\",\n",
+    "    \"pred_3\",\n",
+    "    \"pred_4\",\n",
+    "]\n",
+    "df_ = df[features_to_use + [\"label\", \"split\"]]\n",
+    "# df_ = df.copy()\n",
+    "\n",
+    "x_train, y_train, x_valid, y_valid, features = dataframe_to_training_arrays(\n",
+    "    df_, mode=mode, use_pca=True, use_umap=True, use_pred=True, use_lang=True\n",
+    ")\n",
+    "\n",
+    "clf = train_model(\n",
+    "    mode,\n",
+    "    x_train,\n",
+    "    y_train,\n",
+    "    x_valid,\n",
+    "    y_valid,\n",
+    "    train_baseline=False,\n",
+    ")\n",
+    "evaluate_model(\n",
+    "    clf,\n",
+    "    x_valid,\n",
+    "    y_valid,\n",
+    "    features,\n",
+    "    output_path=None,\n",
+    ")\n",
+    "\n",
+    "# Classify the whole dataset and serialize the result\n",
+    "x_all = df[features_to_use].values\n",
+    "df[\"y_pred\"] = clf.predict(x_all)\n",
+    "df[\"reference_label\"] = df.label.fillna(df.y_pred)\n",
+    "df[[\"dgst\", \"canonical_reference_keyword\", \"reference_label\"]].to_csv(\n",
+    "    \"/var/tmp/xjanovsk/certs/sec-certs/dataset/reference_prediction/predictions.csv\"\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Run feature selection algorithm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_pool = Pool(x_train, y_train, feature_names=features)\n",
+    "valid_pool = Pool(x_valid, y_valid, feature_names=features)\n",
+    "\n",
+    "dct = clf.select_features(\n",
+    "    train_pool,\n",
+    "    eval_set=valid_pool,\n",
+    "    features_for_select=features,\n",
+    "    num_features_to_select=30,\n",
+    "    train_final_model=False,\n",
+    "    verbose=False,\n",
+    ")\n",
+    "\n",
+    "features_to_use = dct[\"selected_features_names\"]\n",
+    "df_lim_features = df[features_to_use + [\"label\", \"split\"]]\n",
+    "x_train, y_train, x_valid, y_valid, features = dataframe_to_training_arrays(\n",
+    "    df_lim_features, mode=mode, use_pca=True, use_umap=True, use_pred=True, use_lang=True\n",
+    ")\n",
+    "\n",
+    "clf = train_model(x_train, y_train, x_valid, y_valid, train_baseline=False)\n",
+    "evaluate_model(\n",
+    "    clf,\n",
+    "    x_valid,\n",
+    "    y_valid,\n",
+    "    features,\n",
+    "    output_path=Path(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/cc_ref_annotator_evaluation/embeddings\"),\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Serialize misclassified instances"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "misclassified_instances = df.loc[df.y_pred != df.label]\n",
+    "misclassified_instances = misclassified_instances[\n",
+    "    [\n",
+    "        \"dgst\",\n",
+    "        \"canonical_reference_keyword\",\n",
+    "        \"actual_reference_keywords\",\n",
+    "        \"label\",\n",
+    "        \"y_pred\",\n",
+    "        \"split\",\n",
+    "        \"segments\",\n",
+    "        \"referenced_cert_name\",\n",
+    "        \"cert_versions\",\n",
+    "        \"referenced_cert_versions\",\n",
+    "        \"lang_partial_ratio\",\n",
+    "        \"lang_token_sort_ratio\",\n",
+    "    ]\n",
+    "]\n",
+    "misclassified_instances[\"report_link\"] = misclassified_instances.dgst.map(\n",
+    "    lambda x: f\"https://seccerts.org/cc/{x}/report.pdf\"\n",
+    ")\n",
+    "misclassified_instances[\"st_link\"] = misclassified_instances.dgst.map(\n",
+    "    lambda x: f\"https://seccerts.org/cc/{x}/target.pdf\"\n",
+    ")\n",
+    "# Then replace all \\\\/ with / in the corresponding json, as the pandas to_json method escapes the slashes.\n",
+    "misclassified_instances.to_json(\n",
+    "    \"/var/tmp/xjanovsk/certs/sec-certs/dataset/misclassified_references_validation_set.json\",\n",
+    "    orient=\"records\",\n",
+    "    indent=4,\n",
+    ")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/cc/reference_annotations/train_validation_test_split.ipynb b/notebooks/cc/reference_annotations/train_validation_test_split.ipynb
index 840adb46..ff005f64 100644
--- a/notebooks/cc/reference_annotations/train_validation_test_split.ipynb
+++ b/notebooks/cc/reference_annotations/train_validation_test_split.ipynb
@@ -1,6 +1,16 @@
 {
  "cells": [
   {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Train / validation / test split\n",
+    "\n",
+    "This is a notebook that was used to split the CC dataset into train/valid/test samples for the reference annotation NLP task."
+   ]
+  },
+  {
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
diff --git a/notebooks/cc/references.ipynb b/notebooks/cc/references.ipynb
index 2fc66b89..6ce7d137 100644
--- a/notebooks/cc/references.ipynb
+++ b/notebooks/cc/references.ipynb
@@ -1,6 +1,7 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {
     "collapsed": true,
@@ -11,15 +12,12 @@
    "source": [
     "# References\n",
     "\n",
-    "This notebook contains analysis of certificate references in Common Criteria certificates.\n",
-    "\n",
-    "The notebook has two parts, an analysis part and a network visualization part.\n",
-    "But first some common initialization and data loading."
+    "This notebook contains analysis of certificate references in Common Criteria certificates."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -27,667 +25,993 @@
    },
    "outputs": [],
    "source": [
+    "import warnings\n",
+    "from pathlib import Path\n",
+    "from typing import Iterable\n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
     "import networkx as nx\n",
     "import networkx.algorithms.community as nx_comm\n",
-    "import matplotlib\n",
-    "import matplotlib.pyplot as plt\n",
-    "import matplotlib.ticker as mtick\n",
-    "from sec_certs.dataset.cc import CCDataset\n",
+    "import numpy as np\n",
     "import pandas as pd\n",
     "import seaborn as sns\n",
-    "import numpy as np\n",
-    "from pysankey import sankey\n",
+    "from notebooks.fixed_sankey_plot import sankey\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "from sec_certs.dataset.cc import CCDataset\n",
+    "\n",
+    "# Surpress user warnings\n",
+    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
     "\n",
     "%matplotlib inline\n",
     "\n",
-    "matplotlib.use(\"pgf\")\n",
-    "sns.set_theme(style='white')\n",
+    "# matplotlib.use(\"pgf\")\n",
+    "sns.set_theme(style=\"white\")\n",
     "plt.rcParams[\"axes.linewidth\"] = 0.5\n",
     "plt.rcParams[\"legend.fontsize\"] = 6.5\n",
     "plt.rcParams[\"xtick.labelsize\"] = 8\n",
     "plt.rcParams[\"ytick.labelsize\"] = 8\n",
     "plt.rcParams[\"ytick.left\"] = True\n",
-    "plt.rcParams['ytick.major.size'] = 5\n",
-    "plt.rcParams['ytick.major.width'] = 0.5\n",
-    "plt.rcParams['ytick.major.pad'] = 0\n",
+    "plt.rcParams[\"ytick.major.size\"] = 5\n",
+    "plt.rcParams[\"ytick.major.width\"] = 0.5\n",
+    "plt.rcParams[\"ytick.major.pad\"] = 0\n",
     "plt.rcParams[\"xtick.bottom\"] = True\n",
-    "plt.rcParams['xtick.major.size'] = 5\n",
-    "plt.rcParams['xtick.major.width'] = 0.5\n",
-    "plt.rcParams['xtick.major.pad'] = 0\n",
-    "plt.rcParams[\"pgf.texsystem\"] = \"pdflatex\"\n",
-    "plt.rcParams[\"font.family\"] = \"serif\"\n",
-    "plt.rcParams[\"text.usetex\"] = True\n",
-    "plt.rcParams[\"pgf.rcfonts\"] = False\n",
+    "plt.rcParams[\"xtick.major.size\"] = 5\n",
+    "plt.rcParams[\"xtick.major.width\"] = 0.5\n",
+    "plt.rcParams[\"xtick.major.pad\"] = 0\n",
+    "# plt.rcParams[\"pgf.texsystem\"] = \"pdflatex\"\n",
+    "# plt.rcParams[\"font.family\"] = \"serif\"\n",
+    "# plt.rcParams[\"text.usetex\"] = True\n",
+    "# plt.rcParams[\"pgf.rcfonts\"] = False\n",
     "plt.rcParams[\"axes.titlesize\"] = 8\n",
     "plt.rcParams[\"legend.handletextpad\"] = 0.3\n",
-    "plt.rcParams['lines.markersize'] = 4\n",
-    "plt.rcParams['savefig.pad_inches'] = 0.01\n",
+    "plt.rcParams[\"lines.markersize\"] = 4\n",
+    "plt.rcParams[\"savefig.pad_inches\"] = 0.01\n",
     "sns.set_palette(\"deep\")\n",
     "\n",
-    "#plt.style.use(\"seaborn-whitegrid\")\n",
-    "#sns.set_palette(\"deep\")\n",
-    "#sns.set_context(\"notebook\")  # Set to \"paper\" for use in paper :)\n",
+    "# plt.style.use(\"seaborn-whitegrid\")\n",
+    "# sns.set_palette(\"deep\")\n",
+    "# sns.set_context(\"notebook\")  # Set to \"paper\" for use in paper :)\n",
+    "\n",
+    "# plt.rcParams['figure.figsize'] = (10, 6)\n",
     "\n",
-    "#plt.rcParams['figure.figsize'] = (10, 6)"
+    "RESULTS_DIR = Path(\"./results/references\")\n",
+    "RESULTS_DIR.mkdir(exist_ok=True, parents=True)\n",
+    "SMARTCARD_CATEGORY = \"ICs, Smart Cards and Smart Card-Related Devices and Systems\"\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Downloading CC Dataset: 100%|███████████████████████████████████████████████████████| 147M/147M [00:18<00:00, 8.41MB/s]\n"
-     ]
-    }
-   ],
+   "cell_type": "markdown",
+   "metadata": {},
    "source": [
-    "# Initialize\n",
-    "dset = CCDataset.from_web_latest()"
+    "## Common processing functions"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = dset.to_pandas()\n",
-    "df_id_rich = df.loc[df.cert_id.notnull()].copy()"
+    "def len_if_exists(x) -> int:\n",
+    "    return len(x) if pd.notnull(x) else 0\n",
+    "\n",
+    "\n",
+    "def compute_reference_numbers(df__: pd.DataFrame) -> pd.DataFrame:\n",
+    "    \"\"\"\n",
+    "    Creates new columns with number of references for each certificate.\n",
+    "    \"\"\"\n",
+    "    return df__.copy().assign(\n",
+    "        n_refs=lambda df_: df_.refs.map(len_if_exists),\n",
+    "        n_trans_refs=lambda df_: df_.trans_refs.map(len_if_exists),\n",
+    "        n_in_refs=lambda df_: df_.in_refs.map(len_if_exists),\n",
+    "        n_in_trans_refs=lambda df_: df_.in_trans_refs.map(len_if_exists),\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def preprocess_cc_df(cc_df: pd.DataFrame) -> pd.DataFrame:\n",
+    "    \"\"\"\n",
+    "    Pre-processing run on the CC dataset for the sake of this notebook.\n",
+    "    \"\"\"\n",
+    "    return (\n",
+    "        cc_df.loc[cc_df.cert_id.notnull()]\n",
+    "        .copy()\n",
+    "        .rename(\n",
+    "            columns={\n",
+    "                \"directly_referencing\": \"refs\",\n",
+    "                \"indirectly_referencing\": \"trans_refs\",\n",
+    "                \"directly_referenced_by\": \"in_refs\",\n",
+    "                \"indirectly_referenced_by\": \"in_trans_refs\",\n",
+    "            }\n",
+    "        )\n",
+    "        .assign(\n",
+    "            longer_than_5_years=lambda df_: df_.not_valid_after - df_.not_valid_before > pd.Timedelta(days=5 * 365),\n",
+    "            not_valid_after=lambda df_: df_.not_valid_after.where(\n",
+    "                ~df_.longer_than_5_years, df_.not_valid_before + pd.Timedelta(days=5 * 365)\n",
+    "            ),\n",
+    "        )\n",
+    "        .drop_duplicates(subset=[\"cert_id\"], keep=\"first\")  # TODO: Investigate high number of duplicates and resolve\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def compute_references(cc_df: pd.DataFrame, graph: nx.DiGraph, label: str | Iterable[str]) -> pd.DataFrame:\n",
+    "    \"\"\"\n",
+    "    Limits the columns with references to a given label.\n",
+    "    \"\"\"\n",
+    "    label = label if isinstance(label, Iterable) else [label]\n",
+    "    sub_edges = [(u, v) for u, v, d in graph.edges(data=True) if d.get(\"reference_label\") in label]\n",
+    "    subgraph = graph.edge_subgraph(sub_edges)\n",
+    "\n",
+    "    return cc_df.assign(\n",
+    "        refs=lambda df_: df_.apply(\n",
+    "            lambda row: set(subgraph.successors(row.cert_id)) if row.cert_id in subgraph else np.nan,\n",
+    "            axis=1,\n",
+    "        ),\n",
+    "        trans_refs=lambda df_: df_.apply(\n",
+    "            lambda row: set(nx.descendants(subgraph, row.cert_id)) if row.cert_id in subgraph else np.nan, axis=1\n",
+    "        ),\n",
+    "        in_refs=lambda df_: df_.apply(\n",
+    "            lambda row: set(subgraph.predecessors(row.cert_id)) if row.cert_id in subgraph else np.nan,\n",
+    "            axis=1,\n",
+    "        ),\n",
+    "        in_trans_refs=lambda df_: df_.apply(\n",
+    "            lambda row: set(nx.ancestors(subgraph, row.cert_id)) if row.cert_id in subgraph else np.nan, axis=1\n",
+    "        ),\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def preprocess_refs_df(csv_path: str | Path, cc_df: pd.DataFrame) -> pd.DataFrame:\n",
+    "    return (\n",
+    "        pd.read_csv(csv_path)\n",
+    "        .pipe(lambda df_: df_.loc[df_.dgst.isin(cc_df.index)])\n",
+    "        .assign(cert_id=lambda df_: df_.dgst.map(cc_df.cert_id.to_dict()))\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "def get_reference_graph_from_refs_df(refs_df: pd.DataFrame) -> nx.DiGraph:\n",
+    "    return nx.from_pandas_edgelist(\n",
+    "        refs_df,\n",
+    "        source=\"cert_id\",\n",
+    "        target=\"reference\",\n",
+    "        create_using=nx.DiGraph,\n",
+    "        edge_attr=[\"reference_label\"],\n",
+    "    )\n"
    ]
   },
   {
    "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
+   "metadata": {},
    "source": [
-    "## Reference analysis\n"
+    "## Load data and compute reference graph"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "### Count numbers of reference-rich certificates\n",
+    "dset = CCDataset.from_json(\"/Users/adam/phd/projects/certificates/sec-certs/dataset/cc_final_run_may_23/dataset.json\")\n",
+    "cc_df = preprocess_cc_df(dset.to_pandas())\n",
+    "refs_df = preprocess_refs_df(\"/Users/adam/Downloads/predictions.csv\", cc_df)\n",
+    "unique_labels = refs_df.reference_label.unique().tolist()\n",
     "\n",
-    "- From the numbers follows that whenever a certificate is directly referencing some else, it also indirectly references some else\n",
-    "- We have more outgoing references than ingoing references, which kinda makes sense. You don't have to be aware that some other cert references you"
+    "# Load labeled reference graph as networkx directed graph\n",
+    "graph = nx.from_pandas_edgelist(\n",
+    "    refs_df,\n",
+    "    source=\"cert_id\",\n",
+    "    target=\"canonical_reference_keyword\",\n",
+    "    edge_attr=\"reference_label\",\n",
+    "    create_using=nx.DiGraph,\n",
+    ")\n",
+    "\n",
+    "cc_df = compute_reference_numbers(compute_references(cc_df, graph, unique_labels))\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    },
-    "scrolled": true
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\\newcommand{\\numCcAllDirectReferencing}{1583}\n",
-      "\\newcommand{\\numCcAllNotDirectReferencing}{3866}\n",
-      "\\newcommand{\\numCcWithIdDirectReferencing}{1583}\n",
-      "\\newcommand{\\numCcWithIdNotDirectReferencing}{3788}\n"
-     ]
-    }
-   ],
+   "cell_type": "markdown",
+   "metadata": {},
    "source": [
-    "df[\"has_outgoing_direct_references\"] = df.directly_referencing.notnull()\n",
-    "df[\"has_incoming_direct_references\"] = df.directly_referenced_by.notnull()\n",
-    "df[\"has_outgoing_indirect_references\"] = df.indirectly_referencing.notnull()\n",
-    "df[\"has_incoming_indirect_references\"] = df.indirectly_referenced_by.notnull()\n",
-    "\n",
-    "#df.loc[:, [\"directly_referenced_by\", \"indirectly_referenced_by\", \"directly_referencing\", \"indirectly_referencing\"]].notnull().describe()\n",
-    "\n",
-    "print(f\"\\\\newcommand{{\\\\numCcAllDirectReferencing}}{{{df.has_outgoing_direct_references.sum()}}}\")\n",
-    "print(f\"\\\\newcommand{{\\\\numCcAllNotDirectReferencing}}{{{len(df) - df.has_outgoing_direct_references.sum()}}}\")\n",
-    "\n",
-    "df_id_rich[\"has_outgoing_direct_references\"] = df_id_rich.directly_referencing.notnull()\n",
-    "df_id_rich[\"has_incoming_direct_references\"] = df_id_rich.directly_referenced_by.notnull()\n",
-    "df_id_rich[\"has_outgoing_indirect_references\"] = df_id_rich.indirectly_referencing.notnull()\n",
-    "df_id_rich[\"has_incoming_indirect_references\"] = df_id_rich.indirectly_referenced_by.notnull()\n",
-    "\n",
-    "print(f\"\\\\newcommand{{\\\\numCcWithIdDirectReferencing}}{{{df_id_rich.has_outgoing_direct_references.sum()}}}\")\n",
-    "print(f\"\\\\newcommand{{\\\\numCcWithIdNotDirectReferencing}}{{{len(df_id_rich) - df_id_rich.has_outgoing_direct_references.sum()}}}\")\n",
-    "\n",
-    "#df_id_rich.loc[:, [\"directly_referenced_by\", \"indirectly_referenced_by\", \"directly_referencing\", \"indirectly_referencing\"]].notnull().describe()"
+    "## Common processing functions"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\\newcommand{\\numCCActiveDirectReferencing}{540}\n",
-      "\\newcommand{\\numCCActiveDirectReferencingArchived}{179}\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "print(f\"\\\\newcommand{{\\\\numCCActiveDirectReferencing}}{{{df_id_rich.loc[df_id_rich.status == 'active'].has_outgoing_direct_references.sum()}}}\")\n",
-    "\n",
-    "archived_cert_id_list = set(df_id_rich[df_id_rich.status == \"archived\"].cert_id)\n",
-    "def contains_archived_cert_reference(referencing):\n",
-    "    if pd.isnull(referencing):\n",
-    "        return False\n",
-    "    return bool(archived_cert_id_list.intersection(referencing))\n",
-    "print(f\"\\\\newcommand{{\\\\numCCActiveDirectReferencingArchived}}{{{df_id_rich[df_id_rich.status == 'active'].directly_referencing.apply(contains_archived_cert_reference).sum()}}}\")"
+    "# Understand which columns I need and limit myself to those columns\n",
+    "# Every analytical cell should be isolated in a function that takes a single input: The dataframe of certificates to work on.\n",
+    "#     - The number of those references is computed in the function itself\n",
+    "#     - Each analytical method should have some tests at the end\n",
+    "#     - If some LaTeX output accompanies the computaiton, the function should return it as a string\n",
+    "#     - Those are stored in a dictionary that keeps expanding\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Plot direct references per category"
+    "### Count numbers of reference-rich certificates"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "figure, axes = plt.subplots(1, 2)\n",
-    "figure.set_size_inches(16, 10)\n",
-    "figure.set_tight_layout(True)\n",
+    "def compute_basic_reference_graph_stats(df__: pd.DataFrame, graph: nx.DiGraph) -> dict[str, str]:\n",
+    "    df = df__.copy().assign(has_refs=lambda df_: df_.refs.notnull()).pipe(compute_reference_numbers)\n",
     "\n",
-    "col_to_depict = [\"has_outgoing_direct_references\", \"has_incoming_direct_references\"]\n",
+    "    n_ref_smartcards = df.loc[(df.has_refs) & (df.category == SMARTCARD_CATEGORY)].shape[0]\n",
+    "    n_ref_others = df.loc[(df.has_refs) & (df.category != SMARTCARD_CATEGORY)].shape[0]\n",
     "\n",
-    "for index, col in enumerate(col_to_depict):\n",
-    "    countplot = sns.countplot(data=df, x=\"category\", hue=col, ax=axes[index])\n",
-    "    countplot.set(\n",
-    "        xlabel=\"Category\",\n",
-    "        ylabel=\"Outgoing direct references\",\n",
-    "        title=f\"Countplot of {' '.join(col.split('_'))}\",\n",
+    "    print(\n",
+    "        f\"Number of smartcard certificates that reference some other certificate: {n_ref_smartcards} ({100 * n_ref_smartcards / df.loc[df.category == SMARTCARD_CATEGORY].shape[0]:.2f}%)\"\n",
+    "    )\n",
+    "    print(\n",
+    "        f\"Number of non-smartcard certificates that reference some other certificate: {n_ref_others} ({100 * n_ref_others / df.loc[df.category != SMARTCARD_CATEGORY].shape[0]:.2f}%)\"\n",
     "    )\n",
-    "    countplot.tick_params(axis=\"x\", rotation=90)\n",
-    "    countplot.legend(title=' '.join(col.split('_')), bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0)\n"
+    "    print(\n",
+    "        f\"Total number of referencing certificates: {n_ref_smartcards + n_ref_others} ({100 * (n_ref_smartcards + n_ref_others) / df.shape[0]:.2f}%)\"\n",
+    "    )\n",
+    "\n",
+    "    df_melted = df[[\"n_refs\", \"n_trans_refs\", \"n_in_refs\", \"n_in_trans_refs\"]].melt()\n",
+    "    df_melted[\"incoming\"] = df_melted.variable.map(lambda x: bool(x.endswith(\"by\")))\n",
+    "    sns.catplot(data=df_melted, kind=\"boxen\", x=\"variable\", y=\"value\", col=\"variable\", sharex=False, sharey=False)\n",
+    "    plt.savefig(RESULTS_DIR / \"boxen_plot_references.pdf\", bbox_inches=\"tight\")\n",
+    "\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "compute_basic_reference_graph_stats(cc_df, graph)\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Evolution of certificate reach for top-10 certificates"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\\newcommand{\\numCCDirectRefsSameCategory}{2257}\n",
-      "\\newcommand{\\numCCDirectRefsOtherCategory}{205}\n",
-      "\\newcommand{\\numCCDirectRefs}{2462}\n",
-      "\\newcommand{\\numCCDirectRefsFromSmartcards}{2007}\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "cert_id_to_category_mapping = dict(zip(df.cert_id, df.category))\n",
-    "cert_id_to_category_mapping[np.NaN] = \"No references\"\n",
+    "# TODO: Check that it actually works, the data on small subset was fairly weird\n",
+    "# TODO: Work only on sub-component references?\n",
+    "def compute_certs_top_reach(df__: pd.DataFrame) -> dict:\n",
+    "    def find_reach_over_time(df_: pd.DataFrame, cert_id: str, date_range: pd.DatetimeIndex) -> pd.Series:\n",
+    "        df = df_.copy().loc[lambda df_: df_.in_trans_refs.apply(lambda x: pd.notnull(x) and cert_id in x)]\n",
+    "        dct = {\n",
+    "            date: df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)].shape[0] for date in date_range\n",
+    "        }\n",
+    "        return pd.Series(dct, name=cert_id)\n",
     "\n",
-    "exploded = df_id_rich.loc[:, [\"category\", \"directly_referencing\"]].explode(\"directly_referencing\")\n",
-    "exploded[\"ref_category\"] = exploded.directly_referencing.map(lambda x: cert_id_to_category_mapping[x] if pd.notnull(x) else np.nan)\n",
-    "exploded = exploded.loc[exploded.ref_category.notnull()]\n",
+    "    df = df__.copy()\n",
+    "    top_10_certs = df.sort_values(by=\"n_in_trans_refs\", ascending=False).head(10)\n",
+    "    print(top_10_certs[[\"cert_id\", \"n_in_trans_refs\"]])\n",
     "\n",
-    "exploded_with_refs = exploded.loc[exploded.ref_category != \"No references\"]\n",
-    "print(f\"\\\\newcommand{{\\\\numCCDirectRefsSameCategory}}{{{(exploded_with_refs.category == exploded_with_refs.ref_category).sum()}}}\")\n",
-    "print(f\"\\\\newcommand{{\\\\numCCDirectRefsOtherCategory}}{{{(exploded_with_refs.category != exploded_with_refs.ref_category).sum()}}}\")\n",
-    "print(f\"\\\\newcommand{{\\\\numCCDirectRefs}}{{{len(exploded_with_refs)}}}\")\n",
-    "print(f\"\\\\newcommand{{\\\\numCCDirectRefsFromSmartcards}}{{{(exploded_with_refs.category == 'ICs, Smart Cards and Smart Card-Related Devices and Systems').sum()}}}\")\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
+    "    data = [find_reach_over_time(df, x, date_range) for x in tqdm(top_10_certs.cert_id.tolist())]\n",
+    "    df_reach_evolution_melted = (\n",
+    "        pd.concat(data, axis=1)\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
+    "        .melt(id_vars=\"date\", var_name=\"certificate\", value_name=\"reach\")\n",
+    "    )\n",
     "\n",
-    "all_categories = set(exploded.category.unique()) | set(exploded.ref_category.unique())\n",
-    "colors = list(sns.color_palette(\"hls\", len(all_categories), as_cmap=False).as_hex())\n",
-    "color_dict = dict(zip(all_categories, colors))\n",
+    "    g = sns.lineplot(data=df_reach_evolution_melted, x=\"date\", y=\"reach\", hue=\"certificate\")\n",
+    "    g.set(title=\"Reach of top-10 certificates in time\", xlabel=\"Time\", ylabel=\"Certificate reach\")\n",
+    "    plt.savefig(RESULTS_DIR / \"lineplot_top_certificate_reach.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
     "\n",
-    "figure, axes = plt.subplots(1, 1)\n",
-    "figure.set_size_inches(24, 10)\n",
-    "figure.set_tight_layout(True)\n",
+    "    return {}\n",
     "\n",
-    "sankey(exploded.category, exploded.ref_category, colorDict=color_dict, leftLabels=list(exploded.category.unique()), rightLabels=list(exploded.ref_category.unique()), fontsize=12, ax=axes)\n",
     "\n",
-    "figure.savefig(\"category_references.pdf\", bbox_inches=\"tight\")\n",
-    "figure.savefig(\"category_references.pgf\", bbox_inches=\"tight\")\n",
-    "plt.close(figure)"
+    "compute_certs_top_reach(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Plot direct references per scheme"
+    "## Average number of references & certificate reach over time"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "figure, axes = plt.subplots(1, 2)\n",
-    "figure.set_size_inches(14, 4)\n",
-    "figure.set_tight_layout(True)\n",
+    "def compute_avg_references(df__: pd.DataFrame, variable: str, date_range: pd.DatetimeIndex) -> dict:\n",
+    "    df = df__.copy()\n",
+    "    return {\n",
+    "        date: df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)][variable].mean()\n",
+    "        for date in tqdm(date_range)\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "def compute_avg_references_over_time(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
+    "    refs_smartcards = compute_avg_references(df.loc[df.category == SMARTCARD_CATEGORY], \"n_refs\", date_range)\n",
+    "    trans_refs_smartcards = compute_avg_references(\n",
+    "        df.loc[df.category == SMARTCARD_CATEGORY], \"n_trans_refs\", date_range\n",
+    "    )\n",
+    "    refs_others = compute_avg_references(df.loc[df.category != SMARTCARD_CATEGORY], \"n_refs\", date_range)\n",
+    "    trans_refs_others = compute_avg_references(df.loc[df.category != SMARTCARD_CATEGORY], \"n_trans_refs\", date_range)\n",
     "\n",
-    "col_to_depict = [\"has_outgoing_direct_references\", \"has_incoming_direct_references\"]\n",
+    "    df_avg_num_refs_melted = (\n",
+    "        pd.concat(\n",
+    "            [\n",
+    "                pd.Series(refs_smartcards, name=\"smartcard references\"),\n",
+    "                pd.Series(refs_others, name=\"other references\"),\n",
+    "                pd.Series(trans_refs_smartcards, name=\"smartcard transitive references\"),\n",
+    "                pd.Series(trans_refs_others, name=\"other transitive references\"),\n",
+    "            ],\n",
+    "            axis=1,\n",
+    "        )\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
+    "        .melt(id_vars=[\"date\"], var_name=\"category\", value_name=\"n_references\")\n",
+    "    )\n",
+    "\n",
+    "    g = sns.lineplot(data=df_avg_num_refs_melted, x=\"date\", y=\"n_references\", hue=\"category\")\n",
+    "    g.set(title=\"Average number of references in certificates\", xlabel=\"Time\", ylabel=\"Number of references\")\n",
+    "    plt.savefig(RESULTS_DIR / \"lineplot_avg_n_references.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "def compute_avg_reach_over_time(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
+    "    reach_smartcards = compute_avg_references(df.loc[df.category == SMARTCARD_CATEGORY], \"n_in_trans_refs\", date_range)\n",
+    "    reach_others = compute_avg_references(df.loc[df.category != SMARTCARD_CATEGORY], \"n_in_trans_refs\", date_range)\n",
+    "\n",
+    "    df_avg_num_refs_melted = (\n",
+    "        pd.concat(\n",
+    "            [\n",
+    "                pd.Series(reach_smartcards, name=\"smartcard reach\"),\n",
+    "                pd.Series(reach_others, name=\"other reach\"),\n",
+    "            ],\n",
+    "            axis=1,\n",
+    "        )\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
+    "        .melt(id_vars=[\"date\"], var_name=\"category\", value_name=\"n_references\")\n",
+    "    )\n",
     "\n",
-    "for index, col in enumerate(col_to_depict):\n",
-    "    countplot = sns.countplot(data=df, x=\"scheme\", hue=col, ax=axes[index])\n",
-    "    countplot.set(\n",
-    "        xlabel=\"Category\",\n",
-    "        ylabel=\"Outgoing direct references\",\n",
-    "        title=f\"Countplot of {' '.join(col.split('_'))}\",\n",
+    "    g = sns.lineplot(data=df_avg_num_refs_melted, x=\"date\", y=\"n_references\", hue=\"category\")\n",
+    "    g.set(\n",
+    "        title=\"Average certificate reach over time\",\n",
+    "        xlabel=\"Time\",\n",
+    "        ylabel=\"Number of (transitively) referencing certificates\",\n",
     "    )\n",
-    "    countplot.tick_params(axis=\"x\", rotation=90)\n",
-    "    countplot.legend(title=' '.join(col.split('_')), bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0)"
+    "    plt.savefig(RESULTS_DIR / \"lineplot_avg_n_references.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "compute_avg_references_over_time(cc_df)\n",
+    "compute_avg_reach_over_time(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Number of certificates referencing archived certificates"
+    "## Number of active vs. number of reference-rich certificates in time"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Number of certificates that reference some archived certificate: 983\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "def references_archived_cert(references):\n",
-    "    if pd.isnull(references):\n",
-    "        return False\n",
+    "def compute_number_of_active_vs_ref_rich_certs_over_time(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
+    "\n",
+    "    dct_active_others = {}\n",
+    "    dct_reference_rich_others = {}\n",
+    "    dct_active_smartcards = {}\n",
+    "    dct_reference_rich_smartcards = {}\n",
+    "\n",
+    "    for date in tqdm(date_range):\n",
+    "        active_certs = df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)]\n",
+    "        dct_active_others[date] = active_certs.loc[active_certs.category != SMARTCARD_CATEGORY].shape[0]\n",
+    "        dct_active_smartcards[date] = active_certs.loc[active_certs.category == SMARTCARD_CATEGORY].shape[0]\n",
+    "        dct_reference_rich_others[date] = active_certs.loc[\n",
+    "            (active_certs.category != SMARTCARD_CATEGORY) & (active_certs.n_refs > 0)\n",
+    "        ].shape[0]\n",
+    "        dct_reference_rich_smartcards[date] = active_certs.loc[\n",
+    "            (active_certs.category == SMARTCARD_CATEGORY) & (active_certs.n_refs > 0)\n",
+    "        ].shape[0]\n",
     "\n",
-    "    return any([x in cert_ids] for x in references)\n",
+    "    df_active_vs_ref_rich_melted = (\n",
+    "        pd.concat(\n",
+    "            [\n",
+    "                pd.Series(dct_active_others, name=\"active other categories\"),\n",
+    "                pd.Series(dct_active_smartcards, name=\"active smartcards\"),\n",
+    "                pd.Series(dct_reference_rich_others, name=\"ref. rich other categories\"),\n",
+    "                pd.Series(dct_reference_rich_smartcards, name=\"ref. rich smartcards\"),\n",
+    "            ],\n",
+    "            axis=1,\n",
+    "        )\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
+    "        .melt(id_vars=[\"date\"], var_name=\"category\", value_name=\"number of certificates\")\n",
+    "    )\n",
+    "\n",
+    "    g = sns.lineplot(data=df_active_vs_ref_rich_melted, x=\"date\", y=\"number of certificates\", hue=\"category\")\n",
+    "    g.set(\n",
+    "        title=\"Number of active certificates vs. reference-rich certificates in time\",\n",
+    "        xlabel=\"Time\",\n",
+    "        ylabel=\"Number of certificates\",\n",
+    "    )\n",
+    "    plt.savefig(RESULTS_DIR / \"lienplot_n_active_certs_vs_n_references.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "    return {}\n",
     "\n",
-    "cert_ids = set(df.loc[((df.cert_id.notnull()) & (df.status == \"archived\")), \"cert_id\"].tolist())\n",
-    "df[\"references_archived_cert\"] = df.directly_referenced_by.map(references_archived_cert)\n",
     "\n",
-    "print(f\"Number of certificates that reference some archived certificate: {df.loc[df.references_archived_cert].shape[0]}\")\n",
+    "def compute_summary_active_vs_ref_rich_over_time(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
     "\n",
-    "col_to_depict = [\"category\", \"scheme\"]\n",
+    "    dct_active_all = {}\n",
+    "    dct_reference_rich_all = {}\n",
+    "    dct_referenced_all = {}\n",
+    "    dct_isolated_all = {}\n",
     "\n",
-    "figure, axes = plt.subplots(1, 2)\n",
-    "figure.set_size_inches(14, 8)\n",
-    "figure.set_tight_layout(True)\n",
+    "    for date in tqdm(date_range):\n",
+    "        active_certs = df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)]\n",
+    "        dct_active_all[date] = active_certs.shape[0]\n",
+    "        dct_isolated_all[date] = active_certs.loc[(active_certs.n_refs == 0) & (active_certs.n_in_refs == 0)].shape[0]\n",
+    "        dct_reference_rich_all[date] = active_certs.loc[active_certs.n_refs > 0].shape[0]\n",
+    "        dct_referenced_all[date] = active_certs.loc[active_certs.n_in_refs > 0].shape[0]\n",
     "\n",
-    "for index, col in enumerate(col_to_depict):\n",
-    "    countplot = sns.countplot(data=df, x=col, hue=\"references_archived_cert\", ax=axes[index])\n",
-    "    countplot.set(\n",
-    "        xlabel=col,\n",
-    "        ylabel=\"Outgoing direct references\",\n",
-    "        title=\"Countplot of certificates that reference some archived certificate\",\n",
+    "    df_summary_references = (\n",
+    "        pd.concat(\n",
+    "            [\n",
+    "                pd.Series(dct_active_all, name=\"active certificates\"),\n",
+    "                pd.Series(dct_reference_rich_all, name=\"ref. rich certificates\"),\n",
+    "                pd.Series(dct_referenced_all, name=\"referenced certificates\"),\n",
+    "                pd.Series(dct_isolated_all, name=\"isolated certificates\"),\n",
+    "            ],\n",
+    "            axis=1,\n",
+    "        )\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
     "    )\n",
-    "    countplot.tick_params(axis=\"x\", rotation=90)\n",
-    "    countplot.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0)"
+    "\n",
+    "    df_summary_references_melted = df_summary_references.melt(\n",
+    "        id_vars=[\"date\"], var_name=\"category\", value_name=\"number of certificates\"\n",
+    "    )\n",
+    "    g = sns.lineplot(\n",
+    "        data=df_summary_references_melted, x=\"date\", y=\"number of certificates\", hue=\"category\", errorbar=None\n",
+    "    )\n",
+    "    g.set(\n",
+    "        title=\"Number of active certificates vs. reference-rich vs. referenced certificates in time\",\n",
+    "        xlabel=\"Time\",\n",
+    "        ylabel=\"Number of certificates\",\n",
+    "    )\n",
+    "    plt.savefig(RESULTS_DIR / \"lineplot_references_summary.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "\n",
+    "    df_ratios = df_summary_references.copy()\n",
+    "    df_ratios[\"ref. rich certificates\"] = df_ratios[\"ref. rich certificates\"] / df_ratios[\"active certificates\"]\n",
+    "    df_ratios[\"referenced certificates\"] = df_ratios[\"referenced certificates\"] / df_ratios[\"active certificates\"]\n",
+    "    df_ratios[\"isolated certificates\"] = df_ratios[\"isolated certificates\"] / df_ratios[\"active certificates\"]\n",
+    "    df_ratios = df_ratios.drop(columns=[\"active certificates\"])\n",
+    "    df_ratios_melted = df_ratios.melt(id_vars=[\"date\"], var_name=\"category\", value_name=\"ratio of certificates\")\n",
+    "\n",
+    "    g = sns.lineplot(data=df_ratios_melted, x=\"date\", y=\"ratio of certificates\", hue=\"category\", errorbar=None)\n",
+    "    g.set(\n",
+    "        title=\"ratio of reference-rich vs. referenced vs. isolated certificates in time\",\n",
+    "        xlabel=\"Time\",\n",
+    "        ylabel=\"Number of certificates\",\n",
+    "    )\n",
+    "    plt.savefig(RESULTS_DIR / \"lineplot_reference_ratio.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "compute_number_of_active_vs_ref_rich_certs_over_time(cc_df)\n",
+    "compute_summary_active_vs_ref_rich_over_time(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Count scheme references"
+    "## Number of active certificates that reference some archived certificate in time"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "cert_id_to_scheme_mapping = dict(zip(df.cert_id, df.scheme))\n",
+    "def compute_certs_referencing_archived_ones(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
     "\n",
-    "df_ref_rich = df_id_rich.loc[df.directly_referencing.notnull()]\n",
-    "exploded = df_ref_rich.loc[:, [\"scheme\", \"directly_referencing\"]].explode(\"directly_referencing\")\n",
+    "    dct_direct_others = {}\n",
+    "    dct_direct_smartcards = {}\n",
+    "    dct_transitive_others = {}\n",
+    "    dct_transitive_smartcards = {}\n",
     "\n",
-    "exploded[\"ref_scheme\"] = exploded.directly_referencing.map(cert_id_to_scheme_mapping)\n",
-    "exploded = exploded.loc[exploded.ref_scheme.notnull()]\n",
+    "    for date in tqdm(date_range):\n",
+    "        active_certs = df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)].copy()\n",
+    "        active_certs_cert_ids = set(active_certs[\"cert_id\"].tolist())\n",
+    "        active_certs[\"no_intersection\"] = active_certs.refs.map(\n",
+    "            lambda x: False if pd.isnull(x) else not x.intersection(active_certs_cert_ids)\n",
+    "        )\n",
+    "        active_certs[\"no_transitive_intersection\"] = active_certs.trans_refs.map(\n",
+    "            lambda x: False if pd.isnull(x) else not x.intersection(active_certs_cert_ids)\n",
+    "        )\n",
+    "        dct_direct_others[date] = active_certs.loc[\n",
+    "            (active_certs.no_intersection) & (active_certs.category != SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
+    "        dct_transitive_others[date] = active_certs.loc[\n",
+    "            (active_certs.no_transitive_intersection) & (active_certs.category != SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
+    "        dct_direct_smartcards[date] = active_certs.loc[\n",
+    "            (active_certs.no_intersection) & (active_certs.category == SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
+    "        dct_transitive_smartcards[date] = active_certs.loc[\n",
+    "            (active_certs.no_transitive_intersection) & (active_certs.category == SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
     "\n",
-    "all_schemes = set(exploded.scheme.unique()) | set(exploded.ref_scheme.unique())\n",
-    "colors = list(sns.color_palette(\"hls\", len(all_schemes), as_cmap=False).as_hex())\n",
-    "color_dict = dict(zip(all_schemes, colors))\n",
+    "    df_refs_to_archived_melted = (\n",
+    "        pd.concat(\n",
+    "            [\n",
+    "                pd.Series(dct_direct_others, name=\"direct reference others\"),\n",
+    "                pd.Series(dct_transitive_others, name=\"transitive reference others\"),\n",
+    "                pd.Series(dct_direct_smartcards, name=\"direct reference smartcards\"),\n",
+    "                pd.Series(dct_transitive_smartcards, name=\"transitive reference smartcards\"),\n",
+    "            ],\n",
+    "            axis=1,\n",
+    "        )\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
+    "        .melt(id_vars=[\"date\"], var_name=\"reference type\", value_name=\"number of certificates\")\n",
+    "    )\n",
+    "\n",
+    "    g = sns.lineplot(data=df_refs_to_archived_melted, x=\"date\", y=\"number of certificates\", hue=\"reference type\")\n",
+    "    g.set(\n",
+    "        title=\"Number of active certificates that reference some archived certificate\",\n",
+    "        xlabel=\"Time\",\n",
+    "        ylabel=\"Number of certificates\",\n",
+    "    )\n",
+    "    plt.savefig(RESULTS_DIR / \"lienplot_active_certs_referencing_archived.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
     "\n",
-    "figure, axes = plt.subplots(1, 1)\n",
-    "figure.set_size_inches(4, 4)\n",
-    "figure.set_tight_layout(True)\n",
+    "    return {}\n",
     "\n",
-    "sankey(exploded.scheme, exploded.ref_scheme, colorDict=color_dict, leftLabels=list(exploded.scheme.unique()), rightLabels=list(exploded.ref_scheme.unique()), fontsize=7, ax=axes)\n",
     "\n",
-    "figure.savefig(\"scheme_references.pdf\", bbox_inches=\"tight\")\n",
-    "figure.savefig(\"scheme_references.pgf\", bbox_inches=\"tight\")\n",
-    "plt.close(figure)"
+    "compute_certs_referencing_archived_ones(cc_df)\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Certificates referencing vulnerable certificates in time"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\\newcommand{\\numCCUSReferencing}{4}\n",
-      "\\newcommand{\\numCCUS}{1027}\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "print(f\"\\\\newcommand{{\\\\numCCUSReferencing}}{{{len(df_id_rich.loc[(df_id_rich.scheme == 'US') & (df_id_rich.directly_referencing.notnull())])}}}\")\n",
-    "print(f\"\\\\newcommand{{\\\\numCCUS}}{{{len(df_id_rich.loc[(df_id_rich.scheme == 'US')])}}}\")"
+    "def compute_certs_referencing_vulnerable_over_time(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
+    "    vulnerable_cert_ids = set(df.loc[df.related_cves.notnull()].cert_id.tolist())\n",
+    "    dct_direct_others = {}\n",
+    "    dct_transitive_others = {}\n",
+    "    dct_direct_smartcards = {}\n",
+    "    dct_transitive_smartcards = {}\n",
+    "\n",
+    "    for date in tqdm(date_range):\n",
+    "        active_certs = df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)].copy()\n",
+    "        active_certs[\"directly_references_vulnerable_cert\"] = active_certs.refs.map(\n",
+    "            lambda x: False if pd.isnull(x) else bool(x.intersection(vulnerable_cert_ids))\n",
+    "        )\n",
+    "        active_certs[\"transitively_references_vulnerable_cert\"] = active_certs.trans_refs.map(\n",
+    "            lambda x: False if pd.isnull(x) else bool(x.intersection(vulnerable_cert_ids))\n",
+    "        )\n",
+    "        dct_direct_others[date] = active_certs.loc[\n",
+    "            (active_certs.directly_references_vulnerable_cert) & (active_certs.category != SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
+    "        dct_transitive_others[date] = active_certs.loc[\n",
+    "            (active_certs.transitively_references_vulnerable_cert) & (active_certs.category != SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
+    "        dct_direct_smartcards[date] = active_certs.loc[\n",
+    "            (active_certs.directly_references_vulnerable_cert) & (active_certs.category == SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
+    "        dct_transitive_smartcards[date] = active_certs.loc[\n",
+    "            (active_certs.transitively_references_vulnerable_cert) & (active_certs.category == SMARTCARD_CATEGORY)\n",
+    "        ].shape[0]\n",
+    "\n",
+    "    df_references_vuln_melted = (\n",
+    "        pd.concat(\n",
+    "            [\n",
+    "                pd.Series(dct_direct_others, name=\"direct references others\"),\n",
+    "                pd.Series(dct_transitive_others, name=\"transitive references others\"),\n",
+    "                pd.Series(dct_direct_smartcards, name=\"direct references smartcards\"),\n",
+    "                pd.Series(dct_transitive_smartcards, name=\"transitive references smartcards\"),\n",
+    "            ],\n",
+    "            axis=1,\n",
+    "        )\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
+    "        .melt(id_vars=[\"date\"], var_name=\"reference type\", value_name=\"number of certificates\")\n",
+    "    )\n",
+    "\n",
+    "    g = sns.lineplot(data=df_references_vuln_melted, x=\"date\", y=\"number of certificates\", hue=\"reference type\")\n",
+    "    g.set(\n",
+    "        title=\"Number of active certificates that reference some vulnerable certificate\",\n",
+    "        xlabel=\"Time\",\n",
+    "        ylabel=\"Number of certificates\",\n",
+    "    )\n",
+    "    plt.savefig(RESULTS_DIR / \"lienplot_active_certs_referencing_vulnerable.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "compute_certs_referencing_vulnerable_over_time(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Temporal evolution of references\n",
-    "\n",
-    "Shows plot with relative number of certificates for a given year that reference some other certificate"
+    "### Plot direct references per category (count plot)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "df_temporal = df.loc[df.year_from < 2022].groupby([\"year_from\"])[\"directly_referencing\"].count().reset_index().set_index(\"year_from\")\n",
-    "n_issued_certs = df.groupby(\"year_from\").name.count().reset_index().rename(columns={\"name\": \"n_certs\"}).set_index(\"year_from\")\n",
-    "df_temporal.directly_referencing = 100 * df_temporal.directly_referencing / n_issued_certs.n_certs\n",
+    "def plot_direct_refs_per_category(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy().assign(\n",
+    "        has_outgoing_direct_references=lambda df_: df_.n_refs > 0,\n",
+    "        has_incoming_direct_references=lambda df_: df_.n_in_refs > 0,\n",
+    "    )\n",
+    "    figure, axes = plt.subplots(1, 2)\n",
+    "    figure.set_size_inches(16, 10)\n",
+    "    figure.set_tight_layout(True)\n",
+    "\n",
+    "    col_to_depict = [\"has_outgoing_direct_references\", \"has_incoming_direct_references\"]\n",
     "\n",
-    "line = sns.lineplot(data=df_temporal, x=\"year_from\", y=\"directly_referencing\")\n",
-    "line.yaxis.set_major_formatter(mtick.PercentFormatter())"
+    "    for index, col in enumerate(col_to_depict):\n",
+    "        countplot = sns.countplot(data=df, x=\"category\", hue=col, ax=axes[index])\n",
+    "        countplot.set(\n",
+    "            xlabel=\"Category\",\n",
+    "            ylabel=\"Outgoing direct references\",\n",
+    "            title=f\"Countplot of {' '.join(col.split('_'))}\",\n",
+    "        )\n",
+    "        countplot.tick_params(axis=\"x\", rotation=90)\n",
+    "        countplot.legend(title=\" \".join(col.split(\"_\")), bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0)\n",
+    "\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "plot_direct_refs_per_category(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Cross references"
+    "### Plot direct references per category (Sankey diagram)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "# Plotting w.r.t. scheme and category (both are interesting)"
+    "def plot_sankey_refs_categories(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy()\n",
+    "\n",
+    "    cert_id_to_category_mapping = dict(zip(df.cert_id, df.category))\n",
+    "    cert_id_to_category_mapping[np.NaN] = \"No references\"\n",
+    "\n",
+    "    exploded = df.loc[:, [\"category\", \"refs\"]].explode(\"refs\")\n",
+    "    exploded[\"ref_category\"] = exploded.refs.map(lambda x: cert_id_to_category_mapping[x] if pd.notnull(x) else np.nan)\n",
+    "    exploded = exploded.loc[exploded.ref_category.notnull()]\n",
+    "\n",
+    "    exploded_with_refs = exploded.loc[exploded.ref_category != \"No references\"]\n",
+    "\n",
+    "    all_categories = set(exploded.category.unique()) | set(exploded.ref_category.unique())\n",
+    "    colors = list(sns.color_palette(\"hls\", len(all_categories), as_cmap=False).as_hex())\n",
+    "    color_dict = dict(zip(all_categories, colors))\n",
+    "\n",
+    "    figure, axes = plt.subplots(1, 1)\n",
+    "    figure.set_size_inches(24, 10)\n",
+    "    figure.set_tight_layout(True)\n",
+    "\n",
+    "    sankey(\n",
+    "        exploded.category,\n",
+    "        exploded.ref_category,\n",
+    "        colorDict=color_dict,\n",
+    "        leftLabels=list(exploded.category.unique()),\n",
+    "        rightLabels=list(exploded.ref_category.unique()),\n",
+    "        fontsize=12,\n",
+    "        ax=axes,\n",
+    "    )\n",
+    "\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "plot_sankey_refs_categories(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Reference network visualization"
+    "### Plot direct references per scheme (count plot)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Certificates in dataset: 5449\n",
-      "Certificates with extracted IDs: 5292\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "certs_with_ids = {cert.heuristics.cert_id: cert for cert in dset if cert.heuristics.cert_id}\n",
+    "def plot_refs_per_scheme(df__: pd.DataFrame) -> dict:\n",
+    "    df = df__.copy().assign(\n",
+    "        has_outgoing_direct_references=lambda df_: df_.n_refs > 0,\n",
+    "        has_incoming_direct_references=lambda df_: df_.n_in_refs > 0,\n",
+    "    )\n",
+    "    figure, axes = plt.subplots(1, 2)\n",
+    "    figure.set_size_inches(14, 4)\n",
+    "    figure.set_tight_layout(True)\n",
+    "\n",
+    "    col_to_depict = [\"has_outgoing_direct_references\", \"has_incoming_direct_references\"]\n",
+    "\n",
+    "    for index, col in enumerate(col_to_depict):\n",
+    "        countplot = sns.countplot(data=df, x=\"scheme\", hue=col, ax=axes[index])\n",
+    "        countplot.set(\n",
+    "            xlabel=\"Category\",\n",
+    "            ylabel=\"Outgoing direct references\",\n",
+    "            title=f\"Countplot of {' '.join(col.split('_'))}\",\n",
+    "        )\n",
+    "        countplot.tick_params(axis=\"x\", rotation=90)\n",
+    "        countplot.legend(title=\" \".join(col.split(\"_\")), bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0)\n",
     "\n",
-    "print(f\"Certificates in dataset: {len(dset)}\")\n",
-    "print(f\"Certificates with extracted IDs: {len(certs_with_ids)}\")"
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "plot_refs_per_scheme(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
+   "metadata": {},
    "source": [
-    "### Certificate report references"
+    "### Number of certificates referencing archived certificates (count plot)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
     }
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "References in certificate reports: 2439\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "refs_cr = nx.DiGraph()\n",
-    "for cert_id, cert in certs_with_ids.items():\n",
-    "    refs_cr.add_node(cert_id, cert=cert)\n",
-    "for cert_id, cert in certs_with_ids.items():\n",
-    "    if cr_refs := cert.heuristics.report_references.directly_referencing:\n",
-    "        for ref_id in cr_refs:\n",
-    "            if ref_id in certs_with_ids:\n",
-    "                refs_cr.add_edge(cert_id, ref_id, type=(\"cr\",))\n",
-    "print(f\"References in certificate reports: {len(refs_cr.edges)}\")"
+    "def countplot_certs_referencing_archived(df__: pd.DataFrame) -> dict:\n",
+    "    def references_archived_cert(references):\n",
+    "        if pd.isnull(references):\n",
+    "            return False\n",
+    "\n",
+    "        return any([x in cert_ids] for x in references)\n",
+    "\n",
+    "    df = df__.copy()\n",
+    "\n",
+    "    cert_ids = set(df.loc[((df.cert_id.notnull()) & (df.status == \"archived\")), \"cert_id\"].tolist())\n",
+    "    df[\"references_archived_cert\"] = df.in_refs.map(references_archived_cert)\n",
+    "\n",
+    "    # TODO: We should limit on the number of certificates that referenced an archived certificate at some point where they were active as well.\n",
+    "    print(\n",
+    "        f\"Number of certificates that reference some archived certificate: {df.loc[df.references_archived_cert].shape[0]}\"\n",
+    "    )\n",
+    "\n",
+    "    col_to_depict = [\"category\", \"scheme\"]\n",
+    "\n",
+    "    figure, axes = plt.subplots(1, 2)\n",
+    "    figure.set_size_inches(14, 8)\n",
+    "    figure.set_tight_layout(True)\n",
+    "\n",
+    "    for index, col in enumerate(col_to_depict):\n",
+    "        countplot = sns.countplot(data=df, x=col, hue=\"references_archived_cert\", ax=axes[index])\n",
+    "        countplot.set(\n",
+    "            xlabel=col,\n",
+    "            ylabel=\"Outgoing direct references\",\n",
+    "            title=\"Countplot of certificates that reference some archived certificate\",\n",
+    "        )\n",
+    "        countplot.tick_params(axis=\"x\", rotation=90)\n",
+    "        countplot.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.0)\n",
+    "\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "countplot_certs_referencing_archived(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
+   "metadata": {},
    "source": [
-    "### Security target references"
+    "### Count scheme references (Sankey diagram)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "References in security targets: 1007\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "refs_st = nx.DiGraph()\n",
-    "for cert_id, cert in certs_with_ids.items():\n",
-    "    refs_st.add_node(cert_id, cert=cert)\n",
-    "for cert_id, cert in certs_with_ids.items():\n",
-    "    if st_refs := cert.heuristics.st_references.directly_referencing:\n",
-    "        for ref_id in st_refs:\n",
-    "            if ref_id in certs_with_ids:\n",
-    "                refs_st.add_edge(cert_id, ref_id, type=(\"st\",))\n",
-    "print(f\"References in security targets: {len(refs_st.edges)}\")"
+    "def plot_sankey_refs_schemes(df__: pd.DataFrame) -> dict:\n",
+    "    cert_id_to_scheme_mapping = dict(zip(df__.cert_id, df__.scheme))\n",
+    "    exploded = (\n",
+    "        df__.copy()\n",
+    "        .loc[lambda df_: df_.refs.notnull(), [\"scheme\", \"refs\"]]\n",
+    "        .explode(\"refs\")\n",
+    "        .assign(ref_scheme=lambda df_: df_.refs.map(cert_id_to_scheme_mapping))\n",
+    "        .loc[lambda df_: df_.ref_scheme.notnull()]\n",
+    "    )\n",
+    "\n",
+    "    all_schemes = set(exploded.scheme.unique()) | set(exploded.ref_scheme.unique())\n",
+    "    colors = list(sns.color_palette(\"hls\", len(all_schemes), as_cmap=False).as_hex())\n",
+    "    color_dict = dict(zip(all_schemes, colors))\n",
+    "\n",
+    "    figure, axes = plt.subplots(1, 1)\n",
+    "    figure.set_size_inches(4, 4)\n",
+    "    figure.set_tight_layout(True)\n",
+    "\n",
+    "    sankey(\n",
+    "        exploded.scheme,\n",
+    "        exploded.ref_scheme,\n",
+    "        colorDict=color_dict,\n",
+    "        leftLabels=list(exploded.scheme.unique()),\n",
+    "        rightLabels=list(exploded.ref_scheme.unique()),\n",
+    "        fontsize=7,\n",
+    "        ax=axes,\n",
+    "    )\n",
+    "\n",
+    "    figure.savefig(str(RESULTS_DIR / \"scheme_references.pdf\"), bbox_inches=\"tight\")\n",
+    "    figure.savefig(str(RESULTS_DIR / \"scheme_references.pgf\"), bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "\n",
+    "    return {}\n",
+    "\n",
+    "\n",
+    "plot_sankey_refs_schemes(cc_df)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Combined references"
+    "## Reference network visualization"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Combined references (not double counted): 2704\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "refs = nx.DiGraph()\n",
-    "for cert_id, cert in certs_with_ids.items():\n",
-    "    refs.add_node(cert_id, cert=cert)\n",
-    "\n",
-    "for cert_id, cert in certs_with_ids.items():\n",
-    "    cr_refs = cert.heuristics.report_references.directly_referencing\n",
-    "    st_refs = cert.heuristics.st_references.directly_referencing\n",
-    "    cr_refs = set(cr_refs) if cr_refs is not None else set()\n",
-    "    st_refs = set(st_refs) if st_refs is not None else set()\n",
-    "    both = cr_refs.union(st_refs)\n",
-    "    for ref in both:\n",
-    "        if ref not in certs_with_ids:\n",
-    "            continue\n",
-    "        if ref in cr_refs and ref not in st_refs:\n",
-    "            refs.add_edge(cert_id, ref, type=(\"cr\", ))\n",
-    "        elif ref in st_refs and ref not in cr_refs:\n",
-    "            refs.add_edge(cert_id, ref, type=(\"st\", ))\n",
-    "        else:\n",
-    "            refs.add_edge(cert_id, ref, type=(\"cr\", \"st\"))\n",
-    "print(f\"Combined references (not double counted): {len(refs.edges)}\")"
+    "# Print:\n",
+    "# - How many references in reports\n",
+    "# - How many references in targets\n",
+    "# - How many references in total\n",
+    "# -\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Certificate overview\n",
-    "Enter the certificate you are interested in below and see its reference graph component."
+    "### Combined references"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 18,
+   "attachments": {},
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "cert_id = \"ANSSI-CC-2019/02\""
+    "### Certificate overview\n",
+    "Enter the certificate you are interested in below and see its reference graph component."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "metadata": {
     "scrolled": false
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Certificate with id ANSSI-CC-2019/02:\n",
-      " - is in a component with 21 certificates and 22 references.\n",
-      " - references ['ANSSI-CC-2019/01', 'ANSSI-CC-2018/13']\n",
-      " - is referenced by ['ANSSI-CC-2019/60']\n",
-      " - its page is at https://seccerts.org/cc/102bbdfcd696d7a9/\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "cert = certs_with_ids.get(cert_id)\n",
-    "if cert is None:\n",
-    "    print(f\"Certificate with id {cert_id} is not present in the dataset.\")\n",
+    "cert_id = \"ANSSI-CC-2014/07\"\n",
     "\n",
-    "for component in nx.weakly_connected_components(refs):\n",
+    "for component in nx.weakly_connected_components(graph):\n",
     "    if cert_id in component:\n",
     "        break\n",
+    "else:\n",
+    "    raise ValueError(f\"Certificate with id {cert_id} not found in graph.\")\n",
     "\n",
-    "view = nx.subgraph_view(refs, lambda node: node in component)\n",
+    "view = nx.subgraph_view(graph, lambda node: node in component)\n",
     "print(f\"Certificate with id {cert_id}:\")\n",
     "print(f\" - is in a component with {len(view.nodes)} certificates and {len(view.edges)} references.\")\n",
     "print(f\" - references {list(view[cert_id].keys())}\")\n",
     "print(f\" - is referenced by {list(view.predecessors(cert_id))}\")\n",
-    "print(f\" - its page is at https://seccerts.org/cc/{cert.dgst}/\")"
+    "for cert in dset:\n",
+    "    if cert.heuristics.cert_id == cert_id:\n",
+    "        break\n",
+    "else:\n",
+    "    raise ValueError(f\"Certificate with id {cert_id} not found in dataset.\")\n",
+    "print(f\" - its page is at https://seccerts.org/cc/{cert.dgst}/\")\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "nx.draw(view, pos=nx.planar_layout(view), with_labels=True)"
+    "nx.draw(view, pos=nx.planar_layout(view), with_labels=True)\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {
     "pycharm": {
@@ -696,212 +1020,96 @@
    },
    "source": [
     "## Some graph metrics\n",
-    "From <https://dataground.io/2021/09/29/simple-graph-metrics-networkx-for-beginners/> and\n",
-    "<https://theslaps.medium.com/centrality-metrics-via-networkx-python-e13e60ba2740>.\n",
-    "Also good <https://www.geeksforgeeks.org/network-centrality-measures-in-a-graph-using-networkx-python/>"
+    "See:\n",
+    "- <https://dataground.io/2021/09/29/simple-graph-metrics-networkx-for-beginners/>\n",
+    "- <https://theslaps.medium.com/centrality-metrics-via-networkx-python-e13e60ba2740>\n",
+    "- <https://www.geeksforgeeks.org/network-centrality-measures-in-a-graph-using-networkx-python/>\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Density = 9.657152514295372e-05\n",
-      "Transitivity = 0.28752821670428896\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(f\"Density = {nx.density(refs)}\")\n",
-    "print(f\"Transitivity = {nx.transitivity(refs)}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Degree centrality <Popularity> (top 20):\n",
-      "\tBSI-DSZ-CC-0555-2009 = 0.008505008505008505\n",
-      "\tBSI-DSZ-CC-0410-2007 = 0.008505008505008505\n",
-      "\tANSSI-CC-2010/02 = 0.008127008127008126\n",
-      "\tBSI-DSZ-CC-0829-2012 = 0.006048006048006048\n",
-      "\tBSI-DSZ-CC-0837-V2-2014 = 0.005859005859005859\n",
-      "\tBSI-DSZ-CC-0813-2012 = 0.0056700056700056695\n",
-      "\tBSI-DSZ-CC-1059-2018 = 0.005103005103005103\n",
-      "\tBSI-DSZ-CC-0891-V2-2016 = 0.004536004536004536\n",
-      "\tBSI-DSZ-CC-1040-2019 = 0.004347004347004347\n",
-      "\tBSI-DSZ-CC-0782-V2-2015 = 0.004347004347004347\n",
-      "\tBSI-DSZ-CC-0645-2010 = 0.004158004158004158\n",
-      "\tBSI-DSZ-CC-0266-2005 = 0.00378000378000378\n",
-      "\tBSI-DSZ-CC-1110-V2-2019 = 0.003591003591003591\n",
-      "\tBSI-DSZ-CC-0322-2005 = 0.003591003591003591\n",
-      "\tBSI-DSZ-CC-1107-V3-2022 = 0.0034020034020034017\n",
-      "\tBSI-DSZ-CC-0973-V2-2016 = 0.0034020034020034017\n",
-      "\tBSI-DSZ-CC-0978-2016 = 0.0034020034020034017\n",
-      "\tANSSI-CC-2015/36 = 0.0034020034020034017\n",
-      "\tBSI-DSZ-CC-0782-2012 = 0.0034020034020034017\n",
-      "\tANSSI-CC-2011/07 = 0.0034020034020034017\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
+    "print(f\"Density = {nx.density(graph)}\")\n",
+    "print(f\"Transitivity = {nx.transitivity(graph)}\")\n",
+    "\n",
     "print(\"Degree centrality <Popularity> (top 20):\")\n",
-    "degree_centrality_vals = [(node, val) for node, val in nx.degree_centrality(refs).items()]\n",
+    "degree_centrality_vals = list(nx.degree_centrality(graph).items())\n",
     "degree_centrality_vals.sort(key=lambda pair: pair[1], reverse=True)\n",
     "for pair in degree_centrality_vals[:20]:\n",
-    "    print(f\"\\t{pair[0]} = {pair[1]}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Eigenvector centrality <Influence> (top 20):\n",
-      "\tBSI-DSZ-CC-0404-2007 = 0.34997916589268285\n",
-      "\tBSI-DSZ-CC-0410-2007 = 0.3281366677692413\n",
-      "\tBSI-DSZ-CC-0858-2013 = 0.30933798637111454\n",
-      "\tBSI-DSZ-CC-0555-2009 = 0.3093378374783434\n",
-      "\tBSI-DSZ-CC-0674-2011 = 0.2900032282073668\n",
-      "\tBSI-DSZ-CC-0709-2010 = 0.2706698440410895\n",
-      "\tBSI-DSZ-CC-0750-V2-2014 = 0.23200295186797337\n",
-      "\tBSI-DSZ-CC-0633-2010 = 0.23200271283890217\n",
-      "\tBSI-DSZ-CC-0710-2010 = 0.23200252391425\n",
-      "\tNSCIB-CC-13-37760-CR2 = 0.17400201952226893\n",
-      "\tBSI-DSZ-CC-0675-2011 = 0.17400191027397377\n",
-      "\tBSI-DSZ-CC-0730-2011 = 0.17400189296987031\n",
-      "\tBSI-DSZ-CC-0913-2014 = 0.1305014197293018\n",
-      "\tBSI-DSZ-CC-0911-2014 = 0.1305014197293018\n",
-      "\tBSI-DSZ-CC-0912-2014 = 0.1305014197293018\n",
-      "\tBSI-DSZ-CC-0798-2012 = 0.1305014197293018\n",
-      "\tBSI-DSZ-CC-0797-2012 = 0.1305014197293018\n",
-      "\tBSI-DSZ-CC-0799-2012 = 0.1305014197293018\n",
-      "\tBSI-DSZ-CC-0914-2014 = 0.1305014197293018\n",
-      "\tBSI-DSZ-CC-0804-2012 = 0.1305014197293018\n"
-     ]
-    }
-   ],
-   "source": [
+    "    print(f\"\\t{pair[0]} = {pair[1]}\")\n",
+    "\n",
     "print(\"Eigenvector centrality <Influence> (top 20):\")\n",
-    "eigenvector_centrality_vals = [(node, val) for node, val in nx.eigenvector_centrality(refs).items()]\n",
+    "eigenvector_centrality_vals = list(nx.eigenvector_centrality(graph).items())\n",
     "eigenvector_centrality_vals.sort(key=lambda pair: pair[1], reverse=True)\n",
     "for pair in eigenvector_centrality_vals[:20]:\n",
-    "    print(f\"\\t{pair[0]} = {pair[1]}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Closeness centrality <Centralness> (top 20):\n"
-     ]
-    }
-   ],
-   "source": [
+    "    print(f\"\\t{pair[0]} = {pair[1]}\")\n",
+    "\n",
     "print(\"Closeness centrality <Centralness> (top 20):\")\n",
-    "closeness_centrality_vals = [(node, val) for node, val in nx.closeness_centrality(refs).items()]\n",
+    "closeness_centrality_vals = list(nx.closeness_centrality(graph).items())\n",
     "closeness_centrality_vals.sort(key=lambda pair: pair[1], reverse=True)\n",
     "for pair in closeness_centrality_vals[:20]:\n",
-    "    print(f\"\\t{pair[0]} = {pair[1]}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [],
-   "source": [
+    "    print(f\"\\t{pair[0]} = {pair[1]}\")\n",
+    "\n",
     "print(\"Betweenness centrality <Bridge> (top 20):\")\n",
-    "betweenness_centrality_vals = [(node, val) for node, val in nx.betweenness_centrality(refs).items()]\n",
+    "betweenness_centrality_vals = list(nx.betweenness_centrality(graph).items())\n",
     "betweenness_centrality_vals.sort(key=lambda pair: pair[1], reverse=True)\n",
     "for pair in betweenness_centrality_vals[:20]:\n",
-    "    print(f\"\\t{pair[0]} = {pair[1]}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "component_lengths = list(filter(lambda comp_len: comp_len > 1, map(len, nx.weakly_connected_components(refs))))\n",
+    "    print(f\"\\t{pair[0]} = {pair[1]}\")\n",
+    "\n",
+    "component_lengths = list(filter(lambda comp_len: comp_len > 1, map(len, nx.weakly_connected_components(graph))))\n",
     "component_lengths.sort(reverse=True)\n",
-    "print(component_lengths)"
+    "# print(component_lengths)\n",
+    "print(f\"Number of weakly connected subgraphs: {len(component_lengths)}\")\n",
+    "print(f\"Size of the largest weakly connected subgraphs: {component_lengths[:10]}\")\n",
+    "\n",
+    "big_boy = graph.subgraph(max(nx.weakly_connected_components(graph), key=len))\n",
+    "communities = list(nx_comm.greedy_modularity_communities(big_boy))\n",
+    "print(len(communities))\n",
+    "\n",
+    "for com in communities:\n",
+    "    for i in sorted(com):\n",
+    "        print(f\"\\t{i}\")\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [],
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
    "source": [
-    "big_boy = refs.subgraph(max(nx.weakly_connected_components(refs), key=len))\n",
-    "communities = list(nx_comm.greedy_modularity_communities(big_boy))\n",
-    "print(len(communities))"
+    "## LaTeX commands"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    },
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "for com in communities:\n",
-    "    for i in sorted(com):\n",
-    "        print(f\"\\t{i}\")"
+    "# TODO: These are old commands that belonged to first paper. Replace them with code that can produce the similar numbers.\n",
+    "# print(f\"\\\\newcommand{{\\\\numCcAllDirectReferencing}}{{{df.has_outgoing_direct_references.sum()}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCcAllNotDirectReferencing}}{{{len(df) - df.has_outgoing_direct_references.sum()}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCcWithIdDirectReferencing}}{{{df_id_rich.has_outgoing_direct_references.sum()}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCcWithIdNotDirectReferencing}}{{{len(df_id_rich) - df_id_rich.has_outgoing_direct_references.sum()}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCCActiveDirectReferencing}}{{{df_id_rich.loc[df_id_rich.status == 'active'].has_outgoing_direct_references.sum()}}}\")\n",
+    "\n",
+    "# print(\"\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCCDirectRefsSameCategory}}{{{(exploded_with_refs.category == exploded_with_refs.ref_category).sum()}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCCDirectRefsOtherCategory}}{{{(exploded_with_refs.category != exploded_with_refs.ref_category).sum()}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCCDirectRefs}}{{{len(exploded_with_refs)}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCCDirectRefsFromSmartcards}}{{{(exploded_with_refs.category == 'ICs, Smart Cards and Smart Card-Related Devices and Systems').sum()}}}\")\n",
+    "\n",
+    "# print(\"\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCCUSReferencing}}{{{len(df_id_rich.loc[(df_id_rich.scheme == 'US') & (df_id_rich.directly_referencing.notnull())])}}}\")\n",
+    "# print(f\"\\\\newcommand{{\\\\numCCUS}}{{{len(df_id_rich.loc[(df_id_rich.scheme == 'US')])}}}\")\n"
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.8.13 ('venv': venv)",
    "language": "python",
    "name": "python3"
   },
@@ -915,7 +1123,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {
diff --git a/notebooks/fixed_sankey_plot.py b/notebooks/fixed_sankey_plot.py
new file mode 100644
index 00000000..b8d062f9
--- /dev/null
+++ b/notebooks/fixed_sankey_plot.py
@@ -0,0 +1,400 @@
+# type: ignore
+# ruff: noqa: UP007
+"""
+This is a fork of https://github.com/anazalea/pySankey/blob/master/pysankey/sankey.py.
+We've had some problems with the plot, mostly related to resizing (likely, I don't remember now).
+This code should fix the problems and should be used to produce figures in the relevant sec-certs papers.
+"""
+
+import logging
+import warnings
+from collections import defaultdict
+from typing import Any, Optional, Union
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+from numpy import float64, ndarray
+from pandas.core.frame import DataFrame
+from pandas.core.series import Series
+
+
+class PySankeyException(Exception):
+    """Generic PySankey Exception."""
+
+
+class NullsInFrame(PySankeyException):
+    pass
+
+
+class LabelMismatch(PySankeyException):
+    pass
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def check_data_matches_labels(labels: Union[list[str], set[str]], data: Series, side: str) -> None:
+    """Check whether data matches labels.
+    Raise a LabelMismatch Exception if not."""
+    if len(labels) > 0:
+        if isinstance(data, list):
+            data = set(data)
+        if isinstance(data, pd.Series):
+            data = set(data.unique().tolist())
+        if isinstance(labels, list):
+            labels = set(labels)
+        if labels != data:
+            msg = "\n"
+            if len(labels) <= 20:
+                msg = "Labels: " + ",".join(labels) + "\n"
+            if len(data) < 20:
+                msg += "Data: " + ",".join(data)
+            raise LabelMismatch(f"{side} labels and data do not match.{msg}")
+
+
+def sankey(
+    left: Union[list, ndarray, Series],
+    right: Union[ndarray, Series],
+    leftWeight: Optional[ndarray] = None,
+    rightWeight: Optional[ndarray] = None,
+    colorDict: Optional[dict[str, str]] = None,
+    leftLabels: Optional[list[str]] = None,
+    rightLabels: Optional[list[str]] = None,
+    aspect: int = 4,
+    rightColor: bool = False,
+    fontsize: int = 14,
+    figureName: Optional[str] = None,
+    closePlot: bool = False,
+    figSize: Optional[tuple[int, int]] = None,
+    ax: Optional[Any] = None,
+) -> Any:
+    """
+    Make Sankey Diagram showing flow from left-->right
+    Inputs:
+        left = NumPy array of object labels on the left of the diagram
+        right = NumPy array of corresponding labels on the right of the diagram
+            len(right) == len(left)
+        leftWeight = NumPy array of weights for each strip starting from the
+            left of the diagram, if not specified 1 is assigned
+        rightWeight = NumPy array of weights for each strip starting from the
+            right of the diagram, if not specified the corresponding leftWeight
+            is assigned
+        colorDict = Dictionary of colors to use for each label
+            {'label':'color'}
+        leftLabels = order of the left labels in the diagram
+        rightLabels = order of the right labels in the diagram
+        aspect = vertical extent of the diagram in units of horizontal extent
+        rightColor = If true, each strip in the diagram will be be colored
+                    according to its left label
+        figSize = tuple setting the width and height of the sankey diagram.
+            Defaults to current figure size
+        ax = optional, matplotlib axes to plot on, otherwise uses current axes.
+    Output:
+        ax : matplotlib Axes
+    """
+    ax, leftLabels, leftWeight, rightLabels, rightWeight = init_values(
+        ax,
+        closePlot,
+        figSize,
+        figureName,
+        left,
+        leftLabels,
+        leftWeight,
+        rightLabels,
+        rightWeight,
+    )
+    plt.rc("text", usetex=False)
+    plt.rc("font", family="serif")
+    data_frame = _create_dataframe(left, leftWeight, right, rightWeight)
+    # Identify all labels that appear 'left' or 'right'
+    all_labels = pd.Series(np.r_[data_frame.left.unique(), data_frame.right.unique()]).unique()
+    LOGGER.debug("Labels to handle : %s", all_labels)
+    leftLabels, rightLabels = identify_labels(data_frame, leftLabels, rightLabels)
+    colorDict = create_colors(all_labels, colorDict)  # type: ignore
+    ns_l, ns_r = determine_widths(data_frame, leftLabels, rightLabels)
+    # Determine positions of left label patches and total widths
+    leftWidths, topEdge = _get_positions_and_total_widths(data_frame, leftLabels, "left")
+    # Determine positions of right label patches and total widths
+    rightWidths, topEdge = _get_positions_and_total_widths(data_frame, rightLabels, "right")
+    # Total vertical extent of diagram
+    xMax = topEdge / aspect
+    draw_vertical_bars(
+        ax,
+        colorDict,  # type: ignore
+        fontsize,
+        leftLabels,
+        leftWidths,
+        rightLabels,
+        rightWidths,
+        xMax,  # type: ignore
+    )
+    plot_strips(
+        ax,
+        colorDict,  # type: ignore
+        data_frame,
+        leftLabels,
+        leftWidths,
+        ns_l,
+        ns_r,
+        rightColor,
+        rightLabels,
+        rightWidths,
+        xMax,
+    )
+    if figSize is not None:
+        plt.gcf().set_size_inches(figSize)
+    save_image(figureName)
+    if closePlot:
+        plt.close()
+    return ax
+
+
+def save_image(figureName: Optional[str]) -> None:
+    if figureName is not None:
+        file_name = f"{figureName}.png"
+        plt.savefig(file_name, bbox_inches="tight", dpi=150)
+        LOGGER.info("Sankey diagram generated in '%s'", file_name)
+
+
+def identify_labels(dataFrame: DataFrame, leftLabels: list[str], rightLabels: list[str]) -> tuple[ndarray, ndarray]:
+    # Identify left labels
+    if len(leftLabels) == 0:
+        leftLabels = pd.Series(dataFrame.left.unique()).unique()
+    else:
+        check_data_matches_labels(leftLabels, dataFrame["left"], "left")
+    # Identify right labels
+    if len(rightLabels) == 0:
+        rightLabels = pd.Series(dataFrame.right.unique()).unique()
+    else:
+        check_data_matches_labels(rightLabels, dataFrame["right"], "right")
+    return leftLabels, rightLabels
+
+
+def init_values(
+    ax: Optional[Any],
+    closePlot: bool,
+    figSize: Optional[tuple[int, int]],
+    figureName: Optional[str],
+    left: Union[list, ndarray, Series],
+    leftLabels: Optional[list[str]],
+    leftWeight: Optional[ndarray],
+    rightLabels: Optional[list[str]],
+    rightWeight: Optional[ndarray],
+) -> tuple[Any, list[str], ndarray, list[str], ndarray]:
+    deprecation_warnings(closePlot, figSize, figureName)
+    if ax is None:
+        ax = plt.gca()
+    if leftWeight is None:
+        leftWeight = []
+    if rightWeight is None:
+        rightWeight = []
+    if leftLabels is None:
+        leftLabels = []
+    if rightLabels is None:
+        rightLabels = []
+    # Check weights
+    if len(leftWeight) == 0:
+        leftWeight = np.ones(len(left))
+    if len(rightWeight) == 0:
+        rightWeight = leftWeight
+    return ax, leftLabels, leftWeight, rightLabels, rightWeight
+
+
+def deprecation_warnings(closePlot: bool, figSize: Optional[tuple[int, int]], figureName: Optional[str]) -> None:
+    warn = []
+    if figureName is not None:
+        msg = "use of figureName in sankey() is deprecated"
+        warnings.warn(msg, DeprecationWarning)
+        warn.append(msg[7:-14])
+    if closePlot is not False:
+        msg = "use of closePlot in sankey() is deprecated"
+        warnings.warn(msg, DeprecationWarning)
+        warn.append(msg[7:-14])
+    if figSize is not None:
+        msg = "use of figSize in sankey() is deprecated"
+        warnings.warn(msg, DeprecationWarning)
+        warn.append(msg[7:-14])
+    if warn:
+        LOGGER.warning(
+            " The following arguments are deprecated and should be removed: %s",
+            ", ".join(warn),
+        )
+
+
+def determine_widths(dataFrame: DataFrame, leftLabels: ndarray, rightLabels: ndarray) -> tuple[dict, dict]:
+    # Determine widths of individual strips
+    ns_l: dict = defaultdict()
+    ns_r: dict = defaultdict()
+    for leftLabel in leftLabels:
+        left_dict = {}
+        right_dict = {}
+        for rightLabel in rightLabels:
+            left_dict[rightLabel] = dataFrame[
+                (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel)
+            ].leftWeight.sum()
+            right_dict[rightLabel] = dataFrame[
+                (dataFrame.left == leftLabel) & (dataFrame.right == rightLabel)
+            ].rightWeight.sum()
+        ns_l[leftLabel] = left_dict
+        ns_r[leftLabel] = right_dict
+    return ns_l, ns_r
+
+
+def draw_vertical_bars(
+    ax: Any,
+    colorDict: Union[dict[str, tuple[float, float, float]], dict[str, str]],
+    fontsize: int,
+    leftLabels: ndarray,
+    leftWidths: dict,
+    rightLabels: ndarray,
+    rightWidths: dict,
+    xMax: float64,
+) -> None:
+    # Draw vertical bars on left and right of each  label's section & print label
+    for leftLabel in leftLabels:
+        ax.fill_between(
+            [-0.02 * xMax, 0],
+            2 * [leftWidths[leftLabel]["bottom"]],
+            2 * [leftWidths[leftLabel]["bottom"] + leftWidths[leftLabel]["left"]],
+            color=colorDict[leftLabel],
+            alpha=0.99,
+        )
+        ax.text(
+            -0.05 * xMax,
+            leftWidths[leftLabel]["bottom"] + 0.5 * leftWidths[leftLabel]["left"],
+            leftLabel,
+            {"ha": "right", "va": "center"},
+            fontsize=fontsize,
+        )
+    for rightLabel in rightLabels:
+        ax.fill_between(
+            [xMax, 1.02 * xMax],
+            2 * [rightWidths[rightLabel]["bottom"]],
+            2 * [rightWidths[rightLabel]["bottom"] + rightWidths[rightLabel]["right"]],
+            color=colorDict[rightLabel],
+            alpha=0.99,
+        )
+        ax.text(
+            1.05 * xMax,
+            rightWidths[rightLabel]["bottom"] + 0.5 * rightWidths[rightLabel]["right"],
+            rightLabel,
+            {"ha": "left", "va": "center"},
+            fontsize=fontsize,
+        )
+
+
+def create_colors(
+    allLabels: ndarray, colorDict: Optional[dict[str, str]]
+) -> Union[dict[str, tuple[float, float, float]], dict[str, str]]:
+    # If no colorDict given, make one
+    if colorDict is None:
+        colorDict = {}
+        palette = "hls"
+        colorPalette = sns.color_palette(palette, len(allLabels))
+        for i, label in enumerate(allLabels):
+            colorDict[label] = colorPalette[i]
+    else:
+        missing = [label for label in allLabels if label not in colorDict]
+        if missing:
+            raise ValueError(
+                "The colorDict parameter is missing values for the following labels : " + ", ".join(missing)
+            )
+    LOGGER.debug("The colordict value are : %s", colorDict)
+    return colorDict
+
+
+def _create_dataframe(
+    left: Union[list, ndarray, Series],
+    leftWeight: Union[ndarray, Series],
+    right: Union[ndarray, Series],
+    rightWeight: Union[ndarray, Series],
+) -> DataFrame:
+    # Create Dataframe
+    if isinstance(left, pd.Series):
+        left = left.reset_index(drop=True)
+    if isinstance(right, pd.Series):
+        right = right.reset_index(drop=True)
+    if isinstance(leftWeight, pd.Series):
+        leftWeight = leftWeight.reset_index(drop=True)
+    if isinstance(rightWeight, pd.Series):
+        rightWeight = rightWeight.reset_index(drop=True)
+    data_frame = pd.DataFrame(
+        {
+            "left": left,
+            "right": right,
+            "leftWeight": leftWeight,
+            "rightWeight": rightWeight,
+        },
+        index=range(len(left)),
+    )
+    if len(data_frame[(data_frame.left.isnull()) | (data_frame.right.isnull())]):
+        raise NullsInFrame("Sankey graph does not support null values.")
+    return data_frame
+
+
+def plot_strips(
+    ax: Any,
+    colorDict: Union[dict[str, tuple[float, float, float]], dict[str, str]],
+    dataFrame: DataFrame,
+    leftLabels: ndarray,
+    leftWidths: dict,
+    ns_l: dict,
+    ns_r: dict,
+    rightColor: bool,
+    rightLabels: ndarray,
+    rightWidths: dict,
+    xMax: float64,
+) -> None:
+    # Plot strips
+    for leftLabel in leftLabels:
+        for rightLabel in rightLabels:
+            label_color = leftLabel
+            if rightColor:
+                label_color = rightLabel
+            if len(dataFrame[(dataFrame.left == leftLabel) & (dataFrame.right == rightLabel)]) > 0:
+                # Create array of y values for each strip, half at left value,
+                # half at right, convolve
+                ys_d = np.array(50 * [leftWidths[leftLabel]["bottom"]] + 50 * [rightWidths[rightLabel]["bottom"]])
+                ys_d = np.convolve(ys_d, 0.05 * np.ones(20), mode="valid")
+                ys_d = np.convolve(ys_d, 0.05 * np.ones(20), mode="valid")
+                ys_u = np.array(
+                    50 * [leftWidths[leftLabel]["bottom"] + ns_l[leftLabel][rightLabel]]
+                    + 50 * [rightWidths[rightLabel]["bottom"] + ns_r[leftLabel][rightLabel]]
+                )
+                ys_u = np.convolve(ys_u, 0.05 * np.ones(20), mode="valid")
+                ys_u = np.convolve(ys_u, 0.05 * np.ones(20), mode="valid")
+
+                # Update bottom edges at each label so next strip starts at the
+                # right place
+                leftWidths[leftLabel]["bottom"] += ns_l[leftLabel][rightLabel]
+                rightWidths[rightLabel]["bottom"] += ns_r[leftLabel][rightLabel]
+                ax.fill_between(
+                    np.linspace(0, xMax, len(ys_d)),
+                    ys_d,
+                    ys_u,
+                    alpha=0.65,
+                    color=colorDict[label_color],
+                )
+    ax.axis("off")
+
+
+def _get_positions_and_total_widths(df: DataFrame, labels: ndarray, side: str) -> tuple[dict, float64]:
+    """Determine positions of label patches and total widths"""
+    widths: dict = defaultdict()
+    for i, label in enumerate(labels):
+        label_widths = {}
+        label_widths[side] = df[df[side] == label][side + "Weight"].sum()
+        if i == 0:
+            label_widths["bottom"] = 0
+            label_widths["top"] = label_widths[side]
+        else:
+            bottom_width = widths[labels[i - 1]]["top"]
+            weighted_sum = 0.05 * df[side + "Weight"].sum()
+            label_widths["bottom"] = bottom_width + weighted_sum
+            label_widths["top"] = label_widths["bottom"] + label_widths[side]
+            topEdge = label_widths["top"]
+        widths[label] = label_widths
+        LOGGER.debug("%s position of '%s' : %s", side, label, label_widths)
+    return widths, topEdge
diff --git a/pyproject.toml b/pyproject.toml
index 6b8ebab7..ad63b424 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,7 +28,6 @@
   dynamic = ["version"]
   dependencies = [
     "beautifulsoup4",
-    "billiard",
     "click",
     "html5lib",
     "jsonschema",
@@ -57,6 +56,7 @@
     "scipy>=1.9.0",
     "networkx",
     "pydantic",
+    "pydantic-settings",
     "psutil",
     "pytesseract",
   ]
@@ -68,6 +68,7 @@
       "types-PyYAML",
       "types-python-dateutil",
       "types-requests",
+      "datasets",
       "pytest",
       "pytest-cov",
       "pytest-monitor",
@@ -82,6 +83,14 @@
       "ipython!=8.7.0",
     ]
     test = ["pytest", "coverage", "pytest-cov"]
+    nlp = [
+      "catboost",
+      "optuna",
+      "setfit",
+      "umap-learn[plot]",
+      "plotly",
+      "scikit-learn",
+    ]
 
   [project.urls]
     Homepage = "https://seccerts.org"
@@ -114,7 +123,12 @@
     max-complexity = 10
 
   [tool.setuptools.package-data]
-    "*" = ["*.yaml", "*.json"]
+    'sec_certs' = ["rules.yaml"]
+    'sec_certs.config' = ["settings.yaml", "settings-schema.json"]
+    'sec_certs.data' = [
+      "reference_annotations/split/*.json",
+      "reference_annotations/manual_annotations/final/*.csv",
+    ]
 
 
 [tool.setuptools_scm]
diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index ce4ea166..eb27d265 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -1,34 +1,47 @@
 accessible-pygments==0.0.4
     # via pydata-sphinx-theme
+aiohttp==3.9.0
+    # via
+    #   datasets
+    #   fsspec
+aiosignal==1.3.1
+    # via aiohttp
 alabaster==0.7.13
     # via sphinx
+alembic==1.12.1
+    # via optuna
 annotated-types==0.6.0
     # via pydantic
 appnope==0.1.3
     # via
     #   ipykernel
     #   ipython
-asttokens==2.4.0
+asttokens==2.4.1
     # via stack-data
+async-timeout==4.0.3
+    # via aiohttp
 attrs==23.1.0
     # via
+    #   aiohttp
     #   jsonschema
     #   jupyter-cache
     #   referencing
-babel==2.13.0
+babel==2.13.1
     # via
     #   pydata-sphinx-theme
     #   sphinx
-backcall==0.2.0
-    # via ipython
 beautifulsoup4==4.12.2
     # via
     #   pydata-sphinx-theme
     #   sec-certs (./../pyproject.toml)
-billiard==4.1.0
-    # via sec-certs (./../pyproject.toml)
+bleach==6.1.0
+    # via panel
 blis==0.7.11
     # via thinc
+bokeh==3.3.1
+    # via
+    #   panel
+    #   umap-learn
 build==1.0.3
     # via pip-tools
 catalogue==2.0.10
@@ -36,23 +49,36 @@ catalogue==2.0.10
     #   spacy
     #   srsly
     #   thinc
-certifi==2023.7.22
+catboost==1.2.2
+    # via sec-certs (./../pyproject.toml)
+certifi==2023.11.17
     # via requests
 cffi==1.16.0
     # via cryptography
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via
+    #   dask
     #   jupyter-cache
+    #   nltk
     #   pip-tools
     #   sec-certs (./../pyproject.toml)
     #   typer
 cloudpathlib==0.16.0
     # via weasel
-comm==0.1.4
+cloudpickle==3.0.0
+    # via dask
+colorcet==3.0.1
+    # via
+    #   datashader
+    #   holoviews
+    #   umap-learn
+colorlog==6.7.0
+    # via optuna
+comm==0.2.0
     # via
     #   ipykernel
     #   ipywidgets
@@ -60,13 +86,15 @@ confection==0.1.3
     # via
     #   thinc
     #   weasel
-contourpy==1.1.1
-    # via matplotlib
+contourpy==1.2.0
+    # via
+    #   bokeh
+    #   matplotlib
 coverage[toml]==7.3.2
     # via
     #   pytest-cov
     #   sec-certs (./../pyproject.toml)
-cryptography==41.0.4
+cryptography==41.0.5
     # via pypdf
 cycler==0.12.1
     # via matplotlib
@@ -75,12 +103,26 @@ cymem==2.0.8
     #   preshed
     #   spacy
     #   thinc
+dask==2023.11.0
+    # via datashader
+datasets==2.15.0
+    # via
+    #   evaluate
+    #   sec-certs (./../pyproject.toml)
+    #   setfit
+datashader==0.16.0
+    # via umap-learn
 debugpy==1.8.0
     # via ipykernel
 decorator==5.1.1
     # via ipython
-deprecation==2.1.0
+deprecated==1.2.14
     # via pikepdf
+dill==0.3.7
+    # via
+    #   datasets
+    #   evaluate
+    #   multiprocess
 distlib==0.3.7
     # via virtualenv
 distro==1.8.0
@@ -90,41 +132,75 @@ docutils==0.19
     #   myst-parser
     #   pydata-sphinx-theme
     #   sphinx
-exceptiongroup==1.1.3
+evaluate==0.4.1
+    # via setfit
+exceptiongroup==1.2.0
     # via
     #   ipython
     #   pytest
-executing==2.0.0
+executing==2.0.1
     # via stack-data
-fastjsonschema==2.18.1
+fastjsonschema==2.19.0
     # via nbformat
-filelock==3.12.4
-    # via virtualenv
-fonttools==4.43.1
+filelock==3.13.1
+    # via
+    #   huggingface-hub
+    #   torch
+    #   transformers
+    #   virtualenv
+fonttools==4.45.0
     # via matplotlib
+frozenlist==1.4.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2023.10.0
+    # via
+    #   dask
+    #   datasets
+    #   evaluate
+    #   fsspec
+    #   huggingface-hub
+    #   torch
 gprof2dot==2022.7.29
     # via pytest-profiling
-greenlet==3.0.0
+graphviz==0.20.1
+    # via catboost
+greenlet==3.0.1
     # via sqlalchemy
+holoviews==1.18.1
+    # via umap-learn
 html5lib==1.1
     # via sec-certs (./../pyproject.toml)
-identify==2.5.30
+huggingface-hub==0.19.4
+    # via
+    #   datasets
+    #   evaluate
+    #   sentence-transformers
+    #   tokenizers
+    #   transformers
+identify==2.5.32
     # via pre-commit
 idna==3.4
-    # via requests
+    # via
+    #   requests
+    #   yarl
+imageio==2.33.0
+    # via scikit-image
 imagesize==1.4.1
     # via sphinx
 importlib-metadata==6.8.0
     # via
+    #   dask
     #   jupyter-cache
     #   myst-nb
 iniconfig==2.0.0
     # via pytest
-ipykernel==6.25.2
+ipykernel==6.27.0
     # via
     #   myst-nb
     #   sec-certs (./../pyproject.toml)
-ipython==8.16.1
+ipython==8.17.2
     # via
     #   ipykernel
     #   ipywidgets
@@ -136,26 +212,29 @@ jedi==0.19.1
     # via ipython
 jinja2==3.1.2
     # via
+    #   bokeh
     #   myst-parser
     #   spacy
     #   sphinx
+    #   torch
 joblib==1.3.2
-    # via scikit-learn
-jpype1==1.4.1
-    # via tabula-py
-jsonschema==4.19.1
+    # via
+    #   nltk
+    #   pynndescent
+    #   scikit-learn
+jsonschema==4.20.0
     # via
     #   nbformat
     #   sec-certs (./../pyproject.toml)
-jsonschema-specifications==2023.7.1
+jsonschema-specifications==2023.11.1
     # via jsonschema
-jupyter-cache==0.6.1
+jupyter-cache==1.0.0
     # via myst-nb
-jupyter-client==8.4.0
+jupyter-client==8.6.0
     # via
     #   ipykernel
     #   nbclient
-jupyter-core==5.4.0
+jupyter-core==5.5.0
     # via
     #   ipykernel
     #   jupyter-client
@@ -167,31 +246,64 @@ kiwisolver==1.4.5
     # via matplotlib
 langcodes==3.3.0
     # via spacy
+lazy-loader==0.3
+    # via scikit-image
+linkify-it-py==2.0.2
+    # via panel
+llvmlite==0.41.1
+    # via
+    #   numba
+    #   pynndescent
+locket==1.0.0
+    # via partd
 lxml==4.9.3
     # via
     #   pikepdf
     #   sec-certs (./../pyproject.toml)
-markdown-it-py==2.2.0
+mako==1.3.0
+    # via alembic
+markdown==3.5.1
+    # via panel
+markdown-it-py==3.0.0
     # via
     #   mdit-py-plugins
     #   myst-parser
+    #   panel
 markupsafe==2.1.3
-    # via jinja2
-matplotlib==3.8.0
     # via
+    #   jinja2
+    #   mako
+matplotlib==3.8.2
+    # via
+    #   catboost
     #   pysankeybeta
     #   seaborn
     #   sec-certs (./../pyproject.toml)
+    #   umap-learn
 matplotlib-inline==0.1.6
     # via
     #   ipykernel
     #   ipython
-mdit-py-plugins==0.3.5
-    # via myst-parser
+mdit-py-plugins==0.4.0
+    # via
+    #   myst-parser
+    #   panel
 mdurl==0.1.2
     # via markdown-it-py
 memory-profiler==0.61.0
     # via pytest-monitor
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+multipledispatch==1.0.0
+    # via datashader
+multiprocess==0.70.15
+    # via
+    #   datasets
+    #   evaluate
 murmurhash==1.0.10
     # via
     #   preshed
@@ -201,11 +313,11 @@ mypy==1.6.1
     # via sec-certs (./../pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
-myst-nb==0.17.2
+myst-nb==1.0.0
     # via sec-certs (./../pyproject.toml)
-myst-parser==0.18.1
+myst-parser==2.0.0
     # via myst-nb
-nbclient==0.7.4
+nbclient==0.9.0
     # via
     #   jupyter-cache
     #   myst-nb
@@ -216,70 +328,135 @@ nbformat==5.9.2
     #   nbclient
 nest-asyncio==1.5.8
     # via ipykernel
-networkx==3.1
-    # via sec-certs (./../pyproject.toml)
+networkx==3.2.1
+    # via
+    #   scikit-image
+    #   sec-certs (./../pyproject.toml)
+    #   torch
+nltk==3.8.1
+    # via sentence-transformers
 nodeenv==1.8.0
     # via pre-commit
-numpy==1.26.1
+numba==0.58.1
+    # via
+    #   datashader
+    #   pynndescent
+    #   umap-learn
+numpy==1.26.2
     # via
     #   blis
+    #   bokeh
+    #   catboost
     #   contourpy
+    #   datasets
+    #   datashader
+    #   evaluate
+    #   holoviews
+    #   imageio
     #   matplotlib
+    #   numba
+    #   optuna
     #   pandas
+    #   pyarrow
     #   pysankeybeta
+    #   scikit-image
     #   scikit-learn
     #   scipy
     #   seaborn
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
     #   spacy
     #   tabula-py
     #   thinc
+    #   tifffile
+    #   torchvision
+    #   transformers
+    #   umap-learn
+    #   xarray
+optuna==3.4.0
+    # via sec-certs (./../pyproject.toml)
 packaging==23.2
     # via
+    #   bokeh
     #   build
-    #   deprecation
+    #   dask
+    #   datasets
+    #   evaluate
+    #   holoviews
+    #   huggingface-hub
     #   ipykernel
-    #   jpype1
     #   matplotlib
+    #   optuna
     #   pikepdf
+    #   plotly
     #   pydata-sphinx-theme
     #   pytesseract
     #   pytest
+    #   scikit-image
     #   setuptools-scm
     #   spacy
     #   sphinx
     #   thinc
+    #   transformers
     #   weasel
-pandas==2.1.1
+    #   xarray
+pandas==2.1.3
     # via
+    #   bokeh
+    #   catboost
+    #   datasets
+    #   datashader
+    #   evaluate
+    #   holoviews
+    #   panel
     #   pysankeybeta
     #   seaborn
     #   sec-certs (./../pyproject.toml)
     #   tabula-py
+    #   umap-learn
+    #   xarray
+panel==1.3.2
+    # via holoviews
+param==2.0.1
+    # via
+    #   datashader
+    #   holoviews
+    #   panel
+    #   pyct
+    #   pyviz-comms
 parso==0.8.3
     # via jedi
+partd==1.4.1
+    # via dask
 pdftotext==2.2.2
     # via sec-certs (./../pyproject.toml)
 pexpect==4.8.0
     # via ipython
-pickleshare==0.7.5
-    # via ipython
-pikepdf==8.5.1
+pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
 pillow==10.1.0
     # via
+    #   bokeh
+    #   datashader
+    #   imageio
     #   matplotlib
     #   pikepdf
     #   pytesseract
+    #   scikit-image
     #   sec-certs (./../pyproject.toml)
+    #   torchvision
 pip-tools==7.3.0
     # via sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-platformdirs==3.11.0
+platformdirs==4.0.0
     # via
     #   jupyter-core
     #   virtualenv
+plotly==5.18.0
+    # via
+    #   catboost
+    #   sec-certs (./../pyproject.toml)
 pluggy==1.3.0
     # via pytest
 pre-commit==3.5.0
@@ -288,7 +465,7 @@ preshed==3.0.9
     # via
     #   spacy
     #   thinc
-prompt-toolkit==3.0.39
+prompt-toolkit==3.0.41
     # via ipython
 psutil==5.9.6
     # via
@@ -300,38 +477,51 @@ ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
     # via stack-data
+pyarrow==14.0.1
+    # via datasets
+pyarrow-hotfix==0.6
+    # via datasets
 pycparser==2.21
     # via cffi
-pydantic==2.4.2
+pyct==0.5.0
+    # via
+    #   colorcet
+    #   datashader
+pydantic==2.5.2
     # via
     #   confection
+    #   pydantic-settings
     #   sec-certs (./../pyproject.toml)
     #   spacy
     #   thinc
     #   weasel
-pydantic-core==2.10.1
+pydantic-core==2.14.5
     # via pydantic
-pydata-sphinx-theme==0.14.1
+pydantic-settings==2.1.0
+    # via sec-certs (./../pyproject.toml)
+pydata-sphinx-theme==0.14.3
     # via sphinx-book-theme
-pygments==2.16.1
+pygments==2.17.2
     # via
     #   accessible-pygments
     #   ipython
     #   pydata-sphinx-theme
     #   sphinx
+pynndescent==0.5.11
+    # via umap-learn
 pyparsing==3.1.1
     # via matplotlib
-pypdf[crypto]==3.16.4
+pypdf[crypto]==3.17.1
     # via
     #   pypdf
     #   sec-certs (./../pyproject.toml)
 pyproject-hooks==1.0.0
     # via build
-pysankeybeta==1.4.0
+pysankeybeta==1.4.1
     # via sec-certs (./../pyproject.toml)
 pytesseract==0.3.10
     # via sec-certs (./../pyproject.toml)
-pytest==7.4.2
+pytest==7.4.3
     # via
     #   pytest-cov
     #   pytest-monitor
@@ -349,53 +539,103 @@ python-dateutil==2.8.2
     #   matplotlib
     #   pandas
     #   sec-certs (./../pyproject.toml)
+python-dotenv==1.0.0
+    # via pydantic-settings
 pytz==2023.3.post1
     # via pandas
+pyviz-comms==3.0.0
+    # via
+    #   holoviews
+    #   panel
 pyyaml==6.0.1
     # via
+    #   bokeh
+    #   dask
+    #   datasets
+    #   huggingface-hub
     #   jupyter-cache
     #   myst-nb
     #   myst-parser
+    #   optuna
     #   pre-commit
     #   sec-certs (./../pyproject.toml)
+    #   transformers
 pyzmq==25.1.1
     # via
     #   ipykernel
     #   jupyter-client
-rapidfuzz==3.4.0
+rapidfuzz==3.5.2
     # via sec-certs (./../pyproject.toml)
-referencing==0.30.2
+referencing==0.31.0
     # via
     #   jsonschema
     #   jsonschema-specifications
+regex==2023.10.3
+    # via
+    #   nltk
+    #   transformers
 requests==2.31.0
     # via
+    #   datasets
+    #   datashader
+    #   evaluate
+    #   fsspec
+    #   huggingface-hub
+    #   panel
     #   pytest-monitor
+    #   responses
     #   sec-certs (./../pyproject.toml)
     #   spacy
     #   sphinx
+    #   torchvision
+    #   transformers
     #   weasel
-rpds-py==0.10.6
+responses==0.18.0
+    # via evaluate
+rpds-py==0.13.1
     # via
     #   jsonschema
     #   referencing
 ruff==0.1.5
     # via sec-certs (./../pyproject.toml)
-scikit-learn==1.3.1
-    # via sec-certs (./../pyproject.toml)
-scipy==1.11.3
+safetensors==0.4.0
+    # via transformers
+scikit-image==0.22.0
+    # via umap-learn
+scikit-learn==1.3.2
     # via
+    #   pynndescent
+    #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
+    #   umap-learn
+scipy==1.11.4
+    # via
+    #   catboost
+    #   datashader
+    #   pynndescent
+    #   scikit-image
     #   scikit-learn
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
+    #   umap-learn
 seaborn==0.13.0
     # via
     #   pysankeybeta
     #   sec-certs (./../pyproject.toml)
+    #   umap-learn
+sentence-transformers==2.2.2
+    # via setfit
+sentencepiece==0.1.99
+    # via sentence-transformers
+setfit==0.7.0
+    # via sec-certs (./../pyproject.toml)
 setuptools-scm==8.0.4
     # via sec-certs (./../pyproject.toml)
 six==1.16.0
     # via
     #   asttokens
+    #   bleach
+    #   catboost
     #   html5lib
     #   pytest-profiling
     #   python-dateutil
@@ -413,7 +653,7 @@ spacy-legacy==3.0.12
     # via spacy
 spacy-loggers==1.0.5
     # via spacy
-sphinx==5.3.0
+sphinx==6.2.1
     # via
     #   myst-nb
     #   myst-parser
@@ -445,8 +685,11 @@ sphinxcontrib-qthelp==1.0.6
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.9
     # via sphinx
-sqlalchemy==2.0.22
-    # via jupyter-cache
+sqlalchemy==2.0.23
+    # via
+    #   alembic
+    #   jupyter-cache
+    #   optuna
 srsly==2.4.8
     # via
     #   confection
@@ -455,14 +698,22 @@ srsly==2.4.8
     #   weasel
 stack-data==0.6.3
     # via ipython
-tabula-py==2.8.2
+sympy==1.12
+    # via torch
+tabula-py==2.9.0
     # via sec-certs (./../pyproject.toml)
 tabulate==0.9.0
     # via jupyter-cache
+tenacity==8.2.3
+    # via plotly
 thinc==8.2.1
     # via spacy
 threadpoolctl==3.2.0
     # via scikit-learn
+tifffile==2023.9.26
+    # via scikit-image
+tokenizers==0.15.0
+    # via transformers
 tomli==2.0.1
     # via
     #   build
@@ -472,15 +723,36 @@ tomli==2.0.1
     #   pyproject-hooks
     #   pytest
     #   setuptools-scm
+toolz==0.12.0
+    # via
+    #   dask
+    #   datashader
+    #   partd
+torch==2.1.1
+    # via
+    #   sentence-transformers
+    #   torchvision
+torchvision==0.16.1
+    # via sentence-transformers
 tornado==6.3.3
     # via
+    #   bokeh
     #   ipykernel
     #   jupyter-client
 tqdm==4.66.1
     # via
+    #   datasets
+    #   evaluate
+    #   huggingface-hub
+    #   nltk
+    #   optuna
+    #   panel
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
     #   spacy
-traitlets==5.11.2
+    #   transformers
+    #   umap-learn
+traitlets==5.13.0
     # via
     #   comm
     #   ipykernel
@@ -491,6 +763,8 @@ traitlets==5.11.2
     #   matplotlib-inline
     #   nbclient
     #   nbformat
+transformers==4.35.2
+    # via sentence-transformers
 typer==0.9.0
     # via
     #   spacy
@@ -503,41 +777,65 @@ types-requests==2.31.0.10
     # via sec-certs (./../pyproject.toml)
 typing-extensions==4.8.0
     # via
+    #   alembic
     #   cloudpathlib
+    #   huggingface-hub
     #   mypy
     #   myst-nb
-    #   myst-parser
+    #   panel
     #   pydantic
     #   pydantic-core
     #   pydata-sphinx-theme
     #   setuptools-scm
     #   sqlalchemy
+    #   torch
     #   typer
 tzdata==2023.3
     # via pandas
-urllib3==2.0.7
+uc-micro-py==1.0.2
+    # via linkify-it-py
+umap-learn[plot]==0.5.5
+    # via sec-certs (./../pyproject.toml)
+urllib3==2.1.0
     # via
     #   requests
+    #   responses
     #   types-requests
-virtualenv==20.24.5
+virtualenv==20.24.7
     # via pre-commit
 wasabi==1.1.2
     # via
     #   spacy
     #   thinc
     #   weasel
-wcwidth==0.2.8
+wcwidth==0.2.12
     # via prompt-toolkit
-weasel==0.3.3
+weasel==0.3.4
     # via spacy
 webencodings==0.5.1
-    # via html5lib
-wheel==0.41.2
+    # via
+    #   bleach
+    #   html5lib
+wheel==0.41.3
     # via
     #   pip-tools
     #   pytest-monitor
 widgetsnbextension==4.0.9
     # via ipywidgets
+wrapt==1.16.0
+    # via deprecated
+xarray==2023.11.0
+    # via datashader
+xxhash==3.4.1
+    # via
+    #   datasets
+    #   evaluate
+xyzservices==2023.10.1
+    # via
+    #   bokeh
+    #   panel
+yarl==1.9.3
+    # via aiohttp
 zipp==3.17.0
     # via importlib-metadata
 
diff --git a/requirements/compile.sh b/requirements/compile.sh
index 063582a1..743006b2 100755
--- a/requirements/compile.sh
+++ b/requirements/compile.sh
@@ -5,4 +5,5 @@
 pip-compile --no-header -o requirements.txt ./../pyproject.toml
 pip-compile --no-header --extra dev -o dev_requirements.txt ./../pyproject.toml
 pip-compile --no-header --extra test -o test_requirements.txt ./../pyproject.toml
-pip-compile --no-header --extra dev --extra test -o all_requirements.txt ./../pyproject.toml
+pip-compile --no-header --extra nlp -o nlp_requirements.txt ./../pyproject.toml
+pip-compile --no-header --extra dev --extra test --extra nlp -o all_requirements.txt ./../pyproject.toml
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index 42ce14d1..55f9d7b9 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -1,162 +1,207 @@
-alabaster==0.7.12
+accessible-pygments==0.0.4
+    # via pydata-sphinx-theme
+aiohttp==3.9.0
+    # via
+    #   datasets
+    #   fsspec
+aiosignal==1.3.1
+    # via aiohttp
+alabaster==0.7.13
     # via sphinx
+annotated-types==0.6.0
+    # via pydantic
 appnope==0.1.3
     # via
     #   ipykernel
     #   ipython
-asttokens==2.2.1
+asttokens==2.4.1
     # via stack-data
-attrs==22.1.0
+async-timeout==4.0.3
+    # via aiohttp
+attrs==23.1.0
     # via
+    #   aiohttp
     #   jsonschema
     #   jupyter-cache
-    #   pytest
-babel==2.11.0
-    # via sphinx
-backcall==0.2.0
-    # via ipython
-beautifulsoup4==4.11.1
+    #   referencing
+babel==2.13.1
+    # via
+    #   pydata-sphinx-theme
+    #   sphinx
+beautifulsoup4==4.12.2
     # via
     #   pydata-sphinx-theme
     #   sec-certs (./../pyproject.toml)
-billiard==4.0.2
-    # via sec-certs (./../pyproject.toml)
-blis==0.7.9
+blis==0.7.11
     # via thinc
-build==0.9.0
+build==1.0.3
     # via pip-tools
-catalogue==2.0.8
+catalogue==2.0.10
     # via
     #   spacy
     #   srsly
     #   thinc
-certifi==2023.7.22
+certifi==2023.11.17
     # via requests
-cfgv==3.3.1
+cffi==1.16.0
+    # via cryptography
+cfgv==3.4.0
     # via pre-commit
-charset-normalizer==2.1.1
+charset-normalizer==3.3.2
     # via requests
-click==8.1.3
+click==8.1.7
     # via
     #   jupyter-cache
     #   pip-tools
     #   sec-certs (./../pyproject.toml)
     #   typer
-comm==0.1.2
-    # via ipykernel
-confection==0.0.3
-    # via thinc
-contourpy==1.0.6
+cloudpathlib==0.16.0
+    # via weasel
+comm==0.2.0
+    # via
+    #   ipykernel
+    #   ipywidgets
+confection==0.1.3
+    # via
+    #   thinc
+    #   weasel
+contourpy==1.2.0
     # via matplotlib
-coverage[toml]==6.5.0
+coverage[toml]==7.3.2
     # via
     #   coverage
     #   pytest-cov
-cycler==0.11.0
+cryptography==41.0.5
+    # via pypdf
+cycler==0.12.1
     # via matplotlib
-cymem==2.0.7
+cymem==2.0.8
     # via
     #   preshed
     #   spacy
     #   thinc
-debugpy==1.6.4
+datasets==2.15.0
+    # via sec-certs (./../pyproject.toml)
+debugpy==1.8.0
     # via ipykernel
 decorator==5.1.1
     # via ipython
-deprecation==2.1.0
+deprecated==1.2.14
     # via pikepdf
-distlib==0.3.6
+dill==0.3.7
+    # via
+    #   datasets
+    #   multiprocess
+distlib==0.3.7
     # via virtualenv
 distro==1.8.0
     # via tabula-py
-docutils==0.17.1
+docutils==0.19
     # via
     #   myst-parser
     #   pydata-sphinx-theme
     #   sphinx
-entrypoints==0.4
-    # via jupyter-client
-exceptiongroup==1.1.3
-    # via pytest
-executing==1.2.0
+exceptiongroup==1.2.0
+    # via
+    #   ipython
+    #   pytest
+executing==2.0.1
     # via stack-data
-fastjsonschema==2.16.2
+fastjsonschema==2.19.0
     # via nbformat
-filelock==3.8.2
-    # via virtualenv
-fonttools==4.38.0
+filelock==3.13.1
+    # via
+    #   huggingface-hub
+    #   virtualenv
+fonttools==4.45.0
     # via matplotlib
+frozenlist==1.4.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2023.10.0
+    # via
+    #   datasets
+    #   fsspec
+    #   huggingface-hub
 gprof2dot==2022.7.29
     # via pytest-profiling
-greenlet==2.0.1
+greenlet==3.0.1
     # via sqlalchemy
 html5lib==1.1
     # via sec-certs (./../pyproject.toml)
-identify==2.5.9
+huggingface-hub==0.19.4
+    # via datasets
+identify==2.5.32
     # via pre-commit
 idna==3.4
-    # via requests
+    # via
+    #   requests
+    #   yarl
 imagesize==1.4.1
     # via sphinx
-importlib-metadata==5.1.0
+importlib-metadata==6.8.0
     # via
     #   jupyter-cache
     #   myst-nb
-iniconfig==1.1.1
+iniconfig==2.0.0
     # via pytest
-ipykernel==6.19.1
+ipykernel==6.27.0
     # via
-    #   ipywidgets
     #   myst-nb
     #   sec-certs (./../pyproject.toml)
-ipython==8.10.0
+ipython==8.17.2
     # via
     #   ipykernel
     #   ipywidgets
     #   myst-nb
     #   sec-certs (./../pyproject.toml)
-ipywidgets==8.0.3
+ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
-jedi==0.18.2
+jedi==0.19.1
     # via ipython
 jinja2==3.1.2
     # via
     #   myst-parser
     #   spacy
     #   sphinx
-joblib==1.2.0
+joblib==1.3.2
     # via scikit-learn
-jsonschema==4.17.3
+jsonschema==4.20.0
     # via
     #   nbformat
     #   sec-certs (./../pyproject.toml)
-jupyter-cache==0.5.0
+jsonschema-specifications==2023.11.1
+    # via jsonschema
+jupyter-cache==1.0.0
     # via myst-nb
-jupyter-client==7.4.8
+jupyter-client==8.6.0
     # via
     #   ipykernel
     #   nbclient
-jupyter-core==5.1.0
+jupyter-core==5.5.0
     # via
+    #   ipykernel
     #   jupyter-client
+    #   nbclient
     #   nbformat
-jupyterlab-widgets==3.0.4
+jupyterlab-widgets==3.0.9
     # via ipywidgets
-kiwisolver==1.4.4
+kiwisolver==1.4.5
     # via matplotlib
 langcodes==3.3.0
     # via spacy
-lxml==4.9.1
+lxml==4.9.3
     # via
     #   pikepdf
     #   sec-certs (./../pyproject.toml)
-markdown-it-py==2.2.0
+markdown-it-py==3.0.0
     # via
     #   mdit-py-plugins
     #   myst-parser
-markupsafe==2.1.1
+markupsafe==2.1.3
     # via jinja2
-matplotlib==3.6.2
+matplotlib==3.8.2
     # via
     #   pysankeybeta
     #   seaborn
@@ -165,13 +210,19 @@ matplotlib-inline==0.1.6
     # via
     #   ipykernel
     #   ipython
-mdit-py-plugins==0.3.3
+mdit-py-plugins==0.4.0
     # via myst-parser
 mdurl==0.1.2
     # via markdown-it-py
 memory-profiler==0.61.0
     # via pytest-monitor
-murmurhash==1.0.9
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.15
+    # via datasets
+murmurhash==1.0.10
     # via
     #   preshed
     #   spacy
@@ -180,34 +231,33 @@ mypy==1.6.1
     # via sec-certs (./../pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
-myst-nb==0.17.1
+myst-nb==1.0.0
     # via sec-certs (./../pyproject.toml)
-myst-parser==0.18.1
+myst-parser==2.0.0
     # via myst-nb
-nbclient==0.5.13
+nbclient==0.9.0
     # via
     #   jupyter-cache
     #   myst-nb
-nbformat==5.7.0
+nbformat==5.9.2
     # via
     #   jupyter-cache
     #   myst-nb
     #   nbclient
-nest-asyncio==1.5.6
-    # via
-    #   ipykernel
-    #   jupyter-client
-    #   nbclient
-networkx==2.8.8
+nest-asyncio==1.5.8
+    # via ipykernel
+networkx==3.2.1
     # via sec-certs (./../pyproject.toml)
-nodeenv==1.7.0
+nodeenv==1.8.0
     # via pre-commit
-numpy==1.23.5
+numpy==1.26.2
     # via
     #   blis
     #   contourpy
+    #   datasets
     #   matplotlib
     #   pandas
+    #   pyarrow
     #   pysankeybeta
     #   scikit-learn
     #   scipy
@@ -216,10 +266,11 @@ numpy==1.23.5
     #   spacy
     #   tabula-py
     #   thinc
-packaging==22.0
+packaging==23.2
     # via
     #   build
-    #   deprecation
+    #   datasets
+    #   huggingface-hub
     #   ipykernel
     #   matplotlib
     #   pikepdf
@@ -229,51 +280,48 @@ packaging==22.0
     #   setuptools-scm
     #   spacy
     #   sphinx
-pandas==1.5.2
+    #   thinc
+    #   weasel
+pandas==2.1.3
     # via
+    #   datasets
     #   pysankeybeta
     #   seaborn
     #   sec-certs (./../pyproject.toml)
     #   tabula-py
 parso==0.8.3
     # via jedi
-pathy==0.10.1
-    # via spacy
 pdftotext==2.2.2
     # via sec-certs (./../pyproject.toml)
-pep517==0.13.0
-    # via build
 pexpect==4.8.0
     # via ipython
-pickleshare==0.7.5
-    # via ipython
-pikepdf==6.2.5
+pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.0.1
+pillow==10.1.0
     # via
     #   matplotlib
     #   pikepdf
     #   pytesseract
     #   sec-certs (./../pyproject.toml)
-pip-tools==6.11.0
+pip-tools==7.3.0
     # via sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-platformdirs==2.6.0
+platformdirs==4.0.0
     # via
     #   jupyter-core
     #   virtualenv
-pluggy==1.0.0
+pluggy==1.3.0
     # via pytest
-pre-commit==2.20.0
+pre-commit==3.5.0
     # via sec-certs (./../pyproject.toml)
-preshed==3.0.8
+preshed==3.0.9
     # via
     #   spacy
     #   thinc
-prompt-toolkit==3.0.36
+prompt-toolkit==3.0.41
     # via ipython
-psutil==5.9.4
+psutil==5.9.6
     # via
     #   ipykernel
     #   memory-profiler
@@ -283,41 +331,53 @@ ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
     # via stack-data
-pycryptodome==3.16.0
-    # via pypdf
-pydantic==1.10.2
+pyarrow==14.0.1
+    # via datasets
+pyarrow-hotfix==0.6
+    # via datasets
+pycparser==2.21
+    # via cffi
+pydantic==2.5.2
     # via
     #   confection
+    #   pydantic-settings
     #   sec-certs (./../pyproject.toml)
     #   spacy
     #   thinc
-pydata-sphinx-theme==0.8.1
+    #   weasel
+pydantic-core==2.14.5
+    # via pydantic
+pydantic-settings==2.1.0
+    # via sec-certs (./../pyproject.toml)
+pydata-sphinx-theme==0.14.3
     # via sphinx-book-theme
-pygments==2.15.0
+pygments==2.17.2
     # via
+    #   accessible-pygments
     #   ipython
+    #   pydata-sphinx-theme
     #   sphinx
-pyparsing==3.0.9
+pyparsing==3.1.1
     # via matplotlib
-pypdf[crypto]==3.2.1
+pypdf[crypto]==3.17.1
     # via
     #   pypdf
     #   sec-certs (./../pyproject.toml)
-pyrsistent==0.19.2
-    # via jsonschema
-pysankeybeta==1.4.0
+pyproject-hooks==1.0.0
+    # via build
+pysankeybeta==1.4.1
     # via sec-certs (./../pyproject.toml)
 pytesseract==0.3.10
     # via sec-certs (./../pyproject.toml)
-pytest==7.2.0
+pytest==7.4.3
     # via
     #   pytest-cov
     #   pytest-monitor
     #   pytest-profiling
     #   sec-certs (./../pyproject.toml)
-pytest-cov==4.0.0
+pytest-cov==4.1.0
     # via sec-certs (./../pyproject.toml)
-pytest-monitor==1.6.5
+pytest-monitor==1.6.6
     # via sec-certs (./../pyproject.toml)
 pytest-profiling==1.7.0
     # via sec-certs (./../pyproject.toml)
@@ -327,43 +387,56 @@ python-dateutil==2.8.2
     #   matplotlib
     #   pandas
     #   sec-certs (./../pyproject.toml)
-pytz==2022.6
-    # via
-    #   babel
-    #   pandas
-pyyaml==6.0
+python-dotenv==1.0.0
+    # via pydantic-settings
+pytz==2023.3.post1
+    # via pandas
+pyyaml==6.0.1
     # via
+    #   datasets
+    #   huggingface-hub
     #   jupyter-cache
     #   myst-nb
     #   myst-parser
     #   pre-commit
     #   sec-certs (./../pyproject.toml)
-    #   sphinx-book-theme
-pyzmq==24.0.1
+pyzmq==25.1.1
     # via
     #   ipykernel
     #   jupyter-client
-rapidfuzz==2.13.3
+rapidfuzz==3.5.2
     # via sec-certs (./../pyproject.toml)
+referencing==0.31.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 requests==2.31.0
     # via
+    #   datasets
+    #   fsspec
+    #   huggingface-hub
     #   pytest-monitor
     #   sec-certs (./../pyproject.toml)
     #   spacy
     #   sphinx
+    #   weasel
+rpds-py==0.13.1
+    # via
+    #   jsonschema
+    #   referencing
 ruff==0.1.5
     # via sec-certs (./../pyproject.toml)
-scikit-learn==1.2.0
+scikit-learn==1.3.2
     # via sec-certs (./../pyproject.toml)
-scipy==1.10.0
+scipy==1.11.4
     # via
     #   scikit-learn
     #   sec-certs (./../pyproject.toml)
-seaborn==0.12.1
+seaborn==0.13.0
     # via
     #   pysankeybeta
     #   sec-certs (./../pyproject.toml)
-setuptools-scm==7.0.5
+setuptools-scm==8.0.4
     # via sec-certs (./../pyproject.toml)
 six==1.16.0
     # via
@@ -371,19 +444,21 @@ six==1.16.0
     #   html5lib
     #   pytest-profiling
     #   python-dateutil
-smart-open==6.2.0
-    # via pathy
+smart-open==6.4.0
+    # via
+    #   spacy
+    #   weasel
 snowballstemmer==2.2.0
     # via sphinx
-soupsieve==2.3.2.post1
+soupsieve==2.5
     # via beautifulsoup4
-spacy==3.4.3
+spacy==3.7.2
     # via sec-certs (./../pyproject.toml)
-spacy-legacy==3.0.10
+spacy-legacy==3.0.12
     # via spacy
-spacy-loggers==1.0.4
+spacy-loggers==1.0.5
     # via spacy
-sphinx==4.5.0
+sphinx==6.2.1
     # via
     #   myst-nb
     #   myst-parser
@@ -392,60 +467,67 @@ sphinx==4.5.0
     #   sphinx-book-theme
     #   sphinx-copybutton
     #   sphinx-design
-sphinx-book-theme==0.3.3
+    #   sphinxcontrib-applehelp
+    #   sphinxcontrib-devhelp
+    #   sphinxcontrib-htmlhelp
+    #   sphinxcontrib-qthelp
+    #   sphinxcontrib-serializinghtml
+sphinx-book-theme==1.0.1
     # via sec-certs (./../pyproject.toml)
-sphinx-copybutton==0.5.1
+sphinx-copybutton==0.5.2
     # via sec-certs (./../pyproject.toml)
-sphinx-design==0.3.0
+sphinx-design==0.5.0
     # via sec-certs (./../pyproject.toml)
-sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-applehelp==1.0.7
     # via sphinx
-sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-devhelp==1.0.5
     # via sphinx
-sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-htmlhelp==2.0.4
     # via sphinx
 sphinxcontrib-jsmath==1.0.1
     # via sphinx
-sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-qthelp==1.0.6
     # via sphinx
-sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-serializinghtml==1.1.9
     # via sphinx
-sqlalchemy==1.4.44
+sqlalchemy==2.0.23
     # via jupyter-cache
-srsly==2.4.5
+srsly==2.4.8
     # via
     #   confection
     #   spacy
     #   thinc
-stack-data==0.6.2
+    #   weasel
+stack-data==0.6.3
     # via ipython
-tabula-py==2.6.0
+tabula-py==2.9.0
     # via sec-certs (./../pyproject.toml)
 tabulate==0.9.0
     # via jupyter-cache
-thinc==8.1.5
+thinc==8.2.1
     # via spacy
-threadpoolctl==3.1.0
+threadpoolctl==3.2.0
     # via scikit-learn
-toml==0.10.2
-    # via pre-commit
 tomli==2.0.1
     # via
     #   build
     #   coverage
     #   mypy
-    #   pep517
+    #   pip-tools
+    #   pyproject-hooks
     #   pytest
     #   setuptools-scm
 tornado==6.3.3
     # via
     #   ipykernel
     #   jupyter-client
-tqdm==4.64.1
+tqdm==4.66.1
     # via
+    #   datasets
+    #   huggingface-hub
     #   sec-certs (./../pyproject.toml)
     #   spacy
-traitlets==5.6.0
+traitlets==5.13.0
     # via
     #   comm
     #   ipykernel
@@ -456,44 +538,60 @@ traitlets==5.6.0
     #   matplotlib-inline
     #   nbclient
     #   nbformat
-typer==0.7.0
+typer==0.9.0
     # via
-    #   pathy
     #   spacy
-types-python-dateutil==2.8.19.4
+    #   weasel
+types-python-dateutil==2.8.19.14
     # via sec-certs (./../pyproject.toml)
-types-pyyaml==6.0.12.2
+types-pyyaml==6.0.12.12
     # via sec-certs (./../pyproject.toml)
-types-requests==2.28.11.5
+types-requests==2.31.0.10
     # via sec-certs (./../pyproject.toml)
-types-urllib3==1.26.25.4
-    # via types-requests
-typing-extensions==4.4.0
+typing-extensions==4.8.0
     # via
+    #   cloudpathlib
+    #   huggingface-hub
     #   mypy
     #   myst-nb
-    #   myst-parser
     #   pydantic
+    #   pydantic-core
+    #   pydata-sphinx-theme
     #   setuptools-scm
-urllib3==1.26.18
-    # via requests
-virtualenv==20.17.1
+    #   sqlalchemy
+    #   typer
+tzdata==2023.3
+    # via pandas
+urllib3==2.1.0
+    # via
+    #   requests
+    #   types-requests
+virtualenv==20.24.7
     # via pre-commit
-wasabi==0.10.1
+wasabi==1.1.2
     # via
     #   spacy
     #   thinc
-wcwidth==0.2.5
+    #   weasel
+wcwidth==0.2.12
     # via prompt-toolkit
+weasel==0.3.4
+    # via spacy
 webencodings==0.5.1
     # via html5lib
-wheel==0.38.4
+wheel==0.41.3
     # via
     #   pip-tools
     #   pytest-monitor
-widgetsnbextension==4.0.4
+widgetsnbextension==4.0.9
     # via ipywidgets
-zipp==3.11.0
+wrapt==1.16.0
+    # via deprecated
+xxhash==3.4.1
+    # via datasets
+yarl==1.9.3
+    # via aiohttp
+zipp==3.17.0
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
new file mode 100644
index 00000000..d9d25188
--- /dev/null
+++ b/requirements/nlp_requirements.txt
@@ -0,0 +1,655 @@
+aiohttp==3.9.0
+    # via
+    #   datasets
+    #   fsspec
+aiosignal==1.3.1
+    # via aiohttp
+alembic==1.12.1
+    # via optuna
+annotated-types==0.6.0
+    # via pydantic
+appnope==0.1.3
+    # via
+    #   ipykernel
+    #   ipython
+asttokens==2.4.1
+    # via stack-data
+async-timeout==4.0.3
+    # via aiohttp
+attrs==23.1.0
+    # via
+    #   aiohttp
+    #   jsonschema
+    #   referencing
+beautifulsoup4==4.12.2
+    # via sec-certs (./../pyproject.toml)
+bleach==6.1.0
+    # via panel
+blis==0.7.11
+    # via thinc
+bokeh==3.3.1
+    # via
+    #   panel
+    #   umap-learn
+catalogue==2.0.10
+    # via
+    #   spacy
+    #   srsly
+    #   thinc
+catboost==1.2.2
+    # via sec-certs (./../pyproject.toml)
+certifi==2023.11.17
+    # via requests
+cffi==1.16.0
+    # via cryptography
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via
+    #   dask
+    #   nltk
+    #   sec-certs (./../pyproject.toml)
+    #   typer
+cloudpathlib==0.16.0
+    # via weasel
+cloudpickle==3.0.0
+    # via dask
+colorcet==3.0.1
+    # via
+    #   datashader
+    #   holoviews
+    #   umap-learn
+colorlog==6.7.0
+    # via optuna
+comm==0.2.0
+    # via
+    #   ipykernel
+    #   ipywidgets
+confection==0.1.3
+    # via
+    #   thinc
+    #   weasel
+contourpy==1.2.0
+    # via
+    #   bokeh
+    #   matplotlib
+cryptography==41.0.5
+    # via pypdf
+cycler==0.12.1
+    # via matplotlib
+cymem==2.0.8
+    # via
+    #   preshed
+    #   spacy
+    #   thinc
+dask==2023.11.0
+    # via datashader
+datasets==2.15.0
+    # via
+    #   evaluate
+    #   setfit
+datashader==0.16.0
+    # via umap-learn
+debugpy==1.8.0
+    # via ipykernel
+decorator==5.1.1
+    # via ipython
+deprecated==1.2.14
+    # via pikepdf
+dill==0.3.7
+    # via
+    #   datasets
+    #   evaluate
+    #   multiprocess
+distro==1.8.0
+    # via tabula-py
+evaluate==0.4.1
+    # via setfit
+exceptiongroup==1.2.0
+    # via ipython
+executing==2.0.1
+    # via stack-data
+filelock==3.13.1
+    # via
+    #   huggingface-hub
+    #   torch
+    #   transformers
+fonttools==4.45.0
+    # via matplotlib
+frozenlist==1.4.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2023.10.0
+    # via
+    #   dask
+    #   datasets
+    #   evaluate
+    #   fsspec
+    #   huggingface-hub
+    #   torch
+graphviz==0.20.1
+    # via catboost
+greenlet==3.0.1
+    # via sqlalchemy
+holoviews==1.18.1
+    # via umap-learn
+html5lib==1.1
+    # via sec-certs (./../pyproject.toml)
+huggingface-hub==0.19.4
+    # via
+    #   datasets
+    #   evaluate
+    #   sentence-transformers
+    #   tokenizers
+    #   transformers
+idna==3.4
+    # via
+    #   requests
+    #   yarl
+imageio==2.33.0
+    # via scikit-image
+importlib-metadata==6.8.0
+    # via dask
+ipykernel==6.27.0
+    # via sec-certs (./../pyproject.toml)
+ipython==8.17.2
+    # via
+    #   ipykernel
+    #   ipywidgets
+ipywidgets==8.1.1
+    # via sec-certs (./../pyproject.toml)
+jedi==0.19.1
+    # via ipython
+jinja2==3.1.2
+    # via
+    #   bokeh
+    #   spacy
+    #   torch
+joblib==1.3.2
+    # via
+    #   nltk
+    #   pynndescent
+    #   scikit-learn
+jsonschema==4.20.0
+    # via sec-certs (./../pyproject.toml)
+jsonschema-specifications==2023.11.1
+    # via jsonschema
+jupyter-client==8.6.0
+    # via ipykernel
+jupyter-core==5.5.0
+    # via
+    #   ipykernel
+    #   jupyter-client
+jupyterlab-widgets==3.0.9
+    # via ipywidgets
+kiwisolver==1.4.5
+    # via matplotlib
+langcodes==3.3.0
+    # via spacy
+lazy-loader==0.3
+    # via scikit-image
+linkify-it-py==2.0.2
+    # via panel
+llvmlite==0.41.1
+    # via
+    #   numba
+    #   pynndescent
+locket==1.0.0
+    # via partd
+lxml==4.9.3
+    # via
+    #   pikepdf
+    #   sec-certs (./../pyproject.toml)
+mako==1.3.0
+    # via alembic
+markdown==3.5.1
+    # via panel
+markdown-it-py==3.0.0
+    # via
+    #   mdit-py-plugins
+    #   panel
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   mako
+matplotlib==3.8.2
+    # via
+    #   catboost
+    #   pysankeybeta
+    #   seaborn
+    #   sec-certs (./../pyproject.toml)
+    #   umap-learn
+matplotlib-inline==0.1.6
+    # via
+    #   ipykernel
+    #   ipython
+mdit-py-plugins==0.4.0
+    # via panel
+mdurl==0.1.2
+    # via markdown-it-py
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+multipledispatch==1.0.0
+    # via datashader
+multiprocess==0.70.15
+    # via
+    #   datasets
+    #   evaluate
+murmurhash==1.0.10
+    # via
+    #   preshed
+    #   spacy
+    #   thinc
+nest-asyncio==1.5.8
+    # via ipykernel
+networkx==3.2.1
+    # via
+    #   scikit-image
+    #   sec-certs (./../pyproject.toml)
+    #   torch
+nltk==3.8.1
+    # via sentence-transformers
+numba==0.58.1
+    # via
+    #   datashader
+    #   pynndescent
+    #   umap-learn
+numpy==1.26.2
+    # via
+    #   blis
+    #   bokeh
+    #   catboost
+    #   contourpy
+    #   datasets
+    #   datashader
+    #   evaluate
+    #   holoviews
+    #   imageio
+    #   matplotlib
+    #   numba
+    #   optuna
+    #   pandas
+    #   pyarrow
+    #   pysankeybeta
+    #   scikit-image
+    #   scikit-learn
+    #   scipy
+    #   seaborn
+    #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
+    #   spacy
+    #   tabula-py
+    #   thinc
+    #   tifffile
+    #   torchvision
+    #   transformers
+    #   umap-learn
+    #   xarray
+optuna==3.4.0
+    # via sec-certs (./../pyproject.toml)
+packaging==23.2
+    # via
+    #   bokeh
+    #   dask
+    #   datasets
+    #   evaluate
+    #   holoviews
+    #   huggingface-hub
+    #   ipykernel
+    #   matplotlib
+    #   optuna
+    #   pikepdf
+    #   plotly
+    #   pytesseract
+    #   scikit-image
+    #   setuptools-scm
+    #   spacy
+    #   thinc
+    #   transformers
+    #   weasel
+    #   xarray
+pandas==2.1.3
+    # via
+    #   bokeh
+    #   catboost
+    #   datasets
+    #   datashader
+    #   evaluate
+    #   holoviews
+    #   panel
+    #   pysankeybeta
+    #   seaborn
+    #   sec-certs (./../pyproject.toml)
+    #   tabula-py
+    #   umap-learn
+    #   xarray
+panel==1.3.2
+    # via holoviews
+param==2.0.1
+    # via
+    #   datashader
+    #   holoviews
+    #   panel
+    #   pyct
+    #   pyviz-comms
+parso==0.8.3
+    # via jedi
+partd==1.4.1
+    # via dask
+pdftotext==2.2.2
+    # via sec-certs (./../pyproject.toml)
+pexpect==4.8.0
+    # via ipython
+pikepdf==8.7.1
+    # via sec-certs (./../pyproject.toml)
+pillow==10.1.0
+    # via
+    #   bokeh
+    #   datashader
+    #   imageio
+    #   matplotlib
+    #   pikepdf
+    #   pytesseract
+    #   scikit-image
+    #   sec-certs (./../pyproject.toml)
+    #   torchvision
+pkgconfig==1.5.5
+    # via sec-certs (./../pyproject.toml)
+platformdirs==4.0.0
+    # via jupyter-core
+plotly==5.18.0
+    # via
+    #   catboost
+    #   sec-certs (./../pyproject.toml)
+preshed==3.0.9
+    # via
+    #   spacy
+    #   thinc
+prompt-toolkit==3.0.41
+    # via ipython
+psutil==5.9.6
+    # via
+    #   ipykernel
+    #   sec-certs (./../pyproject.toml)
+ptyprocess==0.7.0
+    # via pexpect
+pure-eval==0.2.2
+    # via stack-data
+pyarrow==14.0.1
+    # via datasets
+pyarrow-hotfix==0.6
+    # via datasets
+pycparser==2.21
+    # via cffi
+pyct==0.5.0
+    # via
+    #   colorcet
+    #   datashader
+pydantic==2.5.2
+    # via
+    #   confection
+    #   pydantic-settings
+    #   sec-certs (./../pyproject.toml)
+    #   spacy
+    #   thinc
+    #   weasel
+pydantic-core==2.14.5
+    # via pydantic
+pydantic-settings==2.1.0
+    # via sec-certs (./../pyproject.toml)
+pygments==2.17.2
+    # via ipython
+pynndescent==0.5.11
+    # via umap-learn
+pyparsing==3.1.1
+    # via matplotlib
+pypdf[crypto]==3.17.1
+    # via
+    #   pypdf
+    #   sec-certs (./../pyproject.toml)
+pysankeybeta==1.4.1
+    # via sec-certs (./../pyproject.toml)
+pytesseract==0.3.10
+    # via sec-certs (./../pyproject.toml)
+python-dateutil==2.8.2
+    # via
+    #   jupyter-client
+    #   matplotlib
+    #   pandas
+    #   sec-certs (./../pyproject.toml)
+python-dotenv==1.0.0
+    # via pydantic-settings
+pytz==2023.3.post1
+    # via pandas
+pyviz-comms==3.0.0
+    # via
+    #   holoviews
+    #   panel
+pyyaml==6.0.1
+    # via
+    #   bokeh
+    #   dask
+    #   datasets
+    #   huggingface-hub
+    #   optuna
+    #   sec-certs (./../pyproject.toml)
+    #   transformers
+pyzmq==25.1.1
+    # via
+    #   ipykernel
+    #   jupyter-client
+rapidfuzz==3.5.2
+    # via sec-certs (./../pyproject.toml)
+referencing==0.31.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+regex==2023.10.3
+    # via
+    #   nltk
+    #   transformers
+requests==2.31.0
+    # via
+    #   datasets
+    #   datashader
+    #   evaluate
+    #   fsspec
+    #   huggingface-hub
+    #   panel
+    #   responses
+    #   sec-certs (./../pyproject.toml)
+    #   spacy
+    #   torchvision
+    #   transformers
+    #   weasel
+responses==0.18.0
+    # via evaluate
+rpds-py==0.13.1
+    # via
+    #   jsonschema
+    #   referencing
+safetensors==0.4.0
+    # via transformers
+scikit-image==0.22.0
+    # via umap-learn
+scikit-learn==1.3.2
+    # via
+    #   pynndescent
+    #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
+    #   umap-learn
+scipy==1.11.4
+    # via
+    #   catboost
+    #   datashader
+    #   pynndescent
+    #   scikit-image
+    #   scikit-learn
+    #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
+    #   umap-learn
+seaborn==0.13.0
+    # via
+    #   pysankeybeta
+    #   sec-certs (./../pyproject.toml)
+    #   umap-learn
+sentence-transformers==2.2.2
+    # via setfit
+sentencepiece==0.1.99
+    # via sentence-transformers
+setfit==0.7.0
+    # via sec-certs (./../pyproject.toml)
+setuptools-scm==8.0.4
+    # via sec-certs (./../pyproject.toml)
+six==1.16.0
+    # via
+    #   asttokens
+    #   bleach
+    #   catboost
+    #   html5lib
+    #   python-dateutil
+smart-open==6.4.0
+    # via
+    #   spacy
+    #   weasel
+soupsieve==2.5
+    # via beautifulsoup4
+spacy==3.7.2
+    # via sec-certs (./../pyproject.toml)
+spacy-legacy==3.0.12
+    # via spacy
+spacy-loggers==1.0.5
+    # via spacy
+sqlalchemy==2.0.23
+    # via
+    #   alembic
+    #   optuna
+srsly==2.4.8
+    # via
+    #   confection
+    #   spacy
+    #   thinc
+    #   weasel
+stack-data==0.6.3
+    # via ipython
+sympy==1.12
+    # via torch
+tabula-py==2.9.0
+    # via sec-certs (./../pyproject.toml)
+tenacity==8.2.3
+    # via plotly
+thinc==8.2.1
+    # via spacy
+threadpoolctl==3.2.0
+    # via scikit-learn
+tifffile==2023.9.26
+    # via scikit-image
+tokenizers==0.15.0
+    # via transformers
+tomli==2.0.1
+    # via setuptools-scm
+toolz==0.12.0
+    # via
+    #   dask
+    #   datashader
+    #   partd
+torch==2.1.1
+    # via
+    #   sentence-transformers
+    #   torchvision
+torchvision==0.16.1
+    # via sentence-transformers
+tornado==6.3.3
+    # via
+    #   bokeh
+    #   ipykernel
+    #   jupyter-client
+tqdm==4.66.1
+    # via
+    #   datasets
+    #   evaluate
+    #   huggingface-hub
+    #   nltk
+    #   optuna
+    #   panel
+    #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
+    #   spacy
+    #   transformers
+    #   umap-learn
+traitlets==5.13.0
+    # via
+    #   comm
+    #   ipykernel
+    #   ipython
+    #   ipywidgets
+    #   jupyter-client
+    #   jupyter-core
+    #   matplotlib-inline
+transformers==4.35.2
+    # via sentence-transformers
+typer==0.9.0
+    # via
+    #   spacy
+    #   weasel
+typing-extensions==4.8.0
+    # via
+    #   alembic
+    #   cloudpathlib
+    #   huggingface-hub
+    #   panel
+    #   pydantic
+    #   pydantic-core
+    #   setuptools-scm
+    #   sqlalchemy
+    #   torch
+    #   typer
+tzdata==2023.3
+    # via pandas
+uc-micro-py==1.0.2
+    # via linkify-it-py
+umap-learn[plot]==0.5.5
+    # via sec-certs (./../pyproject.toml)
+urllib3==2.1.0
+    # via
+    #   requests
+    #   responses
+wasabi==1.1.2
+    # via
+    #   spacy
+    #   thinc
+    #   weasel
+wcwidth==0.2.12
+    # via prompt-toolkit
+weasel==0.3.4
+    # via spacy
+webencodings==0.5.1
+    # via
+    #   bleach
+    #   html5lib
+widgetsnbextension==4.0.9
+    # via ipywidgets
+wrapt==1.16.0
+    # via deprecated
+xarray==2023.11.0
+    # via datashader
+xxhash==3.4.1
+    # via
+    #   datasets
+    #   evaluate
+xyzservices==2023.10.1
+    # via
+    #   bokeh
+    #   panel
+yarl==1.9.3
+    # via aiohttp
+zipp==3.17.0
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index cb9d6bba..0b370634 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -1,98 +1,110 @@
+annotated-types==0.6.0
+    # via pydantic
 appnope==0.1.3
     # via
     #   ipykernel
     #   ipython
-asttokens==2.2.1
+asttokens==2.4.1
     # via stack-data
-attrs==22.1.0
-    # via jsonschema
-backcall==0.2.0
-    # via ipython
-beautifulsoup4==4.11.1
-    # via sec-certs (./../pyproject.toml)
-billiard==4.0.2
+attrs==23.1.0
+    # via
+    #   jsonschema
+    #   referencing
+beautifulsoup4==4.12.2
     # via sec-certs (./../pyproject.toml)
-blis==0.7.9
+blis==0.7.11
     # via thinc
-catalogue==2.0.8
+catalogue==2.0.10
     # via
     #   spacy
     #   srsly
     #   thinc
-certifi==2023.7.22
+certifi==2023.11.17
     # via requests
-charset-normalizer==2.1.1
+cffi==1.16.0
+    # via cryptography
+charset-normalizer==3.3.2
     # via requests
-click==8.1.3
+click==8.1.7
     # via
     #   sec-certs (./../pyproject.toml)
     #   typer
-comm==0.1.2
-    # via ipykernel
-confection==0.0.3
-    # via thinc
-contourpy==1.0.6
+cloudpathlib==0.16.0
+    # via weasel
+comm==0.2.0
+    # via
+    #   ipykernel
+    #   ipywidgets
+confection==0.1.3
+    # via
+    #   thinc
+    #   weasel
+contourpy==1.2.0
     # via matplotlib
-cycler==0.11.0
+cryptography==41.0.5
+    # via pypdf
+cycler==0.12.1
     # via matplotlib
-cymem==2.0.7
+cymem==2.0.8
     # via
     #   preshed
     #   spacy
     #   thinc
-debugpy==1.6.4
+debugpy==1.8.0
     # via ipykernel
 decorator==5.1.1
     # via ipython
-deprecation==2.1.0
+deprecated==1.2.14
     # via pikepdf
 distro==1.8.0
     # via tabula-py
-entrypoints==0.4
-    # via jupyter-client
-executing==1.2.0
+exceptiongroup==1.2.0
+    # via ipython
+executing==2.0.1
     # via stack-data
-fonttools==4.38.0
+fonttools==4.45.0
     # via matplotlib
 html5lib==1.1
     # via sec-certs (./../pyproject.toml)
 idna==3.4
     # via requests
-ipykernel==6.19.1
-    # via
-    #   ipywidgets
-    #   sec-certs (./../pyproject.toml)
-ipython==8.10.0
+ipykernel==6.27.0
+    # via sec-certs (./../pyproject.toml)
+ipython==8.17.2
     # via
     #   ipykernel
     #   ipywidgets
-ipywidgets==8.0.3
+ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
-jedi==0.18.2
+jedi==0.19.1
     # via ipython
 jinja2==3.1.2
     # via spacy
-joblib==1.2.0
+joblib==1.3.2
     # via scikit-learn
-jsonschema==4.17.3
+jsonschema==4.20.0
     # via sec-certs (./../pyproject.toml)
-jupyter-client==7.4.8
+jsonschema-specifications==2023.11.1
+    # via jsonschema
+jupyter-client==8.6.0
     # via ipykernel
-jupyter-core==5.1.0
-    # via jupyter-client
-jupyterlab-widgets==3.0.4
+jupyter-core==5.5.0
+    # via
+    #   ipykernel
+    #   jupyter-client
+jupyterlab-widgets==3.0.9
     # via ipywidgets
-kiwisolver==1.4.4
+kiwisolver==1.4.5
     # via matplotlib
 langcodes==3.3.0
     # via spacy
-lxml==4.9.1
+lxml==4.9.3
     # via
     #   pikepdf
     #   sec-certs (./../pyproject.toml)
-markupsafe==2.1.1
+markupsafe==2.1.3
     # via jinja2
-matplotlib==3.6.2
+matplotlib==3.8.2
     # via
     #   pysankeybeta
     #   seaborn
@@ -101,18 +113,16 @@ matplotlib-inline==0.1.6
     # via
     #   ipykernel
     #   ipython
-murmurhash==1.0.9
+murmurhash==1.0.10
     # via
     #   preshed
     #   spacy
     #   thinc
-nest-asyncio==1.5.6
-    # via
-    #   ipykernel
-    #   jupyter-client
-networkx==2.8.8
+nest-asyncio==1.5.8
+    # via ipykernel
+networkx==3.2.1
     # via sec-certs (./../pyproject.toml)
-numpy==1.23.5
+numpy==1.26.2
     # via
     #   blis
     #   contourpy
@@ -126,16 +136,17 @@ numpy==1.23.5
     #   spacy
     #   tabula-py
     #   thinc
-packaging==22.0
+packaging==23.2
     # via
-    #   deprecation
     #   ipykernel
     #   matplotlib
     #   pikepdf
     #   pytesseract
     #   setuptools-scm
     #   spacy
-pandas==1.5.2
+    #   thinc
+    #   weasel
+pandas==2.1.3
     # via
     #   pysankeybeta
     #   seaborn
@@ -143,17 +154,13 @@ pandas==1.5.2
     #   tabula-py
 parso==0.8.3
     # via jedi
-pathy==0.10.1
-    # via spacy
 pdftotext==2.2.2
     # via sec-certs (./../pyproject.toml)
 pexpect==4.8.0
     # via ipython
-pickleshare==0.7.5
-    # via ipython
-pikepdf==6.2.5
+pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.0.1
+pillow==10.1.0
     # via
     #   matplotlib
     #   pikepdf
@@ -161,15 +168,15 @@ pillow==10.0.1
     #   sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-platformdirs==2.6.0
+platformdirs==4.0.0
     # via jupyter-core
-preshed==3.0.8
+preshed==3.0.9
     # via
     #   spacy
     #   thinc
-prompt-toolkit==3.0.36
+prompt-toolkit==3.0.41
     # via ipython
-psutil==5.9.4
+psutil==5.9.6
     # via
     #   ipykernel
     #   sec-certs (./../pyproject.toml)
@@ -177,25 +184,29 @@ ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
     # via stack-data
-pycryptodome==3.16.0
-    # via pypdf
-pydantic==1.10.2
+pycparser==2.21
+    # via cffi
+pydantic==2.5.2
     # via
     #   confection
+    #   pydantic-settings
     #   sec-certs (./../pyproject.toml)
     #   spacy
     #   thinc
-pygments==2.15.0
+    #   weasel
+pydantic-core==2.14.5
+    # via pydantic
+pydantic-settings==2.1.0
+    # via sec-certs (./../pyproject.toml)
+pygments==2.17.2
     # via ipython
-pyparsing==3.0.9
+pyparsing==3.1.1
     # via matplotlib
-pypdf[crypto]==3.2.1
+pypdf[crypto]==3.17.1
     # via
     #   pypdf
     #   sec-certs (./../pyproject.toml)
-pyrsistent==0.19.2
-    # via jsonschema
-pysankeybeta==1.4.0
+pysankeybeta==1.4.1
     # via sec-certs (./../pyproject.toml)
 pytesseract==0.3.10
     # via sec-certs (./../pyproject.toml)
@@ -205,59 +216,73 @@ python-dateutil==2.8.2
     #   matplotlib
     #   pandas
     #   sec-certs (./../pyproject.toml)
-pytz==2022.6
+python-dotenv==1.0.0
+    # via pydantic-settings
+pytz==2023.3.post1
     # via pandas
-pyyaml==6.0
+pyyaml==6.0.1
     # via sec-certs (./../pyproject.toml)
-pyzmq==24.0.1
+pyzmq==25.1.1
     # via
     #   ipykernel
     #   jupyter-client
-rapidfuzz==2.13.3
+rapidfuzz==3.5.2
     # via sec-certs (./../pyproject.toml)
+referencing==0.31.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 requests==2.31.0
     # via
     #   sec-certs (./../pyproject.toml)
     #   spacy
-scikit-learn==1.2.0
+    #   weasel
+rpds-py==0.13.1
+    # via
+    #   jsonschema
+    #   referencing
+scikit-learn==1.3.2
     # via sec-certs (./../pyproject.toml)
-scipy==1.10.0
+scipy==1.11.4
     # via
     #   scikit-learn
     #   sec-certs (./../pyproject.toml)
-seaborn==0.12.1
+seaborn==0.13.0
     # via
     #   pysankeybeta
     #   sec-certs (./../pyproject.toml)
-setuptools-scm==7.0.5
+setuptools-scm==8.0.4
     # via sec-certs (./../pyproject.toml)
 six==1.16.0
     # via
     #   asttokens
     #   html5lib
     #   python-dateutil
-smart-open==6.2.0
-    # via pathy
-soupsieve==2.3.2.post1
+smart-open==6.4.0
+    # via
+    #   spacy
+    #   weasel
+soupsieve==2.5
     # via beautifulsoup4
-spacy==3.4.3
+spacy==3.7.2
     # via sec-certs (./../pyproject.toml)
-spacy-legacy==3.0.10
+spacy-legacy==3.0.12
     # via spacy
-spacy-loggers==1.0.4
+spacy-loggers==1.0.5
     # via spacy
-srsly==2.4.5
+srsly==2.4.8
     # via
     #   confection
     #   spacy
     #   thinc
-stack-data==0.6.2
+    #   weasel
+stack-data==0.6.3
     # via ipython
-tabula-py==2.6.0
+tabula-py==2.9.0
     # via sec-certs (./../pyproject.toml)
-thinc==8.1.5
+thinc==8.2.1
     # via spacy
-threadpoolctl==3.1.0
+threadpoolctl==3.2.0
     # via scikit-learn
 tomli==2.0.1
     # via setuptools-scm
@@ -265,11 +290,11 @@ tornado==6.3.3
     # via
     #   ipykernel
     #   jupyter-client
-tqdm==4.64.1
+tqdm==4.66.1
     # via
     #   sec-certs (./../pyproject.toml)
     #   spacy
-traitlets==5.6.0
+traitlets==5.13.0
     # via
     #   comm
     #   ipykernel
@@ -278,26 +303,36 @@ traitlets==5.6.0
     #   jupyter-client
     #   jupyter-core
     #   matplotlib-inline
-typer==0.7.0
+typer==0.9.0
     # via
-    #   pathy
     #   spacy
-typing-extensions==4.4.0
+    #   weasel
+typing-extensions==4.8.0
     # via
+    #   cloudpathlib
     #   pydantic
+    #   pydantic-core
     #   setuptools-scm
-urllib3==1.26.18
+    #   typer
+tzdata==2023.3
+    # via pandas
+urllib3==2.1.0
     # via requests
-wasabi==0.10.1
+wasabi==1.1.2
     # via
     #   spacy
     #   thinc
-wcwidth==0.2.5
+    #   weasel
+wcwidth==0.2.12
     # via prompt-toolkit
+weasel==0.3.4
+    # via spacy
 webencodings==0.5.1
     # via html5lib
-widgetsnbextension==4.0.4
+widgetsnbextension==4.0.9
     # via ipywidgets
+wrapt==1.16.0
+    # via deprecated
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 900cbaf0..6dde2c98 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -1,108 +1,118 @@
+annotated-types==0.6.0
+    # via pydantic
 appnope==0.1.3
     # via
     #   ipykernel
     #   ipython
-asttokens==2.2.1
+asttokens==2.4.1
     # via stack-data
-attrs==22.1.0
+attrs==23.1.0
     # via
     #   jsonschema
-    #   pytest
-backcall==0.2.0
-    # via ipython
-beautifulsoup4==4.11.1
-    # via sec-certs (./../pyproject.toml)
-billiard==4.0.2
+    #   referencing
+beautifulsoup4==4.12.2
     # via sec-certs (./../pyproject.toml)
-blis==0.7.9
+blis==0.7.11
     # via thinc
-catalogue==2.0.8
+catalogue==2.0.10
     # via
     #   spacy
     #   srsly
     #   thinc
-certifi==2023.7.22
+certifi==2023.11.17
     # via requests
-charset-normalizer==2.1.1
+cffi==1.16.0
+    # via cryptography
+charset-normalizer==3.3.2
     # via requests
-click==8.1.3
+click==8.1.7
     # via
     #   sec-certs (./../pyproject.toml)
     #   typer
-comm==0.1.2
-    # via ipykernel
-confection==0.0.3
-    # via thinc
-contourpy==1.0.6
+cloudpathlib==0.16.0
+    # via weasel
+comm==0.2.0
+    # via
+    #   ipykernel
+    #   ipywidgets
+confection==0.1.3
+    # via
+    #   thinc
+    #   weasel
+contourpy==1.2.0
     # via matplotlib
-coverage[toml]==6.5.0
+coverage[toml]==7.3.2
     # via
     #   pytest-cov
     #   sec-certs (./../pyproject.toml)
-cycler==0.11.0
+cryptography==41.0.5
+    # via pypdf
+cycler==0.12.1
     # via matplotlib
-cymem==2.0.7
+cymem==2.0.8
     # via
     #   preshed
     #   spacy
     #   thinc
-debugpy==1.6.4
+debugpy==1.8.0
     # via ipykernel
 decorator==5.1.1
     # via ipython
-deprecation==2.1.0
+deprecated==1.2.14
     # via pikepdf
 distro==1.8.0
     # via tabula-py
-entrypoints==0.4
-    # via jupyter-client
-exceptiongroup==1.1.3
-    # via pytest
-executing==1.2.0
+exceptiongroup==1.2.0
+    # via
+    #   ipython
+    #   pytest
+executing==2.0.1
     # via stack-data
-fonttools==4.38.0
+fonttools==4.45.0
     # via matplotlib
 html5lib==1.1
     # via sec-certs (./../pyproject.toml)
 idna==3.4
     # via requests
-iniconfig==1.1.1
+iniconfig==2.0.0
     # via pytest
-ipykernel==6.19.1
-    # via
-    #   ipywidgets
-    #   sec-certs (./../pyproject.toml)
-ipython==8.10.0
+ipykernel==6.27.0
+    # via sec-certs (./../pyproject.toml)
+ipython==8.17.2
     # via
     #   ipykernel
     #   ipywidgets
-ipywidgets==8.0.3
+ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
-jedi==0.18.2
+jedi==0.19.1
     # via ipython
 jinja2==3.1.2
     # via spacy
-joblib==1.2.0
+joblib==1.3.2
     # via scikit-learn
-jsonschema==4.17.3
+jsonschema==4.20.0
     # via sec-certs (./../pyproject.toml)
-jupyter-client==7.4.8
+jsonschema-specifications==2023.11.1
+    # via jsonschema
+jupyter-client==8.6.0
     # via ipykernel
-jupyter-core==5.1.0
-    # via jupyter-client
-jupyterlab-widgets==3.0.4
+jupyter-core==5.5.0
+    # via
+    #   ipykernel
+    #   jupyter-client
+jupyterlab-widgets==3.0.9
     # via ipywidgets
-kiwisolver==1.4.4
+kiwisolver==1.4.5
     # via matplotlib
 langcodes==3.3.0
     # via spacy
-lxml==4.9.1
+lxml==4.9.3
     # via
     #   pikepdf
     #   sec-certs (./../pyproject.toml)
-markupsafe==2.1.1
+markupsafe==2.1.3
     # via jinja2
-matplotlib==3.6.2
+matplotlib==3.8.2
     # via
     #   pysankeybeta
     #   seaborn
@@ -111,18 +121,16 @@ matplotlib-inline==0.1.6
     # via
     #   ipykernel
     #   ipython
-murmurhash==1.0.9
+murmurhash==1.0.10
     # via
     #   preshed
     #   spacy
     #   thinc
-nest-asyncio==1.5.6
-    # via
-    #   ipykernel
-    #   jupyter-client
-networkx==2.8.8
+nest-asyncio==1.5.8
+    # via ipykernel
+networkx==3.2.1
     # via sec-certs (./../pyproject.toml)
-numpy==1.23.5
+numpy==1.26.2
     # via
     #   blis
     #   contourpy
@@ -136,9 +144,8 @@ numpy==1.23.5
     #   spacy
     #   tabula-py
     #   thinc
-packaging==22.0
+packaging==23.2
     # via
-    #   deprecation
     #   ipykernel
     #   matplotlib
     #   pikepdf
@@ -146,7 +153,9 @@ packaging==22.0
     #   pytest
     #   setuptools-scm
     #   spacy
-pandas==1.5.2
+    #   thinc
+    #   weasel
+pandas==2.1.3
     # via
     #   pysankeybeta
     #   seaborn
@@ -154,17 +163,13 @@ pandas==1.5.2
     #   tabula-py
 parso==0.8.3
     # via jedi
-pathy==0.10.1
-    # via spacy
 pdftotext==2.2.2
     # via sec-certs (./../pyproject.toml)
 pexpect==4.8.0
     # via ipython
-pickleshare==0.7.5
-    # via ipython
-pikepdf==6.2.5
+pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.0.1
+pillow==10.1.0
     # via
     #   matplotlib
     #   pikepdf
@@ -172,17 +177,17 @@ pillow==10.0.1
     #   sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-platformdirs==2.6.0
+platformdirs==4.0.0
     # via jupyter-core
-pluggy==1.0.0
+pluggy==1.3.0
     # via pytest
-preshed==3.0.8
+preshed==3.0.9
     # via
     #   spacy
     #   thinc
-prompt-toolkit==3.0.36
+prompt-toolkit==3.0.41
     # via ipython
-psutil==5.9.4
+psutil==5.9.6
     # via
     #   ipykernel
     #   sec-certs (./../pyproject.toml)
@@ -190,33 +195,37 @@ ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
     # via stack-data
-pycryptodome==3.16.0
-    # via pypdf
-pydantic==1.10.2
+pycparser==2.21
+    # via cffi
+pydantic==2.5.2
     # via
     #   confection
+    #   pydantic-settings
     #   sec-certs (./../pyproject.toml)
     #   spacy
     #   thinc
-pygments==2.15.0
+    #   weasel
+pydantic-core==2.14.5
+    # via pydantic
+pydantic-settings==2.1.0
+    # via sec-certs (./../pyproject.toml)
+pygments==2.17.2
     # via ipython
-pyparsing==3.0.9
+pyparsing==3.1.1
     # via matplotlib
-pypdf[crypto]==3.2.1
+pypdf[crypto]==3.17.1
     # via
     #   pypdf
     #   sec-certs (./../pyproject.toml)
-pyrsistent==0.19.2
-    # via jsonschema
-pysankeybeta==1.4.0
+pysankeybeta==1.4.1
     # via sec-certs (./../pyproject.toml)
 pytesseract==0.3.10
     # via sec-certs (./../pyproject.toml)
-pytest==7.2.0
+pytest==7.4.3
     # via
     #   pytest-cov
     #   sec-certs (./../pyproject.toml)
-pytest-cov==4.0.0
+pytest-cov==4.1.0
     # via sec-certs (./../pyproject.toml)
 python-dateutil==2.8.2
     # via
@@ -224,59 +233,73 @@ python-dateutil==2.8.2
     #   matplotlib
     #   pandas
     #   sec-certs (./../pyproject.toml)
-pytz==2022.6
+python-dotenv==1.0.0
+    # via pydantic-settings
+pytz==2023.3.post1
     # via pandas
-pyyaml==6.0
+pyyaml==6.0.1
     # via sec-certs (./../pyproject.toml)
-pyzmq==24.0.1
+pyzmq==25.1.1
     # via
     #   ipykernel
     #   jupyter-client
-rapidfuzz==2.13.3
+rapidfuzz==3.5.2
     # via sec-certs (./../pyproject.toml)
+referencing==0.31.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 requests==2.31.0
     # via
     #   sec-certs (./../pyproject.toml)
     #   spacy
-scikit-learn==1.2.0
+    #   weasel
+rpds-py==0.13.1
+    # via
+    #   jsonschema
+    #   referencing
+scikit-learn==1.3.2
     # via sec-certs (./../pyproject.toml)
-scipy==1.10.0
+scipy==1.11.4
     # via
     #   scikit-learn
     #   sec-certs (./../pyproject.toml)
-seaborn==0.12.1
+seaborn==0.13.0
     # via
     #   pysankeybeta
     #   sec-certs (./../pyproject.toml)
-setuptools-scm==7.0.5
+setuptools-scm==8.0.4
     # via sec-certs (./../pyproject.toml)
 six==1.16.0
     # via
     #   asttokens
     #   html5lib
     #   python-dateutil
-smart-open==6.2.0
-    # via pathy
-soupsieve==2.3.2.post1
+smart-open==6.4.0
+    # via
+    #   spacy
+    #   weasel
+soupsieve==2.5
     # via beautifulsoup4
-spacy==3.4.3
+spacy==3.7.2
     # via sec-certs (./../pyproject.toml)
-spacy-legacy==3.0.10
+spacy-legacy==3.0.12
     # via spacy
-spacy-loggers==1.0.4
+spacy-loggers==1.0.5
     # via spacy
-srsly==2.4.5
+srsly==2.4.8
     # via
     #   confection
     #   spacy
     #   thinc
-stack-data==0.6.2
+    #   weasel
+stack-data==0.6.3
     # via ipython
-tabula-py==2.6.0
+tabula-py==2.9.0
     # via sec-certs (./../pyproject.toml)
-thinc==8.1.5
+thinc==8.2.1
     # via spacy
-threadpoolctl==3.1.0
+threadpoolctl==3.2.0
     # via scikit-learn
 tomli==2.0.1
     # via
@@ -287,11 +310,11 @@ tornado==6.3.3
     # via
     #   ipykernel
     #   jupyter-client
-tqdm==4.64.1
+tqdm==4.66.1
     # via
     #   sec-certs (./../pyproject.toml)
     #   spacy
-traitlets==5.6.0
+traitlets==5.13.0
     # via
     #   comm
     #   ipykernel
@@ -300,26 +323,36 @@ traitlets==5.6.0
     #   jupyter-client
     #   jupyter-core
     #   matplotlib-inline
-typer==0.7.0
+typer==0.9.0
     # via
-    #   pathy
     #   spacy
-typing-extensions==4.4.0
+    #   weasel
+typing-extensions==4.8.0
     # via
+    #   cloudpathlib
     #   pydantic
+    #   pydantic-core
     #   setuptools-scm
-urllib3==1.26.18
+    #   typer
+tzdata==2023.3
+    # via pandas
+urllib3==2.1.0
     # via requests
-wasabi==0.10.1
+wasabi==1.1.2
     # via
     #   spacy
     #   thinc
-wcwidth==0.2.5
+    #   weasel
+wcwidth==0.2.12
     # via prompt-toolkit
+weasel==0.3.4
+    # via spacy
 webencodings==0.5.1
     # via html5lib
-widgetsnbextension==4.0.4
+widgetsnbextension==4.0.9
     # via ipywidgets
+wrapt==1.16.0
+    # via deprecated
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/src/sec_certs/configuration.py b/src/sec_certs/configuration.py
index d7fb734a..3ebb22bd 100644
--- a/src/sec_certs/configuration.py
+++ b/src/sec_certs/configuration.py
@@ -5,7 +5,8 @@ from pathlib import Path
 from typing import Literal, Optional
 
 import yaml
-from pydantic import AnyHttpUrl, BaseSettings, Field
+from pydantic import AnyHttpUrl, Field
+from pydantic_settings import BaseSettings
 
 
 class Configuration(BaseSettings):
diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
index 4ea4cbd3..01649bb7 100644
--- a/src/sec_certs/constants.py
+++ b/src/sec_certs/constants.py
@@ -1,6 +1,11 @@
 import re
 from pathlib import Path
-from typing import Final
+from typing import Final, Literal
+
+RANDOM_STATE: Final[int] = 42
+REF_ANNOTATION_MODES = Literal["training", "evaluation", "production", "cross-validation"]
+REF_EMBEDDING_METHOD = Literal["tf_idf", "transformer"]
+
 
 DUMMY_NONEXISTING_PATH = Path("/this/is/dummy/nonexisting/path")
 
diff --git a/data/cert_id_eval/duplicate_ids.csv b/src/sec_certs/data/cert_ids/duplicate_ids.csv
index 28ad34fc..28ad34fc 100644
--- a/data/cert_id_eval/duplicate_ids.csv
+++ b/src/sec_certs/data/cert_ids/duplicate_ids.csv
diff --git a/data/cert_id_eval/missing_ids.csv b/src/sec_certs/data/cert_ids/missing_ids.csv
index cf4091ac..cf4091ac 100644
--- a/data/cert_id_eval/missing_ids.csv
+++ b/src/sec_certs/data/cert_ids/missing_ids.csv
diff --git a/data/cert_id_eval/random.csv b/src/sec_certs/data/cert_ids/random.csv
index 2fd2ef56..2fd2ef56 100644
--- a/data/cert_id_eval/random.csv
+++ b/src/sec_certs/data/cert_ids/random.csv
diff --git a/src/sec_certs/data/cert_ids/readme.md b/src/sec_certs/data/cert_ids/readme.md
new file mode 100644
index 00000000..75c38a48
--- /dev/null
+++ b/src/sec_certs/data/cert_ids/readme.md
@@ -0,0 +1,8 @@
+## Certificate ID evaluation
+
+This directory contains data on a manual evaluation of certificate ID assignment.
+
+- `missing_ids.csv` contains an evaluation of certificates to which the sec-certs tool was not able to
+find a certificate ID (to analyze why that happened and whether we could fix that).
+- `duplicate_ids.csv` contains an evaluation of certificates to which the sec-certs tool assigned a duplicate
+ID (to analyze why that happened and whether we could fix that). These files are used by the [cert_id_eval.ipynb](./../../notebooks/cc/cert_id_eval.ipynb) notebook which evaluates a dataset with respect to the manually labeled ground truth in them.
diff --git a/data/cert_id_eval/truth.csv b/src/sec_certs/data/cert_ids/truth.csv
index 80086fc7..80086fc7 100644
--- a/data/cert_id_eval/truth.csv
+++ b/src/sec_certs/data/cert_ids/truth.csv
diff --git a/data/label_studio_interface.txt b/src/sec_certs/data/cpes/label_studio_interface.txt
index 3b2a7e20..3b2a7e20 100644
--- a/data/label_studio_interface.txt
+++ b/src/sec_certs/data/cpes/label_studio_interface.txt
diff --git a/data/cpe_eval/manual_cpe_labels.json b/src/sec_certs/data/cpes/manual_cpe_labels.json
index bd5e1ba6..bd5e1ba6 100644
--- a/data/cpe_eval/manual_cpe_labels.json
+++ b/src/sec_certs/data/cpes/manual_cpe_labels.json
diff --git a/data/old_manual_cpe_labels/cc.json b/src/sec_certs/data/cpes/outdated/cc.json
index 8ad9aaf9..8ad9aaf9 100644
--- a/data/old_manual_cpe_labels/cc.json
+++ b/src/sec_certs/data/cpes/outdated/cc.json
diff --git a/data/old_manual_cpe_labels/fips.json b/src/sec_certs/data/cpes/outdated/fips.json
index 8683da09..8683da09 100644
--- a/data/old_manual_cpe_labels/fips.json
+++ b/src/sec_certs/data/cpes/outdated/fips.json
diff --git a/data/cpe_eval/random.csv b/src/sec_certs/data/cpes/random.csv
index efc62f45..efc62f45 100644
--- a/data/cpe_eval/random.csv
+++ b/src/sec_certs/data/cpes/random.csv
diff --git a/src/sec_certs/data/cpes/readme.md b/src/sec_certs/data/cpes/readme.md
new file mode 100644
index 00000000..b96cd4f3
--- /dev/null
+++ b/src/sec_certs/data/cpes/readme.md
@@ -0,0 +1,8 @@
+## CPEs
+
+- This directory contains digests of 100 randomly sampled certificates, together with predicted and ground-truth labels.
+The file `random.csv` summarizes the data above, while `manual_cpe_labels.json` is a JSON-min export from label studio instance.
+- These files can be utilized from [cpe_eval notebook](../../notebooks/cc/cpe_eval.ipynb) to see the performance of the classifier.
+-Folder `./outdated` contains some old incomplete labeling that was obtained highly unoptimized classifier.
+- [label_studio_interface.txt](label_studio_interface.txt) contains XML-like specification of the labeling interface
+for CPE matching. As such, it was used in the Label studio tool.
diff --git a/data/validation_test_split/cc/test.json b/src/sec_certs/data/cpes/validation_test_split/cc/test.json
index bee8d8f5..bee8d8f5 100644
--- a/data/validation_test_split/cc/test.json
+++ b/src/sec_certs/data/cpes/validation_test_split/cc/test.json
diff --git a/data/validation_test_split/cc/validation.json b/src/sec_certs/data/cpes/validation_test_split/cc/validation.json
index 0ecc3a0e..0ecc3a0e 100644
--- a/data/validation_test_split/cc/validation.json
+++ b/src/sec_certs/data/cpes/validation_test_split/cc/validation.json
diff --git a/data/validation_test_split/fips/test.json b/src/sec_certs/data/cpes/validation_test_split/fips/test.json
index 3768ff72..3768ff72 100644
--- a/data/validation_test_split/fips/test.json
+++ b/src/sec_certs/data/cpes/validation_test_split/fips/test.json
diff --git a/data/validation_test_split/fips/validation.json b/src/sec_certs/data/cpes/validation_test_split/fips/validation.json
index c06ae9df..c06ae9df 100644
--- a/data/validation_test_split/fips/validation.json
+++ b/src/sec_certs/data/cpes/validation_test_split/fips/validation.json
diff --git a/data/information_retrieval_split/metadata.csv b/src/sec_certs/data/information_retrieval/split/metadata.csv
index 2c5898d7..2c5898d7 100644
--- a/data/information_retrieval_split/metadata.csv
+++ b/src/sec_certs/data/information_retrieval/split/metadata.csv
diff --git a/data/information_retrieval_split/sampled_images.json b/src/sec_certs/data/information_retrieval/split/sampled_images.json
index e75a742e..e75a742e 100644
--- a/data/information_retrieval_split/sampled_images.json
+++ b/src/sec_certs/data/information_retrieval/split/sampled_images.json
diff --git a/data/information_retrieval_split/test.json b/src/sec_certs/data/information_retrieval/split/test.json
index be91674f..be91674f 100644
--- a/data/information_retrieval_split/test.json
+++ b/src/sec_certs/data/information_retrieval/split/test.json
diff --git a/data/information_retrieval_split/train.json b/src/sec_certs/data/information_retrieval/split/train.json
index 16d8e45c..16d8e45c 100644
--- a/data/information_retrieval_split/train.json
+++ b/src/sec_certs/data/information_retrieval/split/train.json
diff --git a/data/information_retrieval_split/valid.json b/src/sec_certs/data/information_retrieval/split/valid.json
index fc91875d..fc91875d 100644
--- a/data/information_retrieval_split/valid.json
+++ b/src/sec_certs/data/information_retrieval/split/valid.json
diff --git a/src/sec_certs/data/readme.md b/src/sec_certs/data/readme.md
new file mode 100644
index 00000000..a4556b70
--- /dev/null
+++ b/src/sec_certs/data/readme.md
@@ -0,0 +1,9 @@
+# Data
+
+This folder contains various metadata related to our research. Brief description is provided here. More detailed description is given in the respective folders.
+
+- [cert_ids](cert_ids) contains data related to evaluating `Certificate->certificate_id` matching
+- [cpes](cpes) contains data related to evaluation of `Certificate->list of CPEs` matching
+- [reference_annotations](reference_annotations) contains data and methodology related to annotating the references with their meaning.
+- [sar_correlations](sar_correlations) contains data of correlations between SARs and number/severity of CVEs that affect the certificates.
+- [information_retrieval](information_retrieval) contains data related to information retrieval experiments from certification artifacts.
diff --git a/src/sec_certs/data/reference_annotations/adam/test.csv b/src/sec_certs/data/reference_annotations/adam/test.csv
new file mode 100644
index 00000000..e9702c4f
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/adam/test.csv
@@ -0,0 +1,201 @@
+dgst,canonical_reference_keyword,label,comment
+15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,re-evaluation,None
+15b121492722bffb,BSI-DSZ-CC-0754-2012,re-evaluation,None
+16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,re-evaluation,None
+16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None
+16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None
+173097995b7a7f12,ANSSI-CC-2012/49,component_shared,None
+173097995b7a7f12,ANSSI-CC-2013/28,evaluation_reused,None
+173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None
+173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None
+177dee4c1051e612,BSI-DSZ-CC-0854-2013,re-evaluation,None
+17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None
+183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None
+183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None
+18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,component_used,None
+18b7e5bdf459ca13,NSCIB-CC-07-09219,component_used,None
+195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None
+195366bfba7213a5,BSI-DSZ-CC-0322-2005,component_used,None
+195366bfba7213a5,BSI-DSZ-CC-0399-2007,re-evaluation,None
+1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None
+19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None
+1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,re-evaluation,None
+1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None
+1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None
+1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None
+1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None
+1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,re-evaluation,None
+1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,evaluation_reused,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,re-evaluation,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None
+1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,re-evaluation,None
+2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,re-evaluation,None
+21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None
+21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None
+22ede463dbf1a105,ANSSI-CC-2015/36,evaluation_reused,None
+22ede463dbf1a105,ANSSI-CC-2015/80,evaluation_reused,None
+22ede463dbf1a105,ANSSI-CC-2016/43,evaluation_reused,None
+22ede463dbf1a105,ANSSI-CC-2016/44,evaluation_reused,None
+23a85a7f9c07412f,ANSSI-CC-2014/46,evaluation_reused,None
+23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,None
+24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None
+24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None
+259df98e7476a843,BSI-DSZ-CC-0666-2012,re-evaluation,None
+259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None
+25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None
+25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None
+26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,None
+26c7b173668167d8,BSI-DSZ-CC-0813-2012,component_used,None
+2760aeedce0b79db,ANSSI-CC-2011/64,evaluation_reused,None
+2760aeedce0b79db,BSI-DSZ-CC-0555-2009,component_used,None
+27ccec0740bb7915,ANSSI-CC-2010/02,component_used,None
+27ccec0740bb7915,ANSSI-CC-2010/20,evaluation_reused,None
+27ccec0740bb7915,ANSSI-CC-2010/35,evaluation_reused,None
+296618f10b019d9c,BSI-DSZ-CC-0417-2008,evaluation_reused,None
+29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None
+29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None
+29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,re-evaluation,None
+2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None
+2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None
+2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,re-evaluation,None
+2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None
+2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None
+2bd9661657e78278,ANSSI-CC-2013/13,component_used,None
+2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,re-evaluation,None
+2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,re-evaluation,None
+2d046178aa118fff,BSI-DSZ-CC-0293-2005,re-evaluation,None
+2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None
+2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None
+2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None
+2f9feaf4121720da,BSI-DSZ-CC-0547-2009,re-evaluation,None
+30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None
+3183669bf78db43b,ANSSI-CC-2015/45,component_used,None
+3183669bf78db43b,ANSSI-CC-2016/64,component_used,None
+3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,component_used,None
+31fd07069aebf013,BSI-DSZ-CC-0466-2008,evaluation_reused,None
+328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None
+328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None
+3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,re-evaluation,None
+3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None
+337ece90615ed69d,ANSSI-CC-2010/02,component_used,None
+3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None
+33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None
+33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,re-evaluation,None
+34506b8fab6dd7c3,ANSSI-CC-2014/22,evaluation_reused,None
+34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None
+34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None
+3519c14e4114d93d,BSI-DSZ-CC-0782-2012,component_used,None
+35b9be2203d2b9e6,ANSSI-CC-2012/70,component_used,None
+35efe13fa9e93a68,CRP178,component_used,None
+35efe13fa9e93a68,CRP182,component_used,None
+36641ce7192b5f92,BSI-DSZ-CC-0977-2017,component_used,None
+054bd8196e015197,ANSSI-CC-2011/07,None,unclear
+122245dd15683e26,BSI-DSZ-CC-0269-2006,None,unclear
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,component_used,very difficult
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,component_used,None
+29cc495628e76ffa,BSI-DSZ-CC-0886-2013,irrelevant,None
+2a07b85f61ac08b0,ANSSI-CC-2019/02,component_used,None
+30c6777901ae346c,BSI-DSZ-CC-0438-2007,component_used,None
+35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,irrelevant,None
+36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,component_used,None
+373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,component_used,None
+3746512864b941d4,ANSSI-CC-2016/15,evaluation_reused,None
+37b94c08f8e25249,BSI-DSZ-CC-0404-2007,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0410-2007,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0555-2009,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0633-2010,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0674-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0675-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0709-2010,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0710-2010,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0730-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0797-2012,irrelevant,None
+37b94c08f8e25249,BSI-DSZ-CC-0799-2012,irrelevant,None
+37c8d23c44b95833,ANSSI-CC-2017/16,evaluation_reused,None
+37d28734245a3bb1,BSI-DSZ-CC-0257-2004,component_used,None
+37e0d3cb098458d0,ANSSI-CC-2014/50,None,unclear
+37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,component_used,None
+37fe6036c2ac932b,ANSSI-CC-2014/06,component_used,None
+37fe6036c2ac932b,ANSSI-CC-2014/07,component_shared,None
+37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,component_used,None
+3817c0aca007989e,ANSSI-CC-2011/10,component_used,None
+3817c0aca007989e,ANSSI-CC-2011/12,component_shared,None
+3817c0aca007989e,BSI-DSZ-CC-0523-2008,component_used,None
+3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None
+384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,None,unclear
+384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None
+3976c9e492193315,BSI-DSZ-CC-1113-2021,re-evaluation,None
+39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None
+3a3b0a9113835307,BSI-DSZ-CC-0402-2008,re-evaluation,None
+3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None
+3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None
+3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None
+3a6a5f536bfbd4d4,ANSSI-CC-2009/59,previous_version,None
+3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,component_used,None
+3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,evaluation_reused,None
+3d149fe7c08bfc58,CRP256,previous_version,None
+3d756c83419bba28,ANSSI-CC-2011/07,component_used,None
+3d756c83419bba28,ANSSI-CC-2011/77,previous_version,None
+3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,component_used,None
+3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,component_used,None
+3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,component_used,None
+3e08a27e9d9c9b1e,NSCIB-CC-16-99111,irrelevant,self reference
+3e182053b03f1faf,CCEVS-VR-06-0044,irrelevant,self reference
+3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,component_used,None
+3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None
+3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None
+3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None
+3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,re-evaluation,None
+3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,re-evaluation,None
+3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None
+3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None
+3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,re-evaluation,None
+3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None
+3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None
+4057f5dbe4ffa4cd,ANSSI-CC-2017/57,evaluation_reused,None
+40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,re-evaluation,None
+40fa0a4cb5193977,ANSSI-CC-2018/27,evaluation_reused,None
+40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None
+40fc6ad0aed92913,ANSSI-CC-2017/54,evaluation_reused,None
+40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,component_used,None
+41bd2924e9afced5,ANSSI-CC-2010/49,component_used,None
+41bd2924e9afced5,ANSSI-CC-2010/50,evaluation_reused,None
+420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None
+424f6d496746258b,ANSSI-CC-2019/12,component_used,None
+424f6d496746258b,BSI-DSZ-CC-0995-2018,re-evaluation,None
+4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None
+42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None
+42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,None
+42f53e51476f1a3c,ANSSI-CC-2011/07,previous_version,None
+438b59086f6ebd64,BSI-DSZ-CC-1107-2020,component_used,None
+43a4ca62d0c0b0da,ANSSI-CC-2017/63,evaluation_reused,None
+43a4ca62d0c0b0da,ANSSI-CC-2017/64,component_used,None
+43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,component_used,None
+43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,component_used,None
+43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,component_used,None
+449c74a92ebb61a4,ANSSI-CC-2017/24,component_used,None
+44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,component_used,None
+45098872448f5816,NSCIB-CC-0441513-CR,component_used,None
+46459cfb9c1045d8,ANSSI-CC-2018/27,component_used,None
+46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,component_used,None
+4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,previous_version,None
+47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None
+4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None
+4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,re-evaluation,None
+4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None
+4893140ae5daadaf,CRP278,component_used,None
+48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,re-evaluation,None
+48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None
+48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None
+491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None
+491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,re-evaluation,None
+49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None
+49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None
+49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None
+4abc83fee59586b2,ANSSI-CC-2020/07,component_used,None
+4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,component_used,None
+4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None
+4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None
+4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None
+4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,re-evaluation,None
diff --git a/src/sec_certs/data/reference_annotations/adam/train.csv b/src/sec_certs/data/reference_annotations/adam/train.csv
new file mode 100644
index 00000000..1f74ca84
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/adam/train.csv
@@ -0,0 +1,101 @@
+dgst,canonical_reference_keyword,label,comment
+e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified M7892 G12 component
+0bf7a19b22163465,ANSSI-CC-2016/44,evaluation_reused,unclear whether component used
+ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None
+8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None
+680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None
+3515801dee00995f,BSI-DSZ-CC-0447-2008,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0447-2008
+6a999675c9422dfb,BSI-DSZ-CC-0353-2006,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0353-2006
+12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used
+d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert
+921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned
+00efeb17bcaafce6,ANSSI-CC-2009/34,component_used,None
+c175537338906951,ANSSI-CC-2016/70,component_shared,None
+fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None
+ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None
+f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0863-2013
+cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None
+dba4b5a166f9456a,ANSSI-CC-2014/76,evaluation_reused,None
+aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None
+b42009eb34746731,ANSSI-CC-2021/50,previous_version,maybe previous_version
+463ecd64b7506048,ANSSI-CC-2018/52,component_used,None
+c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,re-evaluation,None
+c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None
+0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None
+2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None
+e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,unclear
+ee1c6dd97918d74a,ANSSI-CC-2012/71,evaluation_reused,None
+e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,component_used,None
+845bb039719ac5d8,ANSSI-CC-2010/40,previous_version,None
+af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None
+d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None
+54273dd266fce692,ANSSI-CC-2015/15,component_used,None
+4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None
+d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,re-evaluation,None
+a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None
+f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None
+8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None
+b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,unclear
+371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None
+f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014
+e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,re-evaluation,None
+4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None
+a867d281d34d34b7,BSI-DSZ-CC-0976-2015,re-evaluation,None
+16bdbde359584f99,ANSSI-CC-2017/61,component_used,None
+34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None
+a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None
+0bf7a19b22163465,ANSSI-CC-2015/80,evaluation_reused,None
+1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,component_used,None
+13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,component_shared,"BAC, EAC thingy"
+f869956d14ea1694,ANSSI-CC-2012/68,component_used,None
+037577fc2019fcfa,ANSSI-CC-2021/49,previous_version,None
+17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,component_used,None
+5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,component_used,None
+1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,component_used,None
+28e70f46ba4394dd,ANSSI-CC-2012/08,component_used,None
+61832cb4291c343f,BSI-DSZ-CC-0955-2016,component_used,None
+42799014c183e1b3,BSI-DSZ-CC-0312-2005,component_used,None
+627bfd69c2a831cf,ANSSI-CC-2014/37,evaluation_reused,None
+aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017
+05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None
+9529eb793550093c,ANSSI-CC-2018/40,previous_version,None
+7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,re-evaluation,None
+ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,re-evaluation,None
+ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,re-evaluation,None
+ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None
+30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None
+6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None
+06e505abb8dad1b8,ANSSI-CC-2017/54,component_used,None
+317713948b4473b3,BSI-DSZ-CC-0798-2012,component_shared,None
+055985699aee1e09,ANSSI-CC-2010/02,component_used,None
+b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None
+691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None
+a07548a64fac5794,BSI-DSZ-CC-0350-2007,re-evaluation,None
+19c60923e31777e7,ANSSI-CC-2017/49,component_used,None
+f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None
+c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None
+42799014c183e1b3,BSI-DSZ-CC-0429-2007,re-evaluation,None
+3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,re-evaluation,None
+5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear
+4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None
+c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None
+ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None
+30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,component_used,None
+c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None
+ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None
+bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”"
+97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None
+744a7a202d909323,ANSSI-CC-2017/24,component_used,None
+df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None
+6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None
+1b8705a0486b3be1,NSCIB-CC-0209053-CR,previous_version,None
+a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,re-evaluation,None
+cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None
+27d11629261d8806,ANSSI-CC-2014/59,component_used,None
+a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,None
+d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,re-evaluation,None
+c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery"
+84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None
+36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None
+01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,component_used,None
+0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,component_used,None
diff --git a/src/sec_certs/data/reference_annotations/adam/valid.csv b/src/sec_certs/data/reference_annotations/adam/valid.csv
new file mode 100644
index 00000000..26954167
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/adam/valid.csv
@@ -0,0 +1,108 @@
+dgst,canonical_reference_keyword,label,comment
+a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None
+657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,re-evaluation,None
+0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None
+983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,re-evaluation,None
+8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,re-evaluation,None
+774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,None
+c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,evaluation_reused,None
+a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None
+4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,component_used,None
+cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None
+24de96ea505af909,ANSSI-CC-2021/29,component_used,None
+2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None
+b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,re-evaluation,None
+d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None
+4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,None
+22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None
+2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,re-evaluation,None
+6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None
+120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None
+994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None
+16abf8ee0697e64f,ANSSI-CC-2014/61,component_shared,None
+f05de4db911ec8af,CRP243,previous_version,None
+38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None
+fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None
+fc875e0381a28435,BSI-DSZ-CC-0432-2007,re-evaluation,None
+c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None
+2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_used,None
+2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None
+833ff87d69cff5db,383-4-184,None,unclear
+c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None
+647d17d44745a532,BSI-DSZ-CC-0904-2015,re-evaluation,None
+4489bfc781a82281,BSI-DSZ-CC-0817-2013,re-evaluation,None
+94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None
+37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None
+5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None
+5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,re-evaluation,None
+d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None
+0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,unclear
+84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None
+1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,re-evaluation,None
+d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None
+38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,re-evaluation,None
+e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None
+db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None
+4b3f577896c80f8a,ANSSI-CC-2016/44,previous_version,None
+5f3c9c6bf76ba72d,ANSSI-CC-2018/51,component_used,None
+22d6578b30f5a227,BSI-DSZ-CC-1107-2020,component_used,None
+a0c38b4389ad7cf7,ANSSI-CC-2018/12,component_used,None
+19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,component_used,None
+f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,unclear
+2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,component_used,None
+3357c28c91cc2ed1,ANSSI-CC-2016/44,evaluation_reused,None
+f6317f134f532696,ANSSI-CC-2017/24,component_shared,None
+234da54efbcf734b,BSI-DSZ-CC-0917-2014,component_used,None
+1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None
+2532fe8c7214c485,BSI-DSZ-CC-0633-2010,evaluation_reused,None
+b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None
+35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,re-evaluation,None
+edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,component_shared,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006.
+Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of
+technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the
+TOE was re-evaluated because of changes on top layers for modified module
+packaging requirements. Additional package types (MOB4 and complete
+passport inlay) and the relevant production sites were included in the evaluation
+process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the
+TOE itself. The Security Target was updated. "
+3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_shared,None
+0fe8df0e85116b61,ANSSI-CC-2010/33,component_shared,None
+bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,re-evaluation,None
+b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,None
+7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,re-evaluation,None
+81273108dd167b98,BSI-DSZ-CC-0523-2008,re-evaluation,None
+63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None
+a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None
+4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None
+f4d510a39278e687,ANSSI-CC-2018/32,component_used,None
+c23bb542201aa511,ANSSI-CC-2014/14,component_shared,None
+625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None
+cae968d4926648bd,BSI-DSZ-CC-0754-2012,re-evaluation,None
+000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None
+66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None
+a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None
+031667f4e242da61,CCEVS-VR-07-0054,component_used,None
+459539da9315ccac,ANSSI-CC-2021/30,previous_version,None
+69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None
+a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None
+50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None
+5cf51622fafaf868,BSI-DSZ-CC-1005-2016,re-evaluation,None
+1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None
+0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None
+8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None
+6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None
+4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None
+e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None
+03fad65e5af65088,BSI-DSZ-CC-0696-2011,re-evaluation,None
+720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None
+9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None
+d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None
+73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None
+2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None
+f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,component_used,None
+f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,component_used,None
+2f43fe24bd8e91e1,ANSSI-CC-2020/65,component_used,None
+d96438a7165e5d82,BSI-DSZ-CC-0411-2007,component_used,None
+7623950728f52a71,ANSSI-CC-2016/06,component_used,None
+ba69597434946260,BSI-DSZ-CC-0293-2005,component_used,None
+50aef770cb4f028f,BSI-DSZ-CC-1059-2018,component_used,None
diff --git a/src/sec_certs/data/reference_annotations/conflicts/test.csv b/src/sec_certs/data/reference_annotations/conflicts/test.csv
new file mode 100644
index 00000000..20701c17
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/conflicts/test.csv
@@ -0,0 +1,47 @@
+dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label
+173097995b7a7f12,ANSSI-CC-2012/49,component_shared,evaluation_reused,component_shared
+173097995b7a7f12,ANSSI-CC-2013/28,evaluation_reused,component_used,component_used
+18b7e5bdf459ca13,NSCIB-CC-07-09219,component_used,previous_version,previous_version
+195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None,component_used
+195366bfba7213a5,BSI-DSZ-CC-0322-2005,component_used,None,previous_version
+195366bfba7213a5,BSI-DSZ-CC-0399-2007,re-evaluation,None,re-evaluation
+1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None,component_used
+1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,evaluation_reused,component_shared,component_shared
+22ede463dbf1a105,ANSSI-CC-2015/36,evaluation_reused,component_used,component_used
+22ede463dbf1a105,ANSSI-CC-2015/80,evaluation_reused,irrelevant,irrelevant
+26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,component_used,component_used
+27ccec0740bb7915,ANSSI-CC-2010/35,evaluation_reused,component_shared,component_shared
+296618f10b019d9c,BSI-DSZ-CC-0417-2008,evaluation_reused,component_used,component_used
+2f9feaf4121720da,BSI-DSZ-CC-0547-2009,re-evaluation,evaluation_reused,re-evaluation
+31fd07069aebf013,BSI-DSZ-CC-0466-2008,evaluation_reused,component_used,previous_version
+34506b8fab6dd7c3,ANSSI-CC-2014/22,evaluation_reused,previous_version,previous_version
+054bd8196e015197,ANSSI-CC-2011/07,None,evaluation_reused,evaluation_reused
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,component_used,None,component_used
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,component_used,evaluation_reused,component_used
+29cc495628e76ffa,BSI-DSZ-CC-0886-2013,irrelevant,component_shared,component_shared
+35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,irrelevant,component_shared,component_shared
+3746512864b941d4,ANSSI-CC-2016/15,evaluation_reused,previous_version,evaluation_reused
+37b94c08f8e25249,BSI-DSZ-CC-0404-2007,component_used,None,component_used
+37b94c08f8e25249,BSI-DSZ-CC-0410-2007,component_used,None,component_used
+37b94c08f8e25249,BSI-DSZ-CC-0555-2009,component_used,None,component_used
+37b94c08f8e25249,BSI-DSZ-CC-0633-2010,component_used,None,component_used
+37b94c08f8e25249,BSI-DSZ-CC-0709-2010,component_used,None,component_used
+37b94c08f8e25249,BSI-DSZ-CC-0710-2010,component_used,None,component_used
+37b94c08f8e25249,BSI-DSZ-CC-0797-2012,irrelevant,component_shared,component_shared
+37b94c08f8e25249,BSI-DSZ-CC-0799-2012,irrelevant,None,component_shared
+37c8d23c44b95833,ANSSI-CC-2017/16,evaluation_reused,previous_version,previous_version
+37d28734245a3bb1,BSI-DSZ-CC-0257-2004,component_used,previous_version,component_used
+37e0d3cb098458d0,ANSSI-CC-2014/50,None,component_used,component_used
+37fe6036c2ac932b,ANSSI-CC-2014/07,component_shared,previous_version,previous_version
+3817c0aca007989e,ANSSI-CC-2011/12,component_shared,previous_version,component_shared
+384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,None,component_used,component_used
+3e08a27e9d9c9b1e,NSCIB-CC-16-99111,irrelevant,previous_version,irrelevant
+4057f5dbe4ffa4cd,ANSSI-CC-2017/57,evaluation_reused,previous_version,previous_version
+40fa0a4cb5193977,ANSSI-CC-2018/27,evaluation_reused,previous_version,component_shared
+40fc6ad0aed92913,ANSSI-CC-2017/54,evaluation_reused,component_used,previous_version
+41bd2924e9afced5,ANSSI-CC-2010/49,component_used,previous_version,previous_version
+41bd2924e9afced5,ANSSI-CC-2010/50,evaluation_reused,previous_version,previous_version
+42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,previous_version,evaluation_reused
+43a4ca62d0c0b0da,ANSSI-CC-2017/64,component_used,irrelevant,component_used
+4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,previous_version,irrelevant,irrelevant
+4893140ae5daadaf,CRP278,component_used,component_shared,component_used
diff --git a/src/sec_certs/data/reference_annotations/conflicts/train.csv b/src/sec_certs/data/reference_annotations/conflicts/train.csv
new file mode 100644
index 00000000..18ed3252
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/conflicts/train.csv
@@ -0,0 +1,10 @@
+dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label
+037577fc2019fcfa,ANSSI-CC-2021/49,previous_version,evaluation_reused,previous_version
+1b8705a0486b3be1,NSCIB-CC-0209053-CR,previous_version,evaluation_reused,previous_version
+9529eb793550093c,ANSSI-CC-2018/40,previous_version,evaluation_reused,previous_version
+a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,evaluation_reused,component_used
+b42009eb34746731,ANSSI-CC-2021/50,previous_version,evaluation_reused,previous_version
+bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,None,evaluation_reused
+c175537338906951,ANSSI-CC-2016/70,component_shared,component_used,component_shared
+c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,re-evaluation,evaluation_reused,evaluation_reused
+dba4b5a166f9456a,ANSSI-CC-2014/76,evaluation_reused,previous_version,re-evaluation
diff --git a/src/sec_certs/data/reference_annotations/conflicts/valid.csv b/src/sec_certs/data/reference_annotations/conflicts/valid.csv
new file mode 100644
index 00000000..ba016b70
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/conflicts/valid.csv
@@ -0,0 +1,17 @@
+dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label
+0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,None,component_used
+0fe8df0e85116b61,ANSSI-CC-2010/33,component_shared,evaluation_reused,previous_version
+16abf8ee0697e64f,ANSSI-CC-2014/61,component_shared,evaluation_reused,component_shared
+2532fe8c7214c485,BSI-DSZ-CC-0633-2010,evaluation_reused,component_used,component_used
+3357c28c91cc2ed1,ANSSI-CC-2016/44,evaluation_reused,component_used,evaluation_reused
+3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_shared,component_used,component_used
+4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,component_used,evaluation_reused
+4b3f577896c80f8a,ANSSI-CC-2016/44,previous_version,component_used,evaluation_reused
+774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,component_used,evaluation_reused
+7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,re-evaluation,previous_version,re-evaluation
+b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,evaluation_reused,previous_version
+c23bb542201aa511,ANSSI-CC-2014/14,component_shared,component_used,previous_version
+c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,evaluation_reused,component_shared,None
+edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,component_shared,evaluation_reused,evaluation_reused
+f05de4db911ec8af,CRP243,previous_version,component_used,previous_version
+f6317f134f532696,ANSSI-CC-2017/24,component_shared,evaluation_reused,previous_version
diff --git a/src/sec_certs/data/reference_annotations/final/test.csv b/src/sec_certs/data/reference_annotations/final/test.csv
new file mode 100644
index 00000000..28695033
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/final/test.csv
@@ -0,0 +1,201 @@
+dgst,canonical_reference_keyword,label,comment
+15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,re-evaluation,None
+15b121492722bffb,BSI-DSZ-CC-0754-2012,re-evaluation,None
+16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,re-evaluation,None
+16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None
+16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None
+173097995b7a7f12,ANSSI-CC-2012/49,component_shared,None
+173097995b7a7f12,ANSSI-CC-2013/28,component_used,None
+173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None
+173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None
+177dee4c1051e612,BSI-DSZ-CC-0854-2013,re-evaluation,None
+17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None
+183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None
+183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None
+18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,component_used,None
+18b7e5bdf459ca13,NSCIB-CC-07-09219,previous_version,None
+195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None
+195366bfba7213a5,BSI-DSZ-CC-0322-2005,previous_version,None
+195366bfba7213a5,BSI-DSZ-CC-0399-2007,re-evaluation,None
+1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None
+19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None
+1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,re-evaluation,None
+1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None
+1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None
+1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None
+1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None
+1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,re-evaluation,None
+1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,component_shared,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,re-evaluation,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None
+1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,re-evaluation,None
+2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,re-evaluation,None
+21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None
+21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None
+22ede463dbf1a105,ANSSI-CC-2015/36,component_used,None
+22ede463dbf1a105,ANSSI-CC-2015/80,irrelevant,None
+22ede463dbf1a105,ANSSI-CC-2016/43,evaluation_reused,None
+22ede463dbf1a105,ANSSI-CC-2016/44,evaluation_reused,None
+23a85a7f9c07412f,ANSSI-CC-2014/46,evaluation_reused,None
+23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,None
+24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None
+24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None
+259df98e7476a843,BSI-DSZ-CC-0666-2012,re-evaluation,None
+259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None
+25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None
+25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None
+26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,component_used,None
+26c7b173668167d8,BSI-DSZ-CC-0813-2012,component_used,None
+2760aeedce0b79db,ANSSI-CC-2011/64,evaluation_reused,None
+2760aeedce0b79db,BSI-DSZ-CC-0555-2009,component_used,None
+27ccec0740bb7915,ANSSI-CC-2010/02,component_used,None
+27ccec0740bb7915,ANSSI-CC-2010/20,evaluation_reused,None
+27ccec0740bb7915,ANSSI-CC-2010/35,component_shared,None
+296618f10b019d9c,BSI-DSZ-CC-0417-2008,component_used,None
+29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None
+29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None
+29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,re-evaluation,None
+2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None
+2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None
+2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,re-evaluation,None
+2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None
+2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None
+2bd9661657e78278,ANSSI-CC-2013/13,component_used,None
+2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,re-evaluation,None
+2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,re-evaluation,None
+2d046178aa118fff,BSI-DSZ-CC-0293-2005,re-evaluation,None
+2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None
+2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None
+2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None
+2f9feaf4121720da,BSI-DSZ-CC-0547-2009,re-evaluation,None
+30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None
+3183669bf78db43b,ANSSI-CC-2015/45,component_used,None
+3183669bf78db43b,ANSSI-CC-2016/64,component_used,None
+3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,component_used,None
+31fd07069aebf013,BSI-DSZ-CC-0466-2008,previous_version,None
+328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None
+328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None
+3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,re-evaluation,None
+3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None
+337ece90615ed69d,ANSSI-CC-2010/02,component_used,None
+3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None
+33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None
+33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,re-evaluation,None
+34506b8fab6dd7c3,ANSSI-CC-2014/22,previous_version,None
+34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None
+34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None
+3519c14e4114d93d,BSI-DSZ-CC-0782-2012,component_used,None
+35b9be2203d2b9e6,ANSSI-CC-2012/70,component_used,None
+35efe13fa9e93a68,CRP178,component_used,None
+35efe13fa9e93a68,CRP182,component_used,None
+36641ce7192b5f92,BSI-DSZ-CC-0977-2017,component_used,None
+054bd8196e015197,ANSSI-CC-2011/07,evaluation_reused,unclear
+122245dd15683e26,BSI-DSZ-CC-0269-2006,None,unclear
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,component_used,very difficult
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,component_used,None
+29cc495628e76ffa,BSI-DSZ-CC-0886-2013,component_shared,None
+2a07b85f61ac08b0,ANSSI-CC-2019/02,component_used,None
+30c6777901ae346c,BSI-DSZ-CC-0438-2007,component_used,None
+35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,component_shared,None
+36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,component_used,None
+373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,component_used,None
+3746512864b941d4,ANSSI-CC-2016/15,evaluation_reused,None
+37b94c08f8e25249,BSI-DSZ-CC-0404-2007,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0410-2007,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0555-2009,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0633-2010,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0674-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0675-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0709-2010,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0710-2010,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0730-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0797-2012,component_shared,None
+37b94c08f8e25249,BSI-DSZ-CC-0799-2012,component_shared,None
+37c8d23c44b95833,ANSSI-CC-2017/16,previous_version,None
+37d28734245a3bb1,BSI-DSZ-CC-0257-2004,component_used,None
+37e0d3cb098458d0,ANSSI-CC-2014/50,component_used,unclear
+37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,component_used,None
+37fe6036c2ac932b,ANSSI-CC-2014/06,component_used,None
+37fe6036c2ac932b,ANSSI-CC-2014/07,previous_version,None
+37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,component_used,None
+3817c0aca007989e,ANSSI-CC-2011/10,component_used,None
+3817c0aca007989e,ANSSI-CC-2011/12,component_shared,None
+3817c0aca007989e,BSI-DSZ-CC-0523-2008,component_used,None
+3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None
+384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,component_used,unclear
+384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None
+3976c9e492193315,BSI-DSZ-CC-1113-2021,re-evaluation,None
+39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None
+3a3b0a9113835307,BSI-DSZ-CC-0402-2008,re-evaluation,None
+3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None
+3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None
+3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None
+3a6a5f536bfbd4d4,ANSSI-CC-2009/59,previous_version,None
+3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,component_used,None
+3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,evaluation_reused,None
+3d149fe7c08bfc58,CRP256,previous_version,None
+3d756c83419bba28,ANSSI-CC-2011/07,component_used,None
+3d756c83419bba28,ANSSI-CC-2011/77,previous_version,None
+3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,component_used,None
+3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,component_used,None
+3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,component_used,None
+3e08a27e9d9c9b1e,NSCIB-CC-16-99111,irrelevant,self reference
+3e182053b03f1faf,CCEVS-VR-06-0044,irrelevant,self reference
+3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,component_used,None
+3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None
+3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None
+3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None
+3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,re-evaluation,None
+3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,re-evaluation,None
+3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None
+3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None
+3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,re-evaluation,None
+3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None
+3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None
+4057f5dbe4ffa4cd,ANSSI-CC-2017/57,previous_version,None
+40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,re-evaluation,None
+40fa0a4cb5193977,ANSSI-CC-2018/27,component_shared,None
+40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None
+40fc6ad0aed92913,ANSSI-CC-2017/54,previous_version,None
+40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,component_used,None
+41bd2924e9afced5,ANSSI-CC-2010/49,previous_version,None
+41bd2924e9afced5,ANSSI-CC-2010/50,previous_version,None
+420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None
+424f6d496746258b,ANSSI-CC-2019/12,component_used,None
+424f6d496746258b,BSI-DSZ-CC-0995-2018,re-evaluation,None
+4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None
+42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None
+42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,None
+42f53e51476f1a3c,ANSSI-CC-2011/07,previous_version,None
+438b59086f6ebd64,BSI-DSZ-CC-1107-2020,component_used,None
+43a4ca62d0c0b0da,ANSSI-CC-2017/63,evaluation_reused,None
+43a4ca62d0c0b0da,ANSSI-CC-2017/64,component_used,None
+43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,component_used,None
+43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,component_used,None
+43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,component_used,None
+449c74a92ebb61a4,ANSSI-CC-2017/24,component_used,None
+44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,component_used,None
+45098872448f5816,NSCIB-CC-0441513-CR,component_used,None
+46459cfb9c1045d8,ANSSI-CC-2018/27,component_used,None
+46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,component_used,None
+4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,irrelevant,None
+47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None
+4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None
+4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,re-evaluation,None
+4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None
+4893140ae5daadaf,CRP278,component_used,None
+48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,re-evaluation,None
+48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None
+48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None
+491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None
+491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,re-evaluation,None
+49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None
+49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None
+49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None
+4abc83fee59586b2,ANSSI-CC-2020/07,component_used,None
+4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,component_used,None
+4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None
+4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None
+4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None
+4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,re-evaluation,None
diff --git a/src/sec_certs/data/reference_annotations/final/train.csv b/src/sec_certs/data/reference_annotations/final/train.csv
new file mode 100644
index 00000000..6e106fc2
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/final/train.csv
@@ -0,0 +1,101 @@
+dgst,canonical_reference_keyword,label,comment
+00efeb17bcaafce6,ANSSI-CC-2009/34,component_used,None
+01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,component_used,None
+037577fc2019fcfa,ANSSI-CC-2021/49,previous_version,None
+055985699aee1e09,ANSSI-CC-2010/02,component_used,None
+05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None
+06e505abb8dad1b8,ANSSI-CC-2017/54,component_used,None
+0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None
+0bf7a19b22163465,ANSSI-CC-2015/80,evaluation_reused,None
+0bf7a19b22163465,ANSSI-CC-2016/44,evaluation_reused,unclear whether component used
+0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,component_used,None
+12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used
+13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,component_shared,"BAC, EAC thingy"
+16bdbde359584f99,ANSSI-CC-2017/61,component_used,None
+17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,component_used,None
+19c60923e31777e7,ANSSI-CC-2017/49,component_used,None
+1b8705a0486b3be1,NSCIB-CC-0209053-CR,previous_version,None
+1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,component_used,None
+1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,component_used,None
+27d11629261d8806,ANSSI-CC-2014/59,component_used,None
+28e70f46ba4394dd,ANSSI-CC-2012/08,component_used,None
+2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None
+30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,component_used,None
+30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None
+317713948b4473b3,BSI-DSZ-CC-0798-2012,component_shared,None
+34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None
+3515801dee00995f,BSI-DSZ-CC-0447-2008,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0447-2008
+36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None
+371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None
+3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,re-evaluation,None
+42799014c183e1b3,BSI-DSZ-CC-0312-2005,component_used,None
+42799014c183e1b3,BSI-DSZ-CC-0429-2007,re-evaluation,None
+463ecd64b7506048,ANSSI-CC-2018/52,component_used,None
+4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None
+4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None
+4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None
+54273dd266fce692,ANSSI-CC-2015/15,component_used,None
+5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,component_used,None
+5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear
+61832cb4291c343f,BSI-DSZ-CC-0955-2016,component_used,None
+627bfd69c2a831cf,ANSSI-CC-2014/37,evaluation_reused,None
+680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None
+691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None
+6a999675c9422dfb,BSI-DSZ-CC-0353-2006,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0353-2006
+6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None
+6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None
+744a7a202d909323,ANSSI-CC-2017/24,component_used,None
+7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,re-evaluation,None
+845bb039719ac5d8,ANSSI-CC-2010/40,previous_version,None
+84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None
+8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None
+8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None
+921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned
+9529eb793550093c,ANSSI-CC-2018/40,previous_version,None
+97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None
+a07548a64fac5794,BSI-DSZ-CC-0350-2007,re-evaluation,None
+a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None
+a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None
+a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,None
+a867d281d34d34b7,BSI-DSZ-CC-0976-2015,re-evaluation,None
+a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,re-evaluation,None
+aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None
+ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None
+ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None
+aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017
+af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None
+b42009eb34746731,ANSSI-CC-2021/50,previous_version,maybe previous_version
+b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None
+bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”"
+c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None
+c175537338906951,ANSSI-CC-2016/70,component_shared,None
+c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,None
+c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,re-evaluation,None
+c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None
+c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None
+cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None
+cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None
+d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert
+d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,re-evaluation,None
+d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None
+d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,re-evaluation,None
+dba4b5a166f9456a,ANSSI-CC-2014/76,re-evaluation,None
+df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None
+e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,re-evaluation,None
+e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified M7892 G12 component
+e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,component_used,None
+ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,re-evaluation,None
+ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None
+ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None
+ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None
+ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,re-evaluation,None
+ee1c6dd97918d74a,ANSSI-CC-2012/71,evaluation_reused,None
+f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None
+f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014
+f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None
+f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0863-2013
+f869956d14ea1694,ANSSI-CC-2012/68,component_used,None
+fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None
+b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,irrelevant,None
+c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery"
+e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,irrelevant,unclear
diff --git a/src/sec_certs/data/reference_annotations/final/valid.csv b/src/sec_certs/data/reference_annotations/final/valid.csv
new file mode 100644
index 00000000..2d662981
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/final/valid.csv
@@ -0,0 +1,108 @@
+dgst,canonical_reference_keyword,label,comment
+a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None
+657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,re-evaluation,None
+0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None
+983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,re-evaluation,None
+8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,re-evaluation,None
+774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,None
+c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,None,None
+a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None
+4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,component_used,None
+cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None
+24de96ea505af909,ANSSI-CC-2021/29,component_used,None
+2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None
+b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,re-evaluation,None
+d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None
+4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,None
+22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None
+2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,re-evaluation,None
+6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None
+120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None
+994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None
+16abf8ee0697e64f,ANSSI-CC-2014/61,component_shared,None
+f05de4db911ec8af,CRP243,previous_version,None
+38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None
+fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None
+fc875e0381a28435,BSI-DSZ-CC-0432-2007,re-evaluation,None
+c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None
+2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_shared,None
+2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None
+833ff87d69cff5db,383-4-184,None,unclear
+c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None
+647d17d44745a532,BSI-DSZ-CC-0904-2015,re-evaluation,None
+4489bfc781a82281,BSI-DSZ-CC-0817-2013,re-evaluation,None
+94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None
+37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None
+5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None
+5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,re-evaluation,None
+d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None
+0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,unclear
+84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None
+1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,re-evaluation,None
+d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None
+38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,re-evaluation,None
+e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None
+db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None
+4b3f577896c80f8a,ANSSI-CC-2016/44,evaluation_reused,None
+5f3c9c6bf76ba72d,ANSSI-CC-2018/51,component_used,None
+22d6578b30f5a227,BSI-DSZ-CC-1107-2020,component_used,None
+a0c38b4389ad7cf7,ANSSI-CC-2018/12,component_used,None
+19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,component_used,None
+f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,unclear
+2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,component_used,None
+3357c28c91cc2ed1,ANSSI-CC-2016/44,evaluation_reused,None
+f6317f134f532696,ANSSI-CC-2017/24,previous_version,None
+234da54efbcf734b,BSI-DSZ-CC-0917-2014,component_used,None
+1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None
+2532fe8c7214c485,BSI-DSZ-CC-0633-2010,component_used,None
+b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None
+35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,re-evaluation,None
+edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006.
+Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of
+technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the
+TOE was re-evaluated because of changes on top layers for modified module
+packaging requirements. Additional package types (MOB4 and complete
+passport inlay) and the relevant production sites were included in the evaluation
+process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the
+TOE itself. The Security Target was updated. "
+3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_used,None
+0fe8df0e85116b61,ANSSI-CC-2010/33,previous_version,None
+bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,re-evaluation,None
+b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,None
+7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,re-evaluation,None
+81273108dd167b98,BSI-DSZ-CC-0523-2008,re-evaluation,None
+63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None
+a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None
+4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None
+f4d510a39278e687,ANSSI-CC-2018/32,component_used,None
+c23bb542201aa511,ANSSI-CC-2014/14,previous_version,None
+625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None
+cae968d4926648bd,BSI-DSZ-CC-0754-2012,re-evaluation,None
+000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None
+66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None
+a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None
+031667f4e242da61,CCEVS-VR-07-0054,component_used,None
+459539da9315ccac,ANSSI-CC-2021/30,previous_version,None
+69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None
+a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None
+50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None
+5cf51622fafaf868,BSI-DSZ-CC-1005-2016,re-evaluation,None
+1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None
+0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None
+8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None
+6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None
+4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None
+e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None
+03fad65e5af65088,BSI-DSZ-CC-0696-2011,re-evaluation,None
+720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None
+9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None
+d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None
+73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None
+2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None
+f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,component_used,None
+f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,component_used,None
+2f43fe24bd8e91e1,ANSSI-CC-2020/65,component_used,None
+d96438a7165e5d82,BSI-DSZ-CC-0411-2007,component_used,None
+7623950728f52a71,ANSSI-CC-2016/06,component_used,None
+ba69597434946260,BSI-DSZ-CC-0293-2005,component_used,None
+50aef770cb4f028f,BSI-DSZ-CC-1059-2018,component_used,None
diff --git a/src/sec_certs/data/reference_annotations/jano/test.csv b/src/sec_certs/data/reference_annotations/jano/test.csv
new file mode 100644
index 00000000..52e72176
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/jano/test.csv
@@ -0,0 +1,201 @@
+dgst,canonical_reference_keyword,label,comment
+15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,re-evaluation,None
+15b121492722bffb,BSI-DSZ-CC-0754-2012,re-evaluation,None
+16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,re-evaluation,evaluation_reused more like
+16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None
+16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None
+173097995b7a7f12,ANSSI-CC-2012/49,evaluation_reused,None
+173097995b7a7f12,ANSSI-CC-2013/28,component_used,None
+173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None
+173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None
+177dee4c1051e612,BSI-DSZ-CC-0854-2013,re-evaluation,None
+17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None
+183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None
+183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None
+18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,component_used,None
+18b7e5bdf459ca13,NSCIB-CC-07-09219,previous_version,also evaluation_reused
+195366bfba7213a5,BSI-DSZ-CC-0266-2005,None,None
+195366bfba7213a5,BSI-DSZ-CC-0322-2005,None,None
+195366bfba7213a5,BSI-DSZ-CC-0399-2007,None,None
+1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,None,None
+19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None
+1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,re-evaluation,None
+1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None
+1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None
+1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None
+1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None
+1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,re-evaluation,None
+1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,also previous version to a degree
+1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,component_shared,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,re-evaluation,None
+1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None
+1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,re-evaluation,None
+2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,re-evaluation,None
+21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None
+21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None
+22ede463dbf1a105,ANSSI-CC-2015/36,component_used,None
+22ede463dbf1a105,ANSSI-CC-2015/80,irrelevant,mistake in report
+22ede463dbf1a105,ANSSI-CC-2016/43,evaluation_reused,None
+22ede463dbf1a105,ANSSI-CC-2016/44,evaluation_reused,None
+23a85a7f9c07412f,ANSSI-CC-2014/46,evaluation_reused,component_used probably
+23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,component_used probably
+24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None
+24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None
+259df98e7476a843,BSI-DSZ-CC-0666-2012,re-evaluation,None
+259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None
+25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None
+25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None
+26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,component_used,None
+26c7b173668167d8,BSI-DSZ-CC-0813-2012,component_used,None
+2760aeedce0b79db,ANSSI-CC-2011/64,evaluation_reused,None
+2760aeedce0b79db,BSI-DSZ-CC-0555-2009,component_used,None
+27ccec0740bb7915,ANSSI-CC-2010/02,component_used,None
+27ccec0740bb7915,ANSSI-CC-2010/20,evaluation_reused,None
+27ccec0740bb7915,ANSSI-CC-2010/35,component_shared,None
+296618f10b019d9c,BSI-DSZ-CC-0417-2008,component_used,None
+29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None
+29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None
+29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,re-evaluation,None
+2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None
+2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None
+2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,re-evaluation,None
+2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None
+2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None
+2bd9661657e78278,ANSSI-CC-2013/13,component_used,None
+2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,re-evaluation,None
+2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,re-evaluation,None
+2d046178aa118fff,BSI-DSZ-CC-0293-2005,re-evaluation,None
+2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None
+2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None
+2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None
+2f9feaf4121720da,BSI-DSZ-CC-0547-2009,evaluation_reused,maybe recertification?
+30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None
+3183669bf78db43b,ANSSI-CC-2015/45,component_used,None
+3183669bf78db43b,ANSSI-CC-2016/64,component_used,None
+3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,component_used,None
+31fd07069aebf013,BSI-DSZ-CC-0466-2008,component_used,None
+328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None
+328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None
+3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,re-evaluation,None
+3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None
+337ece90615ed69d,ANSSI-CC-2010/02,component_used,None
+3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None
+33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None
+33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,re-evaluation,None
+34506b8fab6dd7c3,ANSSI-CC-2014/22,previous_version,None
+34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None
+34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None
+3519c14e4114d93d,BSI-DSZ-CC-0782-2012,component_used,None
+35b9be2203d2b9e6,ANSSI-CC-2012/70,component_used,None
+35efe13fa9e93a68,CRP178,component_used,None
+35efe13fa9e93a68,CRP182,component_used,None
+36641ce7192b5f92,BSI-DSZ-CC-0977-2017,component_used,None
+054bd8196e015197,ANSSI-CC-2011/07,evaluation_reused,None
+122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,None,None
+25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,evaluation_reused,None
+29cc495628e76ffa,BSI-DSZ-CC-0886-2013,component_shared,None
+2a07b85f61ac08b0,ANSSI-CC-2019/02,component_used,None
+30c6777901ae346c,BSI-DSZ-CC-0438-2007,component_used,None
+35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,component_shared,None
+36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,component_used,None
+373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,component_used,None
+3746512864b941d4,ANSSI-CC-2016/15,previous_version,None
+37b94c08f8e25249,BSI-DSZ-CC-0404-2007,None,None
+37b94c08f8e25249,BSI-DSZ-CC-0410-2007,None,None
+37b94c08f8e25249,BSI-DSZ-CC-0555-2009,None,None
+37b94c08f8e25249,BSI-DSZ-CC-0633-2010,None,None
+37b94c08f8e25249,BSI-DSZ-CC-0674-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0675-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0709-2010,None,None
+37b94c08f8e25249,BSI-DSZ-CC-0710-2010,None,None
+37b94c08f8e25249,BSI-DSZ-CC-0730-2011,component_used,None
+37b94c08f8e25249,BSI-DSZ-CC-0797-2012,component_shared,None
+37b94c08f8e25249,BSI-DSZ-CC-0799-2012,None,missing BSI-DSZ-CC-0804-2012
+37c8d23c44b95833,ANSSI-CC-2017/16,previous_version,None
+37d28734245a3bb1,BSI-DSZ-CC-0257-2004,previous_version,missing some CRP ids
+37e0d3cb098458d0,ANSSI-CC-2014/50,component_used,None
+37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,component_used,None
+37fe6036c2ac932b,ANSSI-CC-2014/06,component_used,None
+37fe6036c2ac932b,ANSSI-CC-2014/07,previous_version,None
+37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,component_used,None
+3817c0aca007989e,ANSSI-CC-2011/10,component_used,None
+3817c0aca007989e,ANSSI-CC-2011/12,previous_version,None
+3817c0aca007989e,BSI-DSZ-CC-0523-2008,component_used,None
+3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None
+384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,component_used,None
+384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None
+3976c9e492193315,BSI-DSZ-CC-1113-2021,re-evaluation,None
+39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None
+3a3b0a9113835307,BSI-DSZ-CC-0402-2008,re-evaluation,None
+3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None
+3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None
+3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None
+3a6a5f536bfbd4d4,ANSSI-CC-2009/59,previous_version,None
+3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,component_used,None
+3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,evaluation_reused,None
+3d149fe7c08bfc58,CRP256,previous_version,None
+3d756c83419bba28,ANSSI-CC-2011/07,component_used,None
+3d756c83419bba28,ANSSI-CC-2011/77,previous_version,None
+3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,component_used,None
+3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,component_used,None
+3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,component_used,None
+3e08a27e9d9c9b1e,NSCIB-CC-16-99111,previous_version,"but also an issue of cert ID assignment, we need to better canonicalize these “V2” and -CR things"
+3e182053b03f1faf,CCEVS-VR-06-0044,irrelevant,None
+3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,component_used,None
+3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None
+3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None
+3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None
+3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,re-evaluation,None
+3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,re-evaluation,None
+3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None
+3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None
+3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,re-evaluation,None
+3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None
+3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None
+4057f5dbe4ffa4cd,ANSSI-CC-2017/57,previous_version,None
+40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,re-evaluation,None
+40fa0a4cb5193977,ANSSI-CC-2018/27,previous_version,None
+40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None
+40fc6ad0aed92913,ANSSI-CC-2017/54,component_used,None
+40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,component_used,None
+41bd2924e9afced5,ANSSI-CC-2010/49,previous_version,None
+41bd2924e9afced5,ANSSI-CC-2010/50,previous_version,None
+420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None
+424f6d496746258b,ANSSI-CC-2019/12,component_used,None
+424f6d496746258b,BSI-DSZ-CC-0995-2018,re-evaluation,None
+4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None
+42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None
+42d2e68c29d9eef7,ANSSI-CC-2018/30,previous_version,None
+42f53e51476f1a3c,ANSSI-CC-2011/07,previous_version,None
+438b59086f6ebd64,BSI-DSZ-CC-1107-2020,component_used,None
+43a4ca62d0c0b0da,ANSSI-CC-2017/63,evaluation_reused,None
+43a4ca62d0c0b0da,ANSSI-CC-2017/64,irrelevant,"/63 or /64, unsure"
+43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,component_used,None
+43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,component_used,None
+43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,component_used,None
+449c74a92ebb61a4,ANSSI-CC-2017/24,component_used,None
+44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,component_used,None
+45098872448f5816,NSCIB-CC-0441513-CR,component_used,None
+46459cfb9c1045d8,ANSSI-CC-2018/27,component_used,None
+46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,component_used,None
+4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,irrelevant,BSI-DSZ-CC-0891-V4-2018 mentioned in report
+47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None
+4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None
+4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,re-evaluation,None
+4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None
+4893140ae5daadaf,CRP278,component_shared,None
+48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,re-evaluation,None
+48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None
+48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None
+491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None
+491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,re-evaluation,None
+49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None
+49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None
+49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None
+4abc83fee59586b2,ANSSI-CC-2020/07,component_used,None
+4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,component_used,None
+4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None
+4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None
+4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None
+4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,re-evaluation,None
diff --git a/src/sec_certs/data/reference_annotations/jano/train.csv b/src/sec_certs/data/reference_annotations/jano/train.csv
new file mode 100644
index 00000000..cce5f69f
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/jano/train.csv
@@ -0,0 +1,101 @@
+dgst,canonical_reference_keyword,label,comment
+e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified M7892 G12 component
+0bf7a19b22163465,ANSSI-CC-2016/44,evaluation_reused,unclear whether component used
+ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None
+8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None
+680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None
+3515801dee00995f,BSI-DSZ-CC-0447-2008,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0447-2008
+6a999675c9422dfb,BSI-DSZ-CC-0353-2006,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0353-2006
+12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used
+d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert
+921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned
+00efeb17bcaafce6,ANSSI-CC-2009/34,component_used,None
+c175537338906951,ANSSI-CC-2016/70,component_used,None
+fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None
+ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None
+f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0863-2013
+cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None
+dba4b5a166f9456a,ANSSI-CC-2014/76,previous_version,None
+aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None
+b42009eb34746731,ANSSI-CC-2021/50,evaluation_reused,maybe previous_version
+463ecd64b7506048,ANSSI-CC-2018/52,component_used,None
+c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,re-evaluation,None
+c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None
+0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None
+2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None
+e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,unclear
+ee1c6dd97918d74a,ANSSI-CC-2012/71,evaluation_reused,None
+e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,component_used,None
+845bb039719ac5d8,ANSSI-CC-2010/40,previous_version,None
+af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None
+d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None
+54273dd266fce692,ANSSI-CC-2015/15,component_used,None
+4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None
+d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,re-evaluation,None
+a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None
+f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None
+8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None
+b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,unclear
+371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None
+f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014
+e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,re-evaluation,None
+4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None
+a867d281d34d34b7,BSI-DSZ-CC-0976-2015,re-evaluation,None
+16bdbde359584f99,ANSSI-CC-2017/61,component_used,None
+34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None
+a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None
+0bf7a19b22163465,ANSSI-CC-2015/80,evaluation_reused,None
+1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,component_used,None
+13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,component_shared,"BAC, EAC thingy"
+f869956d14ea1694,ANSSI-CC-2012/68,component_used,None
+037577fc2019fcfa,ANSSI-CC-2021/49,evaluation_reused,None
+17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,component_used,None
+5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,component_used,None
+1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,component_used,None
+28e70f46ba4394dd,ANSSI-CC-2012/08,component_used,None
+61832cb4291c343f,BSI-DSZ-CC-0955-2016,component_used,None
+42799014c183e1b3,BSI-DSZ-CC-0312-2005,component_used,None
+627bfd69c2a831cf,ANSSI-CC-2014/37,evaluation_reused,None
+aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017
+05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None
+9529eb793550093c,ANSSI-CC-2018/40,evaluation_reused,None
+7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,re-evaluation,None
+ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,re-evaluation,None
+ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,re-evaluation,None
+ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None
+30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None
+6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None
+06e505abb8dad1b8,ANSSI-CC-2017/54,component_used,None
+317713948b4473b3,BSI-DSZ-CC-0798-2012,component_shared,None
+055985699aee1e09,ANSSI-CC-2010/02,component_used,None
+b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None
+691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None
+a07548a64fac5794,BSI-DSZ-CC-0350-2007,re-evaluation,None
+19c60923e31777e7,ANSSI-CC-2017/49,component_used,None
+f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None
+c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None
+42799014c183e1b3,BSI-DSZ-CC-0429-2007,re-evaluation,None
+3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,re-evaluation,None
+5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear
+4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None
+c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None
+ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None
+30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,component_used,None
+c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,None
+ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None
+bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,None,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”"
+97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None
+744a7a202d909323,ANSSI-CC-2017/24,component_used,None
+df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None
+6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None
+1b8705a0486b3be1,NSCIB-CC-0209053-CR,evaluation_reused,None
+a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,re-evaluation,None
+cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None
+27d11629261d8806,ANSSI-CC-2014/59,component_used,None
+a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,evaluation_reused,None
+d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,re-evaluation,None
+c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery"
+84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None
+36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None
+01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,component_used,None
+0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,component_used,None
diff --git a/src/sec_certs/data/reference_annotations/jano/valid.csv b/src/sec_certs/data/reference_annotations/jano/valid.csv
new file mode 100644
index 00000000..197ff250
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/jano/valid.csv
@@ -0,0 +1,108 @@
+dgst,canonical_reference_keyword,label,comment
+a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None
+657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,re-evaluation,None
+0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None
+983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,re-evaluation,None
+8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,re-evaluation,None
+774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,component_used,None
+c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,component_shared,None
+a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None
+4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,component_used,None
+cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None
+24de96ea505af909,ANSSI-CC-2021/29,component_used,None
+2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None
+b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,re-evaluation,None
+d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None
+4b3f577896c80f8a,ANSSI-CC-2015/80,component_used,None
+22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None
+2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,re-evaluation,None
+6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None
+120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None
+994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None
+16abf8ee0697e64f,ANSSI-CC-2014/61,evaluation_reused,None
+f05de4db911ec8af,CRP243,component_used,None
+38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None
+fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None
+fc875e0381a28435,BSI-DSZ-CC-0432-2007,re-evaluation,None
+c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None
+2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_used,None
+2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None
+833ff87d69cff5db,383-4-184,None,unclear
+c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None
+647d17d44745a532,BSI-DSZ-CC-0904-2015,re-evaluation,None
+4489bfc781a82281,BSI-DSZ-CC-0817-2013,re-evaluation,None
+94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None
+37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None
+5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None
+5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,re-evaluation,None
+d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None
+0f53257eb74bee54,BSI-DSZ-CC-0417-2008,None,unclear
+84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None
+1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,re-evaluation,None
+d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None
+38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,re-evaluation,None
+e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None
+db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None
+4b3f577896c80f8a,ANSSI-CC-2016/44,component_used,None
+5f3c9c6bf76ba72d,ANSSI-CC-2018/51,component_used,None
+22d6578b30f5a227,BSI-DSZ-CC-1107-2020,component_used,None
+a0c38b4389ad7cf7,ANSSI-CC-2018/12,component_used,None
+19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,component_used,None
+f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,unclear
+2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,component_used,None
+3357c28c91cc2ed1,ANSSI-CC-2016/44,component_used,None
+f6317f134f532696,ANSSI-CC-2017/24,evaluation_reused,None
+234da54efbcf734b,BSI-DSZ-CC-0917-2014,component_used,None
+1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None
+2532fe8c7214c485,BSI-DSZ-CC-0633-2010,component_used,None
+b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None
+35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,re-evaluation,None
+edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006.
+Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of
+technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the
+TOE was re-evaluated because of changes on top layers for modified module
+packaging requirements. Additional package types (MOB4 and complete
+passport inlay) and the relevant production sites were included in the evaluation
+process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the
+TOE itself. The Security Target was updated. "
+3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_used,None
+0fe8df0e85116b61,ANSSI-CC-2010/33,evaluation_reused,None
+bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,re-evaluation,None
+b00bb452e7a8176c,ANSSI-CC-2016/57,evaluation_reused,None
+7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,previous_version,None
+81273108dd167b98,BSI-DSZ-CC-0523-2008,re-evaluation,None
+63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None
+a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None
+4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None
+f4d510a39278e687,ANSSI-CC-2018/32,component_used,None
+c23bb542201aa511,ANSSI-CC-2014/14,component_used,None
+625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None
+cae968d4926648bd,BSI-DSZ-CC-0754-2012,re-evaluation,None
+000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None
+66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None
+a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None
+031667f4e242da61,CCEVS-VR-07-0054,component_used,None
+459539da9315ccac,ANSSI-CC-2021/30,previous_version,None
+69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None
+a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None
+50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None
+5cf51622fafaf868,BSI-DSZ-CC-1005-2016,re-evaluation,None
+1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None
+0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None
+8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None
+6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None
+4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None
+e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None
+03fad65e5af65088,BSI-DSZ-CC-0696-2011,re-evaluation,None
+720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None
+9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None
+d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None
+73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None
+2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None
+f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,component_used,None
+f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,component_used,None
+2f43fe24bd8e91e1,ANSSI-CC-2020/65,component_used,None
+d96438a7165e5d82,BSI-DSZ-CC-0411-2007,component_used,None
+7623950728f52a71,ANSSI-CC-2016/06,component_used,None
+ba69597434946260,BSI-DSZ-CC-0293-2005,component_used,None
+50aef770cb4f028f,BSI-DSZ-CC-1059-2018,component_used,None
diff --git a/data/cert_id_eval/random_references.csv b/src/sec_certs/data/reference_annotations/outdated_manually_annotated_references.csv
index 23a1cd33..346ed3af 100644
--- a/data/cert_id_eval/random_references.csv
+++ b/src/sec_certs/data/reference_annotations/outdated_manually_annotated_references.csv
@@ -30,8 +30,8 @@ b2f3874cdb97fe27,BSI-DSZ-CC-0790-2013,report,self,
 546f9e2496b1d11c,ANSSI-CC-2015/14,report,self,
 1dd7f4bf2e677073,ANSSI-CC-2014/20,report,self,
 b24a14935edd51ad,BSI-DSZ-CC-0783-2013,report,self,
-b24a14935edd51ad,BSI-DSZ-CC-0750-2010,report,component used,"The TOE consists of the Java Card Operating System the NXP Crypto Library for SmartMX v2.7 (certified under BSI-DSZ-CC-0750-2010) and the hardware platform P5CD145V0B or P5CC145V0B (certified under BSI-DSZ-CC-0645-2011)
-b24a14935edd51ad,BSI-DSZ-CC-0645-2011,report,component used,"The TOE consists of the Java Card Operating System the NXP Crypto Library for SmartMX v2.7 (certified under BSI-DSZ-CC-0750-2010) and the hardware platform P5CD145V0B or P5CC145V0B (certified under BSI-DSZ-CC-0645-2011)
+b24a14935edd51ad,BSI-DSZ-CC-0750-2010,report,component used,"The TOE consists of the Java Card Operating System the NXP Crypto Library for SmartMX v2.7 (certified under BSI-DSZ-CC-0750-2010) and the hardware platform P5CD145V0B or P5CC145V0B (certified under BSI-DSZ-CC-0645-2011)"
+b24a14935edd51ad,BSI-DSZ-CC-0645-2011,report,component used,"The TOE consists of the Java Card Operating System the NXP Crypto Library for SmartMX v2.7 (certified under BSI-DSZ-CC-0750-2010) and the hardware platform P5CD145V0B or P5CC145V0B (certified under BSI-DSZ-CC-0645-2011)"
 b24a14935edd51ad,BSI-DSZ-CC-0783,target,self,
 b24a14935edd51ad,BSI-DSZ-CC-0750,target,component used,"Table 2 gives the details of the underlying evaluations of the cryptographic library and the underlying hardware platforms."
 b24a14935edd51ad,BSI-DSZ-CC-0645,target,component used,"Table 2 gives the details of the underlying evaluations of the cryptographic library and the underlying hardware platforms."
@@ -179,4 +179,64 @@ c3b443f243b95913,CCEVS-VR-11219-2021,report,self,
 183a5cd3e2aaa0ec,ANSSI-CC-2017/81,report,self,
 183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,report,component used,"The product consists of the following components: the previously certified M7892 G12 component (see [BSI-DSZ-CC-0891-V2-2016])"
 183a5cd3e2aaa0ec,ANSSI-CC-2017/76,report,component used,"The product consists of the following components: an operating system in the form of an open configuration platform or closed Java Card MultiApp V4.0.1. This platform is certified under the reference [ANSSI-CC-2017/76]"
-183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,target,component used,"The evaluation is a composite evaluation and uses the results of the CC evaluation provided by [CR-IC]"
\ No newline at end of file
+183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,target,component used,"The evaluation is a composite evaluation and uses the results of the CC evaluation provided by [CR-IC]"
+18bb29b147e80caa,BSI-DSZ-CC-1040-2019,report,component used,"et du microcontrôleur « NXP Secure Smart Card Controller N7121 avec son firmware et sa bibliothèque cryptographique dédiés » (certifié sous la référence BSI-DSZ-CC- 1040, voir [CER_IC])"
+d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,target,component used,
+14df54305cad83b4,BSI-DSZ-CC-0385-2006,report,basis of recertification,"The product AIX 6 version 6100-00-02 with optional Virtual I/O Server (VIOS) version 1.5 has undergone the certification procedure at BSI. This is a re-certification based on BSI- DSZ-CC-0385-2006. Specific results from the evaluation process BSI-DSZ-CC-0385-2006 were re-used."
+055985699aee1e09,ANSSI-CC-2010/02,report,component used,
+a9639521d8a5ab09,ANSSI-CC-2010/02,report,component used,"The evaluation has been performed according to the composition scheme as defined in the guide [COMP] in order to assess that no weakness comes from the integration of the software in the microcontroller already certified. Therefore, the results of the evaluation of the microcontroller SB23YR80B with Neslib version 3.0” at EAL6 level augmented with ALC_FLR.1, compliant with the [PP0035] protection profile, have been used. This microcontroller has been certified the 10 February 2010 under the reference ANSSI-CC-2010/02. The maintenance report ANSSI-2010/02-M01 has also been issued the 19th March 2010 for this product."
+5f1bc82727f54376,ANSSI-CC-2021/29,report,component used,
+317713948b4473b3,BSI-DSZ-CC-0710-2010,report,NaN,
+c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,basis of eval,"Also, specific ALC related parts were re-used in the sense of AIS38 from BSI-DSZ-CC-0891-V2-2016 (Infineon Security Controller M7892 D11 and G12)."
+b0b7b073ca2dfe5f,ANSSI-CC-2019/28,report,component used,
+25eade3365f92578,BSI-DSZ-CC-0338-2005,report,component used,
+b4a2999b439b1ba8,BSI-DSZ-CC-1178-2021,report,basis of recertification,"The product Infineon Technologies AG OPTIGATM Trusted Platform Module SLB9672_2.0 v16.10.16488.00 and SLB9673_2.0 v26.10.16688.00, has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-1178-2021. Specific results from the evaluation process BSI-DSZ-CC-1178-2021 were re-used."
+850f5b2c312299d5,BSI-DSZ-CC-0330-2007,report,basis of recertification,"Das Produkt Virtuelle Poststelle des Bundes, (OSCI) Version 2.2.3.2 hat das Zertifizierungsverfahren beim BSI durchlaufen. Es handelt sich um eine Re-Zertifizierung basierend auf BSI-DSZ-CC-0330-2007."
+e9e12b044b495c6f,ANSSI-CC-2010/01,report,component used,
+aea11fd4d2a7709d,BSI-DSZ-CC-0808-V2-2016,report,basis of recertification,"TCOS Passport Version 2.1 Release 3/P60D144 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0808-V2-. Specific results from the evaluation process BSI-DSZ-CC-0808-V2-2016 were re-used"
+845bb039719ac5d8,BSI-DSZ-CC-0645-2010,target,component used,
+4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component used,
+1b0a6ecee1f830fc,ANSSI-CC-2011/07,report,component used,
+9ae8bef18938d2df,BSI-DSZ-CC-0965-2015,report,basis of recertification,"The product Infineon Technologies AG Trusted Platform Module SLB9670_2.0, v7.40.2098.00 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0965-2015. Specific results from the evaluation process BSI-DSZ-CC-0965-2015 were re-used."
+7e82a1ceb29e4019,NSCIB-CC-12-36243,report,self,
+54dd28994604ee35,BSI-DSZ-CC-0481-2008,NaN,NaN,
+1357546ec01b1a65,BSI-DSZ-CC-0438-2007,report,basis of recertification,"The product S3CC91C 16-Bit RISC Microcontroller for Smart Card, Version 0 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0438-2007 . Specific results from the evaluation process based on BSI-DSZ-CC-0438-2007 were re-used."
+30120e4f3aa2f30a,BSI-DSZ-CC-0897-2013,target,component used,
+27b4366a0f1fcdef,CCEVS-VR-07-0054,report,basis for,
+27b4366a0f1fcdef,BSI-DSZ-CC-0403-2008,report,basis of recertification,"The product Oracle Database 11g Standard Edition and Standard Edition One, Release 11.1.0.7 with Critical Patch Updates up to and including July 2009 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0403-2008. Specific results from the evaluation process BSI-DSZ-CC-0403-2008 were re-used."
+e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,basis of recertification,"This is a re-certification based on BSI-DSZ-CC-0961-V2-2018. Specific results from the evaluation process BSI-DSZ-CC-0961-V2-2018 were re-used. Also, specific ALC related parts were re-used in the sense of AIS38 from BSI-DSZ-CC-1079."
+05a8a84b60a30ceb,BSI-DSZ-CC-1110-V3-2020,report,component used,"For details concerning the CC evaluation of the underlying IC see the evaluation documentation under the Certification ID BSI-DSZ-CC-1110-V3-2020 ([16], [17])."
+1b797b27b981aafe,BSI-DSZ-CC-1059-2018,report,component used,"- du microcontrôleur « NXP P60D145 » certifié sous la référence [CER-IC] ;"
+691119fe8a2f8e1c,BSI-DSZ-CC-0944-2014,report,basis of recertification,"The product SLS 32TLC00xS(M) CIPURSETM4move, v1.00.00 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0944-2014. Specific results from the evaluation process BSI-DSZ-CC-0944-2014 were re-used."
+0d164fb0d72dfa00,BSI-DSZ-CC-0645-2010,report,component used,
+a154a01ff4cf04c7,BSI-DSZ-CC-1059-2018,report,component used,
+537154d2c4344d48,ANSSI-CC-2018/31,report,component used,
+265e576fd3517f91,BSI-DSZ-CC-0866-2013,report,basis of recertification,"The product IDeal Pass v2 - SAC/EAC JC ePassport 4.0.0 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0866-2013. Specific results from the evaluation process BSI-DSZ-CC-0866-2013 were re-used."
+38f79f1a48e7844f,ANSSI-CC-2017/49,report,component used,
+66faa094a9333d1a,ANSSI-CC-2014/86,NaN,NaN,
+b8875373ee66e841,ANSSI-CC-2011/07,report,component used,
+3a3ae16e94bc7aba,BSI-DSZ-CC-0995-2018,report,basis of eval,"Please note that in consistency to the claimed protection profile BSI-CC-PP-0055-2009 the security mechanism Basic Access Control is in the focus of this evaluation process. The further security mechanisms Password Authenticated Connection Establishment, Extended Access Control and Active Authentication are subject of the separate evaluation process BSI-DSZ-CC-0995-2018 [25]."
+c18fea56107696a8,BSI-DSZ-CC-0737-2012,report,basis of recertification,"The product Bundesdruckerei Document Reading Application, Version 1.2.1129 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0737-2012. Specific results from the evaluation process BSI-DSZ-CC-0737-2012 were re-used."
+0694786160024e30,BSI-DSZ-CC-0990-2016,report,basis of recertification,"The product SLS 32TLC00xS(M) CIPURSETM 4move, V1.0.2 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0990-2016. Specific results from the evaluation process BSI-DSZ-CC-0990-2016 were re-used."
+c63e878cc60a8682,BSI-DSZ-CC-0883-2013,target,component used,"The HW used to support the OS is the M7794 A12 and G12 (SLE77CLFX2407PM) component. This component M7794 A12 and G12 (BSI-DSZ-CC-0883-2013) is certified CC EAL5+ [9]."
+50aef770cb4f028f,BSI-DSZ-CC-1059-2018,report,component used,
+425f13fb7e94add5,ANSSI-CC-2020/21,report,component used,
+d472c3dfb2f2a810,ANSSI-CC-2011/24,report,previous version,"Par ailleurs, cette évaluation a également pris en compte les résultats de l’évaluation de la version précédente du produit certifié (voir [ANSSI-CC-2011/24])."
+d5074e4b714ac9e7,ANSSI-CC-2011/07,report,component used,
+629372a8091eafae,ANSSI-CC-2020/71,report,component used,
+cefb1202ed3cd997,BSI-DSZ-CC-0344-2005,NaN,NaN,
+34de7910ace6d2e9,BSI-DSZ-CC-0410-2007,report,basis for,
+0fe8df0e85116b61,ANSSI-CC-2010/35,report,previous version,
+43109d44111df5d3,BSI-DSZ-CC-0915-2016,report,component used,
+657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,report,basis of recertification,"The product Infineon Technologies Smart Card IC (Security Controller) M5072 G11 with optional RSA v1.03.006, EC v1.03.006 and Toolbox v1.03.006 with specific IC dedicated software has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0946-2014. Specific results from the evaluation process BSI-DSZ-CC-0946-2014 were re-used."
+a0c38b4389ad7cf7,ANSSI-CC-2018/12,report,component used,
+5c06e4fb1887fdc7,BSI-DSZ-CC-0968-2016,report,basis of eval,
+0bb56716a1fbef1e,ANSSI-CC-2019/35,report,component used,
+83bc5fbf2b54e83b,BSI-DSZ-CC-0786-2012,report,basis of recertification,"The product Infineon smartcard IC (Security Controller) M7794 A12 with optional RSA2048/4096 v1.02.013, EC v1.02.013 and Toolbox v1.02.013, has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0786-2012. Specific results from the evaluation process BSI-DSZ-CC-0786-2012 were re-used."
+2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,report,component used,
+720aab4f782f4c60,BSI-DSZ-CC-0555-2009,report,component used,
+0bae4647efd39e51,BSI-DSZ-CC-1190-2022,report,basis of recertification,
+647d17d44745a532,BSI-DSZ-CC-0904-2015,report,basis of recertification,"The product TCOS FlexCert Version 2.0 Release 2/SLC52 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0904-2015. Specific results from the evaluation process BSI-DSZ-CC-0904-2015 (including subsequent maintenance procedures BSI-DSZ-CC-0904-2015-MA-01 and BSI-DSZ-CC-0904-2015- MA-02) were re-used."
+8927428c96c0b59a,BSI-DSZ-CC-0813-2012,report,component used,
+8928e191ee2fd3b9,BSI-DSZ-CC-0626-2009,report,component used,
+c23bb542201aa511,ANSSI-CC-2014/06,report,component used,
diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md
new file mode 100644
index 00000000..8521eead
--- /dev/null
+++ b/src/sec_certs/data/reference_annotations/readme.md
@@ -0,0 +1,61 @@
+# Reference annotations
+
+This folder contains data related to learning the reference annotations. This document also describeds the utilized methodology.
+
+- The folder [split](split) contains split of the CC Dataset to `train/valid/test` splits for learning.
+- The csv file [outdated_manually_annotated_references.csv](./outdated_manually_annotated_references.csv) contains manually acquired labels to references obtained with **old methodology** for the sake of paper 1 submission.
+- The folder [adam](adam/) contains manual annotations created by Adam
+- The folder [jano](jano/) contains manual annotations created by Jano
+- The folder [conflicts](conflicts/) contains conflicting annotations between Adam and Jano, as weel as their resolution
+- The contents of the [final](final/) folder can thus be obtained by taking annotations either from `adam` or `jano` folder and masking them by `resolution_label` from `conflicts` folder.
+
+## Reference classification methodology
+
+### Data splits and manual annotations
+
+1. Two co-authors independently inspect identical set of 100 random certificates and capture the observed relations into reference taxonomy to form the annotation guidelines. See [reference taxonomy](#reference-taxonomy) below.
+2. We split all certificates for which we register a direct outgoing reference in either security target or certification report into `train/valid/test` splits in `30/20/50` fashion (see [split](split/)).
+3. We sample 100 train, 100 valid, 200 test pairs of reference instances (represented by `(dgst, canonical_reference_keyword)` pairs) for manual annotations.
+4. Two co-authors independently assign each of these instances with a single label from the reference taxonomy.
+5. We measure the inter-annotator agreement with Cohen's Kappa and percentage, see [inter-annotator agreement](#inter-annotator-agreement).
+6. We resolve conflicts in the annotations in a meeting held by the co-authors. We use this consensual annotations for training and evaluation described below.
+
+### Supervised learning of the annotations
+
+1. For each pair `(dgst, referenced_cert_id)`, we recover the relevant text segments both from certification report and security target that mention the `referenced_cert_id`.
+2. We apply text processing on the segments (e.g., unify re-certification vs. recertification, etc.)
+3. We train a baseline model based on TF-IDF (or count vectorization in general), random forest and a soft-voting layer on top of that.
+    - Random forest classifies single segment with a probability of a given label.
+    - Soft voting compares probabilities of the given labels on all segments, takes their square and chooses the maximum.
+4. We train a sentence transformer with the same soft-voting layer on top of that.
+5. Finetune hyperparameters.
+6. We evalute the results on the test set using weighted F1 score.
+
+### Reference taxonomy
+
+After manually inspecting ~100 random certificates, we have identified the following reference meanings:
+
+- **Component used**: The referenced certificate is a component used in the examined certificate (e.g., IC used by a smartcard). Some evaluation results were likely shared/re-used.
+- **Component shared**: The referenced certificate shares some components with the examined certificate. Some evaluation results were likely shared/re-used.
+- **Evaluation reused**: The evaluation results of the referenced certificate were used for evaluation of the examined certificate, due to reasons that could not be resolved.
+- **Re-evaluation**: The examined certificate is a re-evaluation of the referenced certificate. For definition of re-evaluation, see [Assurance Continuity: CCRA Requirements](https://www.commoncriteriaportal.org/files/operatingprocedures/CCDB-011-v2.2-2021-Sep-30-Final-Assurance_Continuity.pdf).
+- **Previous version**: The product in the referenced certificate is a previous version of the product in the examined certificate and the re-certification is not explicitly mentioned.
+- **None**: The annotator could not assign any of the previous contexts.
+- **Irrelevant**: The reference is irrelevant to the studied certificate (typo, left-out reference from a template, ...)
+
+These can be further merged into the following super-categories:
+
+- **Some sub-component relationship** `component_used`, `component_shared`, and `evaluation_reused`
+- **Previous version**: `previous_version` and `re-evaluation`
+- **None**: `None` or `irrelevant`
+
+###  Inter-annotator agreement
+
+The inter-annotator agreement is measured both with Cohen's Kappa and with percentage. The results are as follows:
+
+| Cohen's Kappa | Percentage |
+| ------------- | ---------- |
+| 0.71          | 0.82       |
+
+The code used to measure the agreement is stored in `notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb`.
+
diff --git a/data/reference_annotations_split/test.json b/src/sec_certs/data/reference_annotations/split/test.json
index 41ac06ee..9d7195d0 100644
--- a/data/reference_annotations_split/test.json
+++ b/src/sec_certs/data/reference_annotations/split/test.json
@@ -1,780 +1,713 @@
 [
-    "11a98c5d5ff6f40f",
-    "6dde54b8c6d592ef",
-    "054bd8196e015197",
-    "c902a788d699fe4a",
-    "5d674c06a559a5e7",
-    "9f8c9a3aa64a9f8e",
-    "9545f2b0b8b92c8d",
-    "56ceb1a9c003ac3f",
-    "9299f6cd8a266541",
-    "d1fe94ee32cbbf05",
-    "6313a92bf426d2bb",
-    "e8a37e48a8d34410",
-    "f34df0dd7011366e",
-    "c23218b044706800",
-    "3f4b6e4f245f6fab",
-    "5f99b00bf258f32f",
-    "4f3bf0c433b7f54f",
-    "587055293e9fa51d",
-    "da2d3567a814d2d7",
-    "915e919390e6f58b",
-    "a5e8267322d0d7a6",
-    "82dce1546c69369d",
-    "7d4585a4b5b6e873",
-    "736ede0289146b59",
+    "0160f6becf137107",
+    "05822f92f304fcd6",
+    "06998c36415036aa",
+    "08bff656ff61e949",
+    "0a6525cdbeb38df6",
     "0cbcee9f1cde47a8",
-    "4bf7d6ef245dd393",
-    "3817c0aca007989e",
-    "baead73ecc6e4a33",
+    "0f7ce013d15f6ff5",
+    "10229cdd92c1989f",
+    "13d2556ef392ed41",
+    "15ae64b85b3e28d7",
+    "15b121492722bffb",
+    "15ee7651b7ad610c",
+    "15fae84c1e6c6c57",
+    "16513a1bff79b46a",
+    "16b0a0811bc4fe82",
+    "16b24ff1bf3c079b",
+    "16f1a518cd842fcb",
+    "173097995b7a7f12",
+    "173704f0d2b8a02f",
+    "177dee4c1051e612",
+    "17bfc3d0570ab3d0",
+    "183a5cd3e2aaa0ec",
+    "18a515a0fff0c0c3",
+    "18b7e5bdf459ca13",
+    "195366bfba7213a5",
+    "1999d4ed82b3188e",
     "19d3185c40be008d",
-    "8a7731e9eb363fb4",
-    "78a279b07a6d3367",
-    "47913a485c3c8a18",
-    "5f259ce91f0234a0",
-    "d4a14ea7adb375ed",
-    "994a2bf994b4facd",
-    "ac1257069b2f4afd",
-    "b5985efd378af972",
-    "bc1452fff95744b9",
-    "4f953ae00a24eb85",
-    "a35ed74b7ea3ec4b",
-    "654554952ecb20ac",
-    "157b741a0f529a0a",
-    "30ae700d801bd41c",
-    "2f9feaf4121720da",
-    "e0283686d26420d9",
+    "1ac8de53c0894d18",
     "1b2764a62ffe86a2",
-    "b47b457f73c42c60",
-    "ff45605151ec4eb1",
-    "5346d5a4418466ab",
-    "e5b04a0a3872954f",
-    "981f7ea91d7bdcb6",
-    "f229a8078cb4ff82",
-    "731aac44217611ec",
-    "4e29916c366fe997",
-    "5efe98a1ba4df4d7",
-    "ffd213e55eee8cc8",
-    "7e24168630d5124e",
-    "00ad121fa63e5dfb",
-    "173097995b7a7f12",
-    "4b2f963b48e0f954",
-    "5e4c757231135ba1",
-    "aed7a612d966a786",
-    "1468948b4f0509ab",
-    "4d885e54f1361d04",
-    "8a5f8686ab9de19e",
-    "37d28734245a3bb1",
-    "80fe3ba3d9c9bf64",
-    "b1b385d5bad724ed",
-    "c4ca6565698f0e49",
-    "d9bfffa3cc6d1c53",
-    "d32e5d4d5ff8b871",
-    "8865aed38c79d3fc",
-    "5bf118ffa29520c0",
-    "6136acacc653106c",
-    "05d96aada7bb839c",
-    "0aad0176e35b3749",
-    "64ed7e55bdc50177",
-    "3976c9e492193315",
-    "65fba51313f65938",
-    "55e52bcaab69d573",
-    "4dd0daec78093d31",
-    "d67edade78d25033",
-    "71eae9be60b7fb6c",
-    "3d149fe7c08bfc58",
+    "1bcfaefe46abccf0",
+    "1c8567a1b1a6c12a",
+    "1cc05dbb992431b9",
+    "1d1df0fb541e49b8",
     "1d9ae732c5dec242",
-    "6bdca9f13e19572e",
-    "d6ade0a63d0f9130",
-    "da1139ecdc3512da",
-    "637657613004dbbd",
+    "1ea31bb5f6a15995",
+    "1ff08f79cb89c1de",
+    "2078424be58e4db1",
+    "21951e191e55b66e",
+    "21cffadbbb87c205",
+    "22ede463dbf1a105",
+    "23a85a7f9c07412f",
+    "24624a4e60ddedd4",
+    "259df98e7476a843",
+    "259e0ab1c4b690cf",
+    "25c07c5d0a05d86c",
+    "26c4912b140fde9f",
+    "26c7b173668167d8",
+    "2760aeedce0b79db",
+    "27ccec0740bb7915",
+    "296618f10b019d9c",
+    "29cc495628e76ffa",
+    "29f2dd90f57ea948",
     "29fb07d8f74e734b",
-    "075ddd00e07fa7ed",
-    "ce92195cf4a19084",
-    "68875c05e00bd1aa",
-    "61e862ed07d3cb03",
-    "cdbbbad526f0aa13",
-    "1d1df0fb541e49b8",
-    "5d4aae047020163b",
-    "c67724216ed5a318",
-    "10229cdd92c1989f",
-    "4dbb108406acb1a5",
-    "9292ab2134e8a712",
-    "84501b4439b66fd9",
+    "2a07b85f61ac08b0",
+    "2a91f42389fda90d",
+    "2adb03f5294c31c2",
+    "2b0342d5b70b7a78",
+    "2bd9661657e78278",
+    "2be10f342e68a89e",
+    "2c41f9dfcd5014e6",
+    "2d046178aa118fff",
+    "2d0b4100c3ead88a",
+    "2e997ba7beaed34d",
+    "2eee0fdd5cdaf565",
+    "2f9feaf4121720da",
+    "30ae700d801bd41c",
+    "30c6777901ae346c",
+    "30e9ce0969941ffb",
+    "3183669bf78db43b",
+    "31fd07069aebf013",
+    "328fee1e52f7ac82",
+    "3334d22b6559d0e2",
+    "337ece90615ed69d",
+    "3392558f04e04663",
+    "33cada8c95bf55c5",
+    "34506b8fab6dd7c3",
+    "34ed76f0c32d9e93",
+    "3519c14e4114d93d",
+    "35b9be2203d2b9e6",
     "35efe13fa9e93a68",
-    "5c5431cca026fb3e",
-    "e3a743ef04ecfb64",
-    "89d1f7ab8b77e41d",
-    "70908a4cfdc5585c",
-    "8035f242b12cec6d",
-    "67240adc18be338d",
+    "36641ce7192b5f92",
+    "36df04ab9c978ab9",
+    "373e7b1bc9066563",
+    "3746512864b941d4",
+    "37b94c08f8e25249",
+    "37c8d23c44b95833",
+    "37d28734245a3bb1",
+    "37e0d3cb098458d0",
+    "37fe6036c2ac932b",
+    "3817c0aca007989e",
+    "384dca297bd4f1ca",
+    "3976c9e492193315",
+    "39c07ebbca541145",
+    "3a3b0a9113835307",
+    "3a6a5f536bfbd4d4",
+    "3ba9f330ce93636b",
+    "3d149fe7c08bfc58",
+    "3d756c83419bba28",
+    "3d8083b1e6c7b336",
     "3e08a27e9d9c9b1e",
-    "5eb91b620bbf5bfc",
-    "7e92dcac20e009a5",
+    "3e182053b03f1faf",
+    "3e345d3c6002ad55",
+    "3ea02a62856c752a",
+    "3f188bbf2af01b25",
+    "3f22bd3eaef5d64d",
+    "3f4b6e4f245f6fab",
+    "3f8475de6ea558ac",
+    "3feda0b8b5637540",
+    "4057f5dbe4ffa4cd",
+    "40f62cd468a546cc",
+    "40fa0a4cb5193977",
+    "40fc6ad0aed92913",
+    "41bd2924e9afced5",
+    "420876ec1e5ba657",
+    "424f6d496746258b",
+    "4285d9b580f8a2a6",
+    "42bc15eb26cf2eec",
+    "42d2e68c29d9eef7",
+    "42f53e51476f1a3c",
+    "438b59086f6ebd64",
+    "43a4ca62d0c0b0da",
+    "43d9cb9d3fe30ff2",
+    "449c74a92ebb61a4",
+    "44becd7e128f4ba0",
+    "45098872448f5816",
+    "46459cfb9c1045d8",
+    "4750f5114dcaa60d",
+    "47913a485c3c8a18",
+    "4832a44c0df0bad2",
+    "4871a084d74a4bc0",
+    "4893140ae5daadaf",
+    "48a41f3f5110db1f",
+    "48d897f17754c7ce",
+    "48fd3d84b1a0bc68",
+    "491c766e35ae9f6e",
     "49e6665ee56d3bfc",
+    "4abc83fee59586b2",
+    "4b2f963b48e0f954",
+    "4b5521f85ab3fff7",
+    "4bf7d6ef245dd393",
+    "4c0e2794cb729825",
+    "4c90a96fb1ee09f6",
+    "4d21561351ecd00a",
+    "4d2a177384b23fd6",
+    "4d885e54f1361d04",
+    "4dbb108406acb1a5",
+    "4dc023ea2e3c4115",
+    "4dd0daec78093d31",
+    "4e29916c366fe997",
+    "4eb33712d4b16f43",
+    "4f3bf0c433b7f54f",
+    "4f8db13958be0f75",
+    "4f953ae00a24eb85",
+    "5103648897ba1b3a",
+    "5162627872566ada",
+    "51768783ab3e67f3",
+    "51a4bea9a77a55cf",
+    "51c589803c349e44",
+    "5346d5a4418466ab",
+    "53bba35f7ea6edc9",
+    "53fe111411edfa45",
+    "541da571bac5e279",
+    "5423483cd3c3b75a",
+    "5469d2b3e5fb8323",
+    "5498bfcecf19dfa3",
+    "54f754dc95137c47",
     "54fada9909a1edb3",
-    "8ae9a7fbbb50b6ed",
+    "5506b5232076d409",
+    "55669bcb3a9e09c0",
+    "55925bc39df20912",
+    "559e0907fef71cf6",
+    "55e52bcaab69d573",
+    "55e85aac6594c2fc",
+    "55f9f2fedfc7f069",
+    "568c473e6e5e95c7",
+    "56a56d39dbfe3516",
+    "56bca69ce012d4c1",
+    "56ceb1a9c003ac3f",
+    "56ebfd7a918891c6",
+    "5787ff8207fd751d",
+    "57e108dcd6ee072b",
+    "57e929939f1fd1c0",
+    "583c12b5bf2423c6",
+    "58688dfc2c59d3ab",
+    "587055293e9fa51d",
+    "58ece75f7f2b5099",
+    "58ef8ca403038960",
+    "596a0fd33f70da59",
+    "59820117bb7fa240",
+    "598e2a9978d3c79c",
+    "59b026d889aa0309",
+    "5a1764ef77908457",
+    "5b1b55f1997a9439",
+    "5b67a83de2aaa93a",
+    "5bf118ffa29520c0",
+    "5bfbc4a6cbab9eb1",
+    "5c5431cca026fb3e",
+    "5caf1c90ec7001b7",
+    "5cbd46653c3bea1a",
+    "5cec11774890d61e",
+    "5cfe68ed449e0478",
+    "5d4aae047020163b",
+    "5d674c06a559a5e7",
+    "5e4c757231135ba1",
+    "5e4e16da9d0630dd",
+    "5eb91b620bbf5bfc",
+    "5eda77c801e2df86",
+    "5edefd19948ad581",
+    "5ee9898421c06859",
+    "5efe98a1ba4df4d7",
+    "5f259ce91f0234a0",
+    "5f4cd1504f8b18e3",
+    "5f84cd800ee96468",
+    "5f99b00bf258f32f",
+    "60ec8738947badb0",
+    "60f08c85a518de1c",
+    "60f0dd83c8f32b8c",
+    "6136acacc653106c",
     "61e7ec1c62704f39",
-    "f3bedaa5ffeeadb1",
-    "a3d152e25f8ce6c9",
-    "de2ce0c1b72d5c90",
-    "92ae986997c1d45c",
-    "29cc495628e76ffa",
-    "83bb8420c3cccf85",
-    "ddd9691cb063b07d",
+    "61e862ed07d3cb03",
+    "621335449637e335",
+    "62313881916550ca",
+    "62eb991153211de6",
+    "6313a92bf426d2bb",
+    "6337d3dcc5362dc6",
+    "637657613004dbbd",
+    "63e55780fa1766de",
+    "64e80c255d51ff37",
+    "64ed7e55bdc50177",
+    "6529877d9799822a",
+    "654554952ecb20ac",
+    "6587b26878da24b5",
+    "65a47f93451f13de",
+    "65af80e5c6538215",
+    "65fba51313f65938",
+    "6700028a309c3f7f",
+    "67240adc18be338d",
+    "67e75c6c6538110e",
+    "68875c05e00bd1aa",
+    "691b730a7bbb28b0",
+    "692e91451741ef49",
     "693325ca20a6325b",
-    "37b94c08f8e25249",
-    "26c7b173668167d8",
-    "60ec8738947badb0",
-    "37fe6036c2ac932b",
-    "d69309b0a27c929c",
-    "01e5c6d31f763f56",
-    "adc6971cdb0613b0",
-    "ffc97a8fff838fa8",
-    "9ac6a8bda4fb7dc4",
-    "e247b50a301ce26c",
-    "e359ab2e436adf36",
-    "723ba544431f7874",
-    "fff289a66cb9ce4b",
-    "583c12b5bf2423c6",
-    "98a578f7758bb518",
-    "10233188161519ee",
-    "123c94418ddf1ad8",
-    "0bd651fcbb86f2a9",
+    "696c193b3d13924e",
+    "6999db84bd629b32",
+    "69b877d99c360f97",
     "6a46ff1d1a5154b0",
-    "a5898123ec124b34",
+    "6abef2102c86c0fc",
+    "6ae356d8abbfdeb9",
+    "6b605ff743c859d0",
+    "6bdca9f13e19572e",
+    "6becc4c21730d391",
+    "6c2fc8308efeccb4",
+    "6c3e0fe95400f1f5",
+    "6cef8442092da1da",
+    "6d9c2bab8cc8d28c",
+    "6dde54b8c6d592ef",
+    "6df8f86449e56c05",
+    "6dfa24c82e89436b",
+    "6e9496b7254e02e2",
+    "6eacef3b2f976ce9",
+    "6eeed6fefb1f0243",
+    "6ef82dead8e3437f",
+    "6f13e8b708b43383",
+    "6f480809fc73b635",
+    "6f90b0bf6bbe884b",
+    "6fd73557577cf9ed",
     "6fe5828936663aed",
-    "5506b5232076d409",
-    "1ea31bb5f6a15995",
-    "45098872448f5816",
-    "112fe9f123e277ee",
-    "09220515a7edf3f9",
-    "114e39b4da949e6d",
-    "621335449637e335",
-    "b82b48b0e915aaac",
-    "db9267bdc717c801",
-    "195366bfba7213a5",
-    "177dee4c1051e612",
-    "dcd67613012044b3",
-    "60f0dd83c8f32b8c",
-    "bff1330c13ccd99b",
-    "c06c7eb3afa4aea0",
-    "9ae7edbd0444982d",
-    "ec21d3c76c5ef65d",
-    "449c74a92ebb61a4",
-    "23a85a7f9c07412f",
+    "6ff89f3123a6a98f",
+    "70908a4cfdc5585c",
+    "7096b882b315807a",
+    "70f75a14390522b6",
+    "711a7b593ed41b02",
+    "71d3720cad53b289",
+    "71eae9be60b7fb6c",
+    "723ba544431f7874",
+    "731aac44217611ec",
+    "736ede0289146b59",
+    "7386d04b20f354f4",
+    "73dbc35c2b91c4fd",
+    "743318f3a2b4e281",
+    "74d685c3723bdf92",
+    "75609423e1f40c07",
+    "75c08d873d0dcc06",
+    "75ce95f62fc65fac",
+    "76d52f916b4222b1",
+    "781c243d0021d4c5",
+    "78a279b07a6d3367",
+    "78ef46252d580af0",
+    "793075287bf70a81",
     "7942e835fb18c4ac",
-    "51c589803c349e44",
-    "f577607298c4b574",
-    "8944c4f14bbcaa74",
-    "f1a73b2232787b26",
-    "96c904c81aa2a241",
-    "0ef56cba2362e463",
-    "b965c3b6a03aefc4",
-    "35b9be2203d2b9e6",
-    "da6845ee99cfa275",
-    "977b04d04def60a7",
+    "797ea41cc2e95f08",
+    "7980d204dbc3ecee",
+    "7a059d2754e088a3",
+    "7a26d92281df3e8e",
+    "7a5a4a9578503a55",
+    "7b63c31cfd0eb65f",
+    "7c04881c9a46b337",
+    "7c708f62e3280ec3",
+    "7c7e48e3aed8bfd4",
     "7d3546a4000b5b75",
+    "7d4585a4b5b6e873",
+    "7dfcefd1ffeaa941",
+    "7e24168630d5124e",
+    "7e37ff763a97bc72",
+    "7e8763f553e43d12",
+    "7e92dcac20e009a5",
+    "7e9fc9f948e5cb17",
+    "7f02a580f5219377",
+    "8035f242b12cec6d",
+    "808de32aa819821d",
+    "80fe3ba3d9c9bf64",
+    "819d92a6492415d3",
     "82a2bd8fe4ed2087",
-    "135ca3602f0005bf",
-    "95774db1eb19b767",
+    "82dce1546c69369d",
     "83020f91ce1aaf68",
-    "950ba1c7368c397f",
-    "98642c6f99438194",
-    "b7209eec8fe7fc58",
-    "b21be5dd6fdec83b",
-    "60f08c85a518de1c",
-    "62eb991153211de6",
-    "f9d9fe0dd9f7f110",
-    "017f3cea8275a3e1",
-    "b644b78aafc367b0",
-    "808de32aa819821d",
-    "cc99b5d3806b1e97",
-    "d07b6e3036abb8f1",
-    "db28c1d7f201af1a",
-    "997281c99ac93b3b",
-    "a596f2412c0f8fe2",
-    "dc51cf4e51f01086",
-    "692e91451741ef49",
-    "d3d257648d4c3105",
-    "d6d129c94c368155",
-    "f9a3871824bcd47a",
-    "d063a93beba2dd65",
-    "e4dc90de190437b8",
-    "3feda0b8b5637540",
-    "8da3e1e348d36e07",
-    "d905d620783d3768",
-    "bb509db67c18bc40",
-    "902dbd430adb77a5",
-    "cb9a110799bc37f8",
-    "41bd2924e9afced5",
-    "a2cbd066271b6b3b",
-    "6587b26878da24b5",
-    "5498bfcecf19dfa3",
-    "2e997ba7beaed34d",
-    "0208e6b60a0b1a32",
-    "17bfc3d0570ab3d0",
-    "c459c68e95d54f87",
-    "0a6525cdbeb38df6",
-    "eac2041efb4cf9f7",
-    "a9f542d8e2be69fd",
-    "be9ca77bec616fe5",
-    "09f539aedefa9756",
+    "835211383d281d89",
+    "83b4d7d35a747d1b",
+    "83bb8420c3cccf85",
+    "83c1ee8ab43ea00a",
+    "84444ea4fabcc66c",
+    "84501b4439b66fd9",
+    "84b1520706c3d784",
+    "84c8bf1f809380bc",
+    "84ff17001564ea47",
+    "85fa17a19251a2ea",
+    "86462bcec4492bfa",
     "86e9da10bc176267",
-    "ad2351479ff6ee84",
-    "6b605ff743c859d0",
-    "59820117bb7fa240",
-    "f5a53fa6d2c3d9b2",
-    "9cc6c08cbaa694f2",
-    "5ee9898421c06859",
-    "93f14cad1cb201fa",
-    "183a5cd3e2aaa0ec",
-    "53bba35f7ea6edc9",
-    "6eeed6fefb1f0243",
+    "86f1afdbbd29e48c",
+    "871066e9e2eaba52",
+    "8782c7c292ef2759",
+    "878af76320373ba0",
+    "87a80325171d8add",
+    "87abd998f3b7e26f",
+    "8817adf6eaf4f06c",
+    "8820852f4a1163b3",
+    "8825e2c99223c084",
+    "882a9d016c7e1f36",
+    "8865aed38c79d3fc",
+    "8870ddf70bf57890",
+    "8944c4f14bbcaa74",
+    "89d1f7ab8b77e41d",
+    "8a5f8686ab9de19e",
+    "8a7731e9eb363fb4",
+    "8ade0910a89717a8",
+    "8ae9a7fbbb50b6ed",
     "8b0204e041dd3eb8",
-    "7386d04b20f354f4",
-    "a57845715134651a",
-    "2a91f42389fda90d",
-    "f7cbb33cc639b5b5",
-    "b5b8bd0cb8bb7658",
-    "86462bcec4492bfa",
-    "75c08d873d0dcc06",
-    "6d9c2bab8cc8d28c",
-    "102bbdfcd696d7a9",
-    "bbc9a3a1d685d5f1",
-    "ce22219c2049985c",
-    "e2a45ce49bb2b731",
-    "21cffadbbb87c205",
-    "b4768c31e98b2170",
-    "16b0a0811bc4fe82",
-    "0ee132dec087b772",
+    "8b7145c13f36c324",
+    "8bbbd89d09bb82ab",
     "8bc3658bc958f81f",
-    "18b7e5bdf459ca13",
-    "f2516946c797b1a0",
-    "e56d5d27cb2b3e6f",
-    "0a6e50445a906052",
-    "33cada8c95bf55c5",
-    "ecee66b37d7576db",
-    "bc25a76284e7c7ce",
-    "e9c5d02aae54cdc5",
-    "b3855f3a6c06c01c",
-    "b24a14935edd51ad",
-    "69b877d99c360f97",
-    "f731aaaab07bdca2",
-    "7980d204dbc3ecee",
     "8c6c4b4606608dfe",
-    "6e9496b7254e02e2",
-    "aadbb9992a672a1b",
-    "ca1fa7c3f26bb3c5",
-    "27ccec0740bb7915",
-    "ee4e169753085913",
-    "ea4f0c0a7b82f526",
-    "16f1a518cd842fcb",
-    "83c1ee8ab43ea00a",
-    "76d52f916b4222b1",
-    "4d2a177384b23fd6",
-    "43a4ca62d0c0b0da",
-    "e23f9c02819688f6",
-    "16b24ff1bf3c079b",
-    "cd72fc2b62780a43",
-    "6eacef3b2f976ce9",
-    "fbdaf22c4f50391c",
-    "107c348f4ba0a41c",
-    "fbe41655a05a01ff",
-    "39c07ebbca541145",
-    "173704f0d2b8a02f",
-    "096b05114ee5e09f",
-    "a289b9d43310182b",
-    "cc6918e95b803734",
-    "5f84cd800ee96468",
-    "07ce7e4844f62c5c",
-    "22ede463dbf1a105",
+    "8cfcf8dc64485a74",
+    "8cfd0c9f4bcd21b8",
+    "8d2c50e41a1a0061",
+    "8d68ee5c283ca6b0",
+    "8d92202f35f15712",
+    "8da3e1e348d36e07",
+    "8dcdbce3262bfcd6",
+    "8e713eb632acc712",
     "8ee7145a1b48b578",
-    "85fa17a19251a2ea",
-    "58ece75f7f2b5099",
-    "420876ec1e5ba657",
-    "2d0b4100c3ead88a",
-    "42f53e51476f1a3c",
+    "8f3037c89c498e88",
+    "8f7cc0f456186028",
+    "8f7e22207cb3dd8b",
+    "8f81bb35e37f1b4c",
+    "902dbd430adb77a5",
+    "91108cfcecb98297",
+    "915e919390e6f58b",
+    "921ecc7c3e5f4884",
+    "9231adbf747903df",
+    "925da3b468003e72",
+    "9292ab2134e8a712",
+    "9299f6cd8a266541",
+    "92ae986997c1d45c",
+    "92c7355f5ea28e52",
+    "932cef2b9698dde8",
+    "9337ab9573b366c1",
+    "93f14cad1cb201fa",
+    "940a18f62c79ffae",
+    "950ba1c7368c397f",
+    "953e132dd43ec03f",
+    "9545f2b0b8b92c8d",
+    "95774db1eb19b767",
+    "95c52fa37ef1eb3b",
+    "96106c2d31eea8b2",
+    "966a03f9f111aa35",
+    "96b0895b9f712d2f",
     "96c685a3cdd9ed59",
-    "eacc2cf0173eb438",
+    "96c904c81aa2a241",
+    "975dae6e02765a90",
+    "977b04d04def60a7",
+    "97eb8171b51f9aad",
+    "97f302708066dc04",
+    "981f7ea91d7bdcb6",
+    "983d431922822165",
+    "98642c6f99438194",
+    "988394e53c8e52d9",
+    "98a578f7758bb518",
     "9938be53082a02e6",
-    "d75ff490c057793c",
-    "7dfcefd1ffeaa941",
-    "131ec924c42ef90b",
-    "d3323745a51a604d",
-    "2eee0fdd5cdaf565",
-    "bdf8d5624e230986",
-    "7096b882b315807a",
-    "84b1520706c3d784",
-    "b49efd086851f84d",
-    "e56e14e8cbf89e3c",
-    "3183669bf78db43b",
-    "c28289a45fc012a6",
+    "994a2bf994b4facd",
+    "996691f0e03e303d",
+    "997281c99ac93b3b",
+    "9973eb31c46f8637",
+    "9ac6a8bda4fb7dc4",
+    "9ae7edbd0444982d",
+    "9b30bcdbccfd4c68",
+    "9ba1fa1ae8c3ae67",
+    "9be0c86102117436",
+    "9be76c10474e0c80",
+    "9cc6c08cbaa694f2",
+    "9d36f8ab4e1b45dc",
+    "9e55246befe96fb4",
+    "9e7c513fdd35df17",
+    "9f400f89490fb8d5",
+    "9f8c9a3aa64a9f8e",
+    "a00e93f2981ef418",
+    "a0621c7e36e9ecb1",
+    "a0fff0700364df49",
     "a14f221cf1761a8c",
-    "e7a197d6aefb66df",
-    "0349c893b81f7c1a",
-    "84ff17001564ea47",
-    "b1401488f998c1f0",
-    "e367ecaf05dacedb",
-    "6f90b0bf6bbe884b",
-    "4abc83fee59586b2",
-    "e64266a9c13fc74e",
-    "37c8d23c44b95833",
-    "5edefd19948ad581",
-    "932cef2b9698dde8",
-    "158950ed4bc35274",
-    "4dc023ea2e3c4115",
-    "8bbbd89d09bb82ab",
+    "a191b2da409f3518",
+    "a289b9d43310182b",
+    "a29d46ab6efee468",
+    "a2a5ccdf0846ce67",
+    "a2cbd066271b6b3b",
+    "a334142059d865a2",
+    "a35ed74b7ea3ec4b",
+    "a3d152e25f8ce6c9",
+    "a465a767ceb7fd30",
+    "a49706f436a6873d",
+    "a4d0e44f4527180f",
+    "a5451ba38050a44d",
+    "a57845715134651a",
+    "a5898123ec124b34",
+    "a596f2412c0f8fe2",
+    "a5e8267322d0d7a6",
+    "a64e6bf63ab6a299",
+    "a67919286833a7df",
+    "a6c1dd30884a197c",
+    "a76054a284f0ce61",
+    "a851c3b4377f4923",
+    "a9004216663480de",
+    "a925d8f1adc5cfe2",
+    "a93edea77d9f3338",
+    "a9f542d8e2be69fd",
+    "aa0213dd9f727219",
+    "aa3f466daa34d3ff",
+    "aadbb9992a672a1b",
+    "ab769495f97aab9c",
+    "ab88698a0c6ee1bb",
+    "ab8cf6af80a5c0e0",
+    "abef2b316c54f9e7",
+    "ac1257069b2f4afd",
     "ac4687f4b15507b6",
-    "3392558f04e04663",
-    "64e80c255d51ff37",
-    "78ef46252d580af0",
-    "4b5521f85ab3fff7",
-    "6c3e0fe95400f1f5",
-    "988394e53c8e52d9",
-    "983d431922822165",
-    "0319b03323f257d1",
-    "3a3b0a9113835307",
-    "91108cfcecb98297",
-    "b883e389d34b1bb1",
+    "acee5f619f09bf79",
+    "ad2351479ff6ee84",
+    "adb678f7143014a0",
+    "adc6971cdb0613b0",
+    "adc9df2489ce090a",
+    "ae175aa839dbf692",
+    "ae9574428bd713e6",
+    "aeb79c7e573de0e5",
+    "aed7a612d966a786",
+    "af373587f2c57a24",
+    "af4c9b6e93062fde",
     "af58ba642f4fc3d2",
+    "af78198822da03a7",
+    "afcf1165734847de",
+    "afe441f40a42865e",
     "b0fd694500bfffab",
-    "d7501476836cf315",
-    "1cc05dbb992431b9",
-    "d4a1feebc1e1cb5d",
-    "ea316d47c03f9fdf",
-    "781c243d0021d4c5",
-    "aa3f466daa34d3ff",
-    "568c473e6e5e95c7",
-    "cdecc0eedcf05d2c",
-    "d3987535a95fbffd",
-    "f4da9e13977580f4",
-    "c483baec90d76587",
-    "6becc4c21730d391",
-    "7c04881c9a46b337",
-    "a93edea77d9f3338",
-    "c1e14f36f031a342",
     "b12b4e4eacad1497",
-    "fd583001a87023fd",
-    "63e55780fa1766de",
-    "58ef8ca403038960",
-    "b9a6922899b66b8b",
-    "0ff82b1cc24b3105",
-    "0c9e215997d96f44",
-    "f0bd6a29eee94a2c",
-    "4f8db13958be0f75",
-    "c4f5a7748428e7d1",
-    "adc9df2489ce090a",
-    "e3231fac88242d81",
-    "c62b878b1ac43df9",
-    "a6c1dd30884a197c",
-    "5469d2b3e5fb8323",
-    "f588fbe6fd36bc6b",
-    "04c5a5a66fb7f89a",
-    "34ed76f0c32d9e93",
-    "6dfa24c82e89436b",
-    "d12a50d33b1c4253",
-    "ab88698a0c6ee1bb",
-    "13e5b88ad3becb77",
-    "cce214002aca738f",
-    "b6c3a9d53b9e784a",
-    "b1b04d862f4c32af",
-    "dbbb56b48acbaf22",
-    "adb678f7143014a0",
-    "e0999b85d2f77733",
-    "438b59086f6ebd64",
-    "6700028a309c3f7f",
-    "9337ab9573b366c1",
-    "4c0e2794cb729825",
-    "6f480809fc73b635",
-    "2760aeedce0b79db",
-    "bec58e970e8cd6c8",
-    "d3420c2bf8974a5d",
-    "57e929939f1fd1c0",
-    "65a47f93451f13de",
+    "b1401488f998c1f0",
+    "b15c099c68671895",
     "b168db2651cbc1d6",
-    "75609423e1f40c07",
-    "0d4a95a8b52b787b",
-    "296618f10b019d9c",
-    "fce7e01f2cd0cac0",
-    "36641ce7192b5f92",
-    "4d21561351ecd00a",
-    "06b24bc51eb69188",
-    "dbe20d7c305b24eb",
-    "dbcf1311b3d95cdc",
-    "51a4bea9a77a55cf",
-    "9f400f89490fb8d5",
-    "55669bcb3a9e09c0",
-    "6337d3dcc5362dc6",
-    "55f9f2fedfc7f069",
-    "3d756c83419bba28",
-    "8f3037c89c498e88",
-    "2be10f342e68a89e",
-    "e501cafa3b025fb9",
-    "42bc15eb26cf2eec",
-    "05822f92f304fcd6",
-    "44becd7e128f4ba0",
-    "696c193b3d13924e",
-    "c5bc8b961a199646",
-    "2a07b85f61ac08b0",
-    "5cfe68ed449e0478",
-    "02c6fbcd90dcf05a",
-    "4285d9b580f8a2a6",
-    "16513a1bff79b46a",
-    "a334142059d865a2",
-    "a67919286833a7df",
-    "2078424be58e4db1",
-    "0f684159ad31f883",
-    "a76054a284f0ce61",
-    "8870ddf70bf57890",
-    "15b121492722bffb",
-    "a781ac2579798523",
-    "bf1338e9abe85c39",
-    "86f1afdbbd29e48c",
-    "afcf1165734847de",
-    "f569fb2a6ddae452",
-    "882a9d016c7e1f36",
+    "b1b04d862f4c32af",
+    "b1b385d5bad724ed",
+    "b21be5dd6fdec83b",
+    "b2436dcf1c8e5dc5",
+    "b24a14935edd51ad",
+    "b252ff42ce0b7443",
+    "b2ba4d07e03b47ea",
+    "b3855f3a6c06c01c",
+    "b4768c31e98b2170",
+    "b47b457f73c42c60",
+    "b49efd086851f84d",
+    "b4fc6a7b6c200b1b",
     "b584e424a8b3dabe",
-    "d80c26c2484dc87d",
-    "30e9ce0969941ffb",
-    "8cfcf8dc64485a74",
-    "d1df1bf926a69ca9",
-    "f621ca835cf98a6d",
-    "3519c14e4114d93d",
-    "db28c203846775d8",
-    "2d046178aa118fff",
-    "40fc6ad0aed92913",
-    "48d897f17754c7ce",
     "b5968cd571a14cf3",
-    "31fd07069aebf013",
-    "c18dc9f29967fe7f",
-    "afe441f40a42865e",
-    "f8a3fde3a557f44d",
-    "c8e381408b191a02",
-    "f9e9366287e79e4d",
-    "cc64f3b8666b17bf",
-    "e535b7a4343399e7",
-    "01ce6f422bec823d",
-    "e64eedbb40d09a50",
-    "ca88a34b0fcc5b0e",
-    "bd346dd3e46dff68",
-    "966a03f9f111aa35",
+    "b5985efd378af972",
+    "b5b8bd0cb8bb7658",
+    "b5b8de155e4e40e6",
+    "b644b78aafc367b0",
+    "b6a96b0dcf6eadf0",
+    "b6c15035f759ec50",
+    "b6c3a9d53b9e784a",
+    "b6e60b9025d116e4",
+    "b7132fe23fbdf9e1",
+    "b7209eec8fe7fc58",
+    "b764f5d5391e431e",
+    "b804c318fc0c1e9b",
+    "b82b48b0e915aaac",
+    "b883e389d34b1bb1",
+    "b887b7dcd6f1fdbb",
     "b95f2b009d70fad1",
-    "337ece90615ed69d",
-    "f14484e630df1dec",
-    "10af4a6378d0fb7a",
-    "5cec11774890d61e",
+    "b965c3b6a03aefc4",
     "b99c572f58cfb19e",
-    "b6c15035f759ec50",
-    "975dae6e02765a90",
-    "c96ac4c4015414ad",
-    "ea1b5ef156502d98",
-    "082380caaa153ee6",
-    "1ff08f79cb89c1de",
-    "87abd998f3b7e26f",
-    "13d2556ef392ed41",
-    "00a2b91e5b58cef1",
-    "cc5a3d2a52f99ab3",
-    "7a5a4a9578503a55",
-    "59b026d889aa0309",
-    "157f811757a0b29e",
-    "8825e2c99223c084",
-    "ed511caf8988dc45",
-    "04cb360502da6492",
-    "d317c9c2e53ce427",
-    "f9fb8170cecedc23",
-    "e3129aad84d5c40c",
+    "b9a6922899b66b8b",
+    "ba521df2c728c5a2",
+    "baead73ecc6e4a33",
+    "bb02df07a0d95e77",
     "bb21cecb8bbfdf63",
-    "46459cfb9c1045d8",
-    "102707a8c4d8a1b0",
-    "2bd9661657e78278",
-    "a191b2da409f3518",
-    "62313881916550ca",
-    "e5863682239df24b",
+    "bb509db67c18bc40",
+    "bbc9a3a1d685d5f1",
+    "bc1452fff95744b9",
+    "bc25a76284e7c7ce",
+    "bc715a45d7584d95",
+    "bd346dd3e46dff68",
+    "bdf8d5624e230986",
+    "be080cda1829826e",
+    "be9ca77bec616fe5",
+    "bec58e970e8cd6c8",
+    "bf0f48ffd0dff903",
+    "bf1338e9abe85c39",
+    "bff1330c13ccd99b",
+    "c06c7eb3afa4aea0",
+    "c10a3a62bf9fe782",
+    "c16f3dafab92df61",
+    "c1754dc2e7d666d8",
+    "c18dc9f29967fe7f",
+    "c1e14f36f031a342",
+    "c23218b044706800",
+    "c28289a45fc012a6",
+    "c3dc8cfd97735115",
+    "c459c68e95d54f87",
+    "c483baec90d76587",
+    "c497da3fdb2f5027",
+    "c4ca6565698f0e49",
+    "c4e6a655a15144dd",
+    "c4f5a7748428e7d1",
+    "c518dd59d12daf17",
     "c572cacbeb840621",
-    "e93cb94a06c6957e",
+    "c5bc8b961a199646",
+    "c621d1371b314913",
+    "c62b878b1ac43df9",
+    "c67724216ed5a318",
+    "c6c291cd525225b7",
+    "c8e381408b191a02",
+    "c8fc487eec95c21e",
+    "c902a788d699fe4a",
+    "c96ac4c4015414ad",
+    "ca1fa7c3f26bb3c5",
+    "ca63425a882729f3",
+    "ca88a34b0fcc5b0e",
+    "cb9a110799bc37f8",
+    "cbe5be4b63677c25",
+    "cc5a3d2a52f99ab3",
     "cc5e2f58b681e555",
-    "259e0ab1c4b690cf",
-    "fb2cfdfef4f93536",
-    "abef2b316c54f9e7",
-    "7f02a580f5219377",
-    "819d92a6492415d3",
-    "dd4cfd03d9b3f25f",
-    "996691f0e03e303d",
-    "5e4e16da9d0630dd",
-    "5162627872566ada",
-    "a465a767ceb7fd30",
+    "cc64f3b8666b17bf",
+    "cc6918e95b803734",
+    "cc932dbc293df39e",
+    "cc99b5d3806b1e97",
     "cce1a95634c6972b",
-    "7a26d92281df3e8e",
-    "109eb2158ca6a2f9",
-    "743318f3a2b4e281",
-    "06998c36415036aa",
-    "a49706f436a6873d",
-    "75ce95f62fc65fac",
-    "43d9cb9d3fe30ff2",
-    "4871a084d74a4bc0",
-    "8ade0910a89717a8",
-    "00b539d629e60965",
-    "74d685c3723bdf92",
+    "cce214002aca738f",
+    "cd72fc2b62780a43",
+    "cdbbbad526f0aa13",
+    "cdecc0eedcf05d2c",
+    "ce22219c2049985c",
+    "ce92195cf4a19084",
+    "cecb6a5d441fba98",
+    "cf1c544add4bf860",
+    "d063a93beba2dd65",
+    "d07b6e3036abb8f1",
+    "d07cacdb732c0b0f",
     "d08b5ddedd34448d",
-    "6999db84bd629b32",
-    "f7941427d5fa40ea",
-    "92c7355f5ea28e52",
-    "0f7ce013d15f6ff5",
-    "7b63c31cfd0eb65f",
-    "15fae84c1e6c6c57",
-    "4eb33712d4b16f43",
-    "67e75c6c6538110e",
-    "8817adf6eaf4f06c",
-    "97f302708066dc04",
-    "02cb3c385865ad2c",
-    "c6c291cd525225b7",
-    "f2e7c6dfd6a431c8",
-    "8820852f4a1163b3",
-    "ec8a2a25cdbda634",
-    "5bfbc4a6cbab9eb1",
-    "491c766e35ae9f6e",
-    "3f22bd3eaef5d64d",
-    "c497da3fdb2f5027",
-    "b6e60b9025d116e4",
-    "e89a1a7817dae7e4",
-    "15ae64b85b3e28d7",
-    "7e8763f553e43d12",
-    "691b730a7bbb28b0",
-    "8d92202f35f15712",
+    "d12a50d33b1c4253",
+    "d1df1bf926a69ca9",
+    "d1fe94ee32cbbf05",
+    "d2bece2b0eebce5b",
+    "d2d43918e5447c09",
+    "d317c9c2e53ce427",
+    "d32e5d4d5ff8b871",
+    "d3323745a51a604d",
+    "d3420c2bf8974a5d",
     "d34363168b1590bd",
-    "e6aaffd588f2832e",
-    "57e108dcd6ee072b",
-    "6529877d9799822a",
-    "384dca297bd4f1ca",
-    "b764f5d5391e431e",
-    "b5b8de155e4e40e6",
+    "d3987535a95fbffd",
+    "d3c2a59a2cffa30c",
+    "d3d257648d4c3105",
     "d4647f5217d84ba3",
+    "d4a14ea7adb375ed",
+    "d4a1feebc1e1cb5d",
+    "d5187fc1cd590073",
+    "d67edade78d25033",
+    "d69309b0a27c929c",
+    "d6ade0a63d0f9130",
+    "d6d129c94c368155",
+    "d71d0b082233758e",
+    "d7501476836cf315",
+    "d75ff490c057793c",
+    "d80c26c2484dc87d",
+    "d905d620783d3768",
+    "d9bfffa3cc6d1c53",
+    "da1139ecdc3512da",
+    "da2d3567a814d2d7",
+    "da6845ee99cfa275",
+    "db28c1d7f201af1a",
+    "db28c203846775d8",
+    "db9267bdc717c801",
+    "dbbb56b48acbaf22",
+    "dbcf1311b3d95cdc",
     "dbd62ef3bd952bad",
-    "8dcdbce3262bfcd6",
-    "835211383d281d89",
-    "b7132fe23fbdf9e1",
+    "dbe20d7c305b24eb",
+    "dc51cf4e51f01086",
+    "dcd67613012044b3",
+    "dd4cfd03d9b3f25f",
+    "ddd9691cb063b07d",
+    "de2ce0c1b72d5c90",
     "de50abaf95e79d34",
-    "c621d1371b314913",
-    "6ff89f3123a6a98f",
-    "5caf1c90ec7001b7",
-    "596a0fd33f70da59",
-    "40fa0a4cb5193977",
-    "d3c2a59a2cffa30c",
-    "9ba1fa1ae8c3ae67",
-    "598e2a9978d3c79c",
-    "08bff656ff61e949",
-    "0160f6becf137107",
-    "f67f3737a3b47208",
-    "c3dc8cfd97735115",
-    "541da571bac5e279",
-    "55925bc39df20912",
-    "af78198822da03a7",
-    "6f13e8b708b43383",
-    "5787ff8207fd751d",
-    "424f6d496746258b",
-    "2c41f9dfcd5014e6",
-    "e6dd3d435bd1b82e",
-    "a29d46ab6efee468",
-    "f4b4b026e87b5086",
-    "ae175aa839dbf692",
-    "97eb8171b51f9aad",
-    "a2a5ccdf0846ce67",
-    "aeb79c7e573de0e5",
-    "42d2e68c29d9eef7",
-    "f1300cb717ea9335",
-    "9231adbf747903df",
-    "e9845b4128afd538",
+    "e0283686d26420d9",
+    "e0999b85d2f77733",
     "e0be32a08b66d264",
-    "8f7e22207cb3dd8b",
-    "f68006f6c05e1211",
-    "40f62cd468a546cc",
-    "56ebfd7a918891c6",
-    "cf1c544add4bf860",
-    "c1754dc2e7d666d8",
-    "a4d0e44f4527180f",
-    "3a6a5f536bfbd4d4",
-    "d07cacdb732c0b0f",
-    "9973eb31c46f8637",
-    "925da3b468003e72",
-    "7c7e48e3aed8bfd4",
-    "24624a4e60ddedd4",
-    "3e182053b03f1faf",
-    "7a059d2754e088a3",
+    "e23f9c02819688f6",
+    "e247b50a301ce26c",
+    "e2a45ce49bb2b731",
+    "e3129aad84d5c40c",
+    "e3231fac88242d81",
+    "e359ab2e436adf36",
+    "e367ecaf05dacedb",
+    "e3a743ef04ecfb64",
+    "e4dc90de190437b8",
+    "e501cafa3b025fb9",
+    "e535b7a4343399e7",
+    "e56d5d27cb2b3e6f",
+    "e56e14e8cbf89e3c",
+    "e5863682239df24b",
+    "e5b04a0a3872954f",
+    "e64266a9c13fc74e",
+    "e64eedbb40d09a50",
     "e6959c2b66202cb8",
-    "b252ff42ce0b7443",
-    "5f4cd1504f8b18e3",
-    "be080cda1829826e",
-    "0f1b31ac93459d1f",
-    "ea17788d9a9d12eb",
-    "5103648897ba1b3a",
-    "b887b7dcd6f1fdbb",
-    "1494cff85b3dc2bb",
-    "2adb03f5294c31c2",
-    "5cbd46653c3bea1a",
-    "259df98e7476a843",
-    "c4e6a655a15144dd",
-    "9b30bcdbccfd4c68",
-    "84444ea4fabcc66c",
-    "01dc77d7f8f94d97",
-    "3ea02a62856c752a",
+    "e6aaffd588f2832e",
+    "e6dd3d435bd1b82e",
+    "e7a197d6aefb66df",
+    "e89a1a7817dae7e4",
+    "e8a37e48a8d34410",
     "e8adf99d340489ce",
-    "65af80e5c6538215",
-    "a00e93f2981ef418",
+    "e92ba97c2742e63d",
+    "e93cb94a06c6957e",
+    "e9845b4128afd538",
+    "e9c5d02aae54cdc5",
+    "ea17788d9a9d12eb",
+    "ea1b5ef156502d98",
+    "ea316d47c03f9fdf",
+    "ea330145d3ba738d",
+    "ea4f0c0a7b82f526",
+    "eac2041efb4cf9f7",
+    "eacc2cf0173eb438",
+    "ec21d3c76c5ef65d",
+    "ec8a2a25cdbda634",
+    "ecee66b37d7576db",
+    "ed511caf8988dc45",
+    "ee35d872179a1397",
+    "ee3a996c18ace68e",
+    "ee4e169753085913",
+    "ee6718f7a7e3d30b",
     "ef02bc67ebead2a2",
-    "9be76c10474e0c80",
-    "54f754dc95137c47",
-    "a0621c7e36e9ecb1",
-    "797ea41cc2e95f08",
-    "b4fc6a7b6c200b1b",
-    "fca98ecd003e1b82",
-    "a5451ba38050a44d",
-    "58688dfc2c59d3ab",
+    "f0bd6a29eee94a2c",
+    "f1300cb717ea9335",
+    "f14484e630df1dec",
+    "f1a73b2232787b26",
+    "f1b9158cc2b98388",
+    "f229a8078cb4ff82",
     "f2327302df1aa660",
-    "30c6777901ae346c",
-    "6fd73557577cf9ed",
-    "96b0895b9f712d2f",
-    "e92ba97c2742e63d",
-    "fb1fff87c87b82d6",
-    "122245dd15683e26",
-    "34506b8fab6dd7c3",
-    "8cfd0c9f4bcd21b8",
-    "5b1b55f1997a9439",
-    "09b07fbbb24e1b04",
+    "f2516946c797b1a0",
+    "f2e7c6dfd6a431c8",
     "f2eb0ef953f3f147",
-    "7e9fc9f948e5cb17",
-    "4893140ae5daadaf",
     "f2fd56c195593dd8",
-    "953e132dd43ec03f",
-    "1c8567a1b1a6c12a",
-    "48a41f3f5110db1f",
-    "032faef0207b715f",
-    "d2bece2b0eebce5b",
-    "8d2c50e41a1a0061",
-    "a9004216663480de",
-    "8f7cc0f456186028",
-    "10b17081dd7cad8f",
-    "6ae356d8abbfdeb9",
-    "1bcfaefe46abccf0",
-    "8f81bb35e37f1b4c",
-    "51768783ab3e67f3",
-    "b6a96b0dcf6eadf0",
-    "5b67a83de2aaa93a",
-    "4c90a96fb1ee09f6",
-    "cecb6a5d441fba98",
-    "48fd3d84b1a0bc68",
-    "00eb3e5c7a4537b1",
-    "3f188bbf2af01b25",
-    "7e37ff763a97bc72",
-    "3ba9f330ce93636b",
-    "21951e191e55b66e",
-    "ba521df2c728c5a2",
-    "ffe4328dcdc6b538",
-    "ae9574428bd713e6",
-    "ff0ffb9a32873433",
-    "56a56d39dbfe3516",
-    "a925d8f1adc5cfe2",
-    "7c708f62e3280ec3",
-    "aa0213dd9f727219",
-    "3334d22b6559d0e2",
-    "b2436dcf1c8e5dc5",
-    "8d68ee5c283ca6b0",
-    "4057f5dbe4ffa4cd",
-    "18a515a0fff0c0c3",
-    "4832a44c0df0bad2",
-    "c8fc487eec95c21e",
-    "8782c7c292ef2759",
-    "36df04ab9c978ab9",
-    "06e70fa70393d60f",
-    "559e0907fef71cf6",
-    "af373587f2c57a24",
-    "3746512864b941d4",
-    "1999d4ed82b3188e",
-    "793075287bf70a81",
-    "878af76320373ba0",
-    "ca63425a882729f3",
-    "95c52fa37ef1eb3b",
-    "d2d43918e5447c09",
-    "ab769495f97aab9c",
-    "2b0342d5b70b7a78",
-    "ee3a996c18ace68e",
-    "9d36f8ab4e1b45dc",
-    "3d8083b1e6c7b336",
-    "fa8384342c25ec59",
-    "373e7b1bc9066563",
-    "26c4912b140fde9f",
-    "d71d0b082233758e",
-    "9be0c86102117436",
+    "f34df0dd7011366e",
+    "f3bedaa5ffeeadb1",
+    "f4b4b026e87b5086",
+    "f4da9e13977580f4",
+    "f569fb2a6ddae452",
+    "f577607298c4b574",
+    "f588fbe6fd36bc6b",
+    "f5a53fa6d2c3d9b2",
+    "f621ca835cf98a6d",
+    "f67f3737a3b47208",
+    "f68006f6c05e1211",
     "f70d226e1598ac41",
-    "328fee1e52f7ac82",
-    "70f75a14390522b6",
-    "1188aebd8681e4fe",
-    "08408cbc394e7421",
-    "cc932dbc293df39e",
-    "d5187fc1cd590073",
-    "0b0648a9b9a26747",
-    "0cd406e7c5013b74",
-    "25c07c5d0a05d86c",
-    "9e55246befe96fb4",
-    "a0fff0700364df49",
+    "f731aaaab07bdca2",
+    "f7941427d5fa40ea",
+    "f7cbb33cc639b5b5",
+    "f8a3fde3a557f44d",
+    "f96bfb09bb1bd640",
+    "f9a3871824bcd47a",
     "f9d96d0eb2bc85cc",
-    "5eda77c801e2df86",
-    "ab8cf6af80a5c0e0",
-    "73dbc35c2b91c4fd",
-    "8e713eb632acc712",
-    "56bca69ce012d4c1",
-    "bc715a45d7584d95",
-    "bb02df07a0d95e77",
-    "5423483cd3c3b75a",
-    "6df8f86449e56c05",
-    "3e345d3c6002ad55",
-    "29f2dd90f57ea948",
-    "ee6718f7a7e3d30b",
-    "ea330145d3ba738d",
+    "f9d9fe0dd9f7f110",
+    "f9e9366287e79e4d",
+    "f9fb8170cecedc23",
+    "fa8384342c25ec59",
+    "fb1fff87c87b82d6",
+    "fb2cfdfef4f93536",
     "fb8010928a4bb56b",
-    "921ecc7c3e5f4884",
-    "940a18f62c79ffae",
-    "83b4d7d35a747d1b",
-    "b15c099c68671895",
-    "f1b9158cc2b98388",
-    "8b7145c13f36c324",
-    "37e0d3cb098458d0",
-    "080aa6966d757f05",
-    "ee35d872179a1397",
-    "6abef2102c86c0fc",
-    "096ccb6bcb3d2731",
-    "71d3720cad53b289",
-    "15ee7651b7ad610c",
-    "0da9d81c9f563df5",
-    "6ef82dead8e3437f",
-    "fd966f807d3ba9fe",
-    "b804c318fc0c1e9b",
-    "4750f5114dcaa60d",
-    "9e7c513fdd35df17",
-    "b2ba4d07e03b47ea",
-    "cbe5be4b63677c25",
-    "53fe111411edfa45",
-    "87a80325171d8add",
-    "3f8475de6ea558ac",
-    "a64e6bf63ab6a299",
-    "c10a3a62bf9fe782",
-    "711a7b593ed41b02",
-    "5a1764ef77908457",
-    "6c2fc8308efeccb4",
-    "1ac8de53c0894d18",
-    "bf0f48ffd0dff903",
-    "a851c3b4377f4923",
-    "f96bfb09bb1bd640",
+    "fbdaf22c4f50391c",
+    "fbe41655a05a01ff",
+    "fca98ecd003e1b82",
+    "fce7e01f2cd0cac0",
     "fd3f988b7a505c45",
-    "84c8bf1f809380bc",
-    "c16f3dafab92df61",
-    "55e85aac6594c2fc",
-    "6cef8442092da1da",
-    "1172a88e009dc248",
-    "af4c9b6e93062fde",
-    "96106c2d31eea8b2",
-    "c518dd59d12daf17",
-    "871066e9e2eaba52",
-    "acee5f619f09bf79",
-    "0fe0bb1054f49b7f"
-]
\ No newline at end of file
+    "fd583001a87023fd",
+    "fd966f807d3ba9fe",
+    "ff0ffb9a32873433",
+    "ff45605151ec4eb1",
+    "ffc97a8fff838fa8",
+    "ffd213e55eee8cc8",
+    "ffe4328dcdc6b538",
+    "fff289a66cb9ce4b"
+]
diff --git a/data/reference_annotations_split/train.json b/src/sec_certs/data/reference_annotations/split/train.json
index b2e65750..bc52e9b3 100644
--- a/data/reference_annotations_split/train.json
+++ b/src/sec_certs/data/reference_annotations/split/train.json
@@ -1,466 +1,532 @@
 [
-    "412316e54e63beef",
-    "ce3ee1fa409676b3",
-    "cb862d0020c32547",
-    "bda89f3b2429ab6a",
-    "388653542ec9fb6c",
-    "7f1f7bffe46d4c85",
+    "00a2b91e5b58cef1",
+    "00ad121fa63e5dfb",
+    "00b539d629e60965",
+    "00eb3e5c7a4537b1",
+    "00efeb17bcaafce6",
+    "0167c92c0d8c8b47",
+    "017f3cea8275a3e1",
+    "01ce6f422bec823d",
+    "01dc77d7f8f94d97",
+    "01e5c6d31f763f56",
+    "01e8805514b4ef67",
+    "0208e6b60a0b1a32",
+    "0213bdb5ebb8dc19",
+    "02c6fbcd90dcf05a",
+    "02cb3c385865ad2c",
+    "0319b03323f257d1",
+    "03281d5d8a77be56",
+    "032faef0207b715f",
+    "0349c893b81f7c1a",
+    "037577fc2019fcfa",
+    "03aded94fb04c62e",
+    "04c5a5a66fb7f89a",
+    "04cb360502da6492",
+    "054bd8196e015197",
+    "055985699aee1e09",
+    "05726637bd47a762",
+    "05a8a84b60a30ceb",
+    "05c03233d710cf54",
+    "05d96aada7bb839c",
+    "06b1a65aa87ffa78",
+    "06b24bc51eb69188",
+    "06e505abb8dad1b8",
+    "06e70fa70393d60f",
+    "075ddd00e07fa7ed",
+    "07ce7e4844f62c5c",
+    "080aa6966d757f05",
+    "082380caaa153ee6",
+    "0832672073492f50",
+    "08408cbc394e7421",
+    "09220515a7edf3f9",
+    "096b05114ee5e09f",
+    "096ccb6bcb3d2731",
+    "09963e79b21cde12",
+    "09b07fbbb24e1b04",
+    "09b1192474bc8b27",
+    "09b17cb9b3c8b1bb",
+    "09be49631855ae19",
+    "09f539aedefa9756",
+    "0a6e50445a906052",
+    "0aad0176e35b3749",
+    "0ac0120f667a8dcf",
+    "0b0648a9b9a26747",
+    "0b3e1cdf3ef9413d",
+    "0baac042dac42abc",
+    "0bd651fcbb86f2a9",
+    "0bd759633c0ceec6",
+    "0bf7a19b22163465",
+    "0c9e215997d96f44",
+    "0cd406e7c5013b74",
+    "0ce083454814c1ba",
+    "0d08a211bd29f963",
+    "0d164fb0d72dfa00",
+    "0d4385899adc0781",
+    "0d4a95a8b52b787b",
+    "0d730f18ae9d5694",
+    "0da9d81c9f563df5",
+    "0ee132dec087b772",
+    "0ef56cba2362e463",
+    "0eff4063e68d7cd6",
+    "0f040d1ec658b8a5",
+    "0f1b31ac93459d1f",
     "0f1bd208c27d202c",
-    "b8370ceababbca40",
-    "12e20466a0f08342",
-    "9b8b9b7fcd97bc60",
-    "32e083734907ad15",
-    "294ef6cb28408ec9",
-    "f17fe9ceea628f62",
-    "6f38b9bce48590e3",
-    "b61363e51ed90c7a",
-    "9328c028356ab522",
+    "0f3900cdcd0c7f3e",
+    "0f684159ad31f883",
+    "0fe0bb1054f49b7f",
+    "0ff82b1cc24b3105",
+    "0ff9337564f4f742",
+    "10107b83e7393c50",
+    "10233188161519ee",
+    "102707a8c4d8a1b0",
+    "102bbdfcd696d7a9",
+    "107c348f4ba0a41c",
+    "10868774b23c5730",
+    "109eb2158ca6a2f9",
+    "10af4a6378d0fb7a",
+    "10b17081dd7cad8f",
+    "112fe9f123e277ee",
+    "114e39b4da949e6d",
     "11687f8008b8198d",
+    "1172a88e009dc248",
+    "1188aebd8681e4fe",
+    "11a98c5d5ff6f40f",
+    "122245dd15683e26",
+    "123c94418ddf1ad8",
     "12c3b8b612c581cb",
-    "db3b58c1836b220e",
-    "f87b5bc03e18d010",
-    "a3de319045809828",
-    "ed22d4c0f09c3e3a",
-    "09b1192474bc8b27",
-    "d4e1874210b91dd4",
-    "c9b0cae8b5784d2d",
-    "dfe9de5d26e08949",
-    "97d00a9f198a6e57",
-    "d75282fdda80b6fd",
-    "6d41460480570d22",
-    "a7666a5157a31d32",
-    "5c3806bf79eeab7f",
-    "d2d0b2521a1f186d",
-    "6dfe436482047387",
-    "7e426c62da58175b",
-    "9dbb21687510991c",
-    "2b2dd14fee46498d",
-    "cf32640f396edb87",
-    "4d5cb991d2675a6a",
-    "751a324c3f87f58a",
-    "8bb25a14b21fcce6",
-    "243291b6c7ecde61",
+    "12e20466a0f08342",
+    "12fd6b1b6c4aecf3",
+    "131ec924c42ef90b",
+    "1357546ec01b1a65",
+    "135ca3602f0005bf",
     "13a92431f9683226",
-    "7f4d3b659fc17c09",
-    "dba4b5a166f9456a",
-    "768dda4099b6f2d9",
-    "668b5c95eb0229c1",
-    "1b797b27b981aafe",
+    "13d95239226aa537",
+    "13e5b88ad3becb77",
+    "1416afc51d8ce022",
+    "1468948b4f0509ab",
+    "1494cff85b3dc2bb",
+    "14df54305cad83b4",
+    "14e053dfe9057aa8",
+    "157b741a0f529a0a",
+    "157f811757a0b29e",
+    "158950ed4bc35274",
+    "15d68159595eae09",
+    "15d935e36152eb15",
+    "161216fcc9eb8d34",
+    "16bdbde359584f99",
+    "17a23970b35a4f44",
     "17b5257b3755a07c",
+    "185aa4f76ff15d62",
+    "188d3cda4e6f2d5d",
+    "18bb29b147e80caa",
+    "18e68a485e648ddc",
+    "19c60923e31777e7",
+    "19d14dec2d11fd96",
+    "19f06ccd7136a2b4",
+    "1a191f3535ccaea3",
+    "1b0a6ecee1f830fc",
+    "1b797b27b981aafe",
+    "1b8705a0486b3be1",
     "1c34d31598257da8",
-    "775b1fdecba5cb81",
-    "e0265f0fb8e196c0",
-    "161216fcc9eb8d34",
-    "7574381766e293c7",
-    "9529eb793550093c",
-    "4960693ee45f2ae6",
-    "cacdfdafce47678c",
-    "60f3bd10ee9be85b",
-    "4f5f41ecf7517e63",
-    "c7fc146c5ccf4821",
+    "1c9bd4db3d8388f7",
+    "1ca9b79a98ef45fa",
+    "1d89cece70c6cb64",
+    "1e40bd733acd56bc",
+    "1e58fd6b31d72e2a",
+    "1eabf957a9401fab",
+    "1ee7ecee9e7e131c",
+    "1ee8a6a4d8f26d65",
+    "1f9f4b843070fc1b",
+    "1fa2284c3dcf136a",
+    "1fb1564dfb0f0b04",
+    "1ff5d648a625f3c7",
+    "21c3236216f620ff",
+    "21d71bac070403e0",
+    "22dd52fbce13f27e",
+    "23ce630f1ab642b8",
+    "242fdec5d0ed5010",
+    "243291b6c7ecde61",
     "2477ebdeb9bcf4aa",
-    "6ca1063d82ab795a",
-    "d2a4b8cb9ae7fe8f",
-    "c2024b43eeaa96de",
-    "a6fe8fe0aaf2fa92",
-    "815d67f8292ad7c3",
-    "19c60923e31777e7",
-    "97d5f9e6e0a11e37",
-    "c39e6ef4060594e2",
-    "a76d1c0d9964d583",
-    "a52f807db0eb7fdb",
-    "6e4de0c194952a21",
-    "317713948b4473b3",
-    "73e4f65789159996",
-    "54375fc2889bdfb4",
+    "253e4d53f952a882",
+    "25eade3365f92578",
+    "2674228d88c4f3d6",
     "2731d8ddba404fad",
-    "617810f97f3f0b4f",
-    "e8e5f42a4e4005c7",
-    "03aded94fb04c62e",
-    "1fb1564dfb0f0b04",
-    "e7e366f3619b5953",
-    "cd43abdf668dbb23",
-    "1fa2284c3dcf136a",
-    "c96343baf608174d",
-    "c32378a010479b33",
-    "99223aca5d9eb3b3",
-    "678b2b51e429444a",
-    "727bce62137f97b7",
-    "62bf5de838cd3a7c",
-    "732271484650d27a",
-    "b42009eb34746731",
+    "27b4366a0f1fcdef",
+    "27d11629261d8806",
+    "28e70f46ba4394dd",
+    "294ef6cb28408ec9",
+    "29b0321ec6b75ebd",
+    "29c65bb00fb9aa6c",
+    "2b2dd14fee46498d",
     "2b944ab6ce8fe8e1",
-    "8ff91b407ccd4b18",
-    "8fbe28bc70a821bd",
-    "86255c61e33c2caf",
+    "2bb4685a03857b2c",
+    "2c15dfc106ebf8d8",
+    "2c4caf71ca3735a8",
+    "2d2ce200fea72359",
+    "2e2d56abe9dd6e22",
+    "2ebb5bdbe7e00a78",
+    "2f418fb3ced3820a",
+    "30120e4f3aa2f30a",
+    "30f71b100c5cebae",
+    "317713948b4473b3",
+    "32431779b1c54c0a",
+    "32e083734907ad15",
+    "335800e03030fb93",
+    "34370b67b5e675c3",
+    "3515801dee00995f",
+    "366786e98797ebf3",
+    "36ed04f4b45e3ab9",
+    "371dca18821f7714",
+    "376b8eb06fc0b40a",
+    "388653542ec9fb6c",
+    "388b6899fb38b762",
+    "3897d8d16ba333e8",
+    "3b0c5666e54a2394",
+    "3cdca4cc05854ade",
+    "3da6e0f0f97b3d2f",
+    "3db023c27932ad93",
+    "3ebe69af702fa116",
+    "3ef6d59aeaa6d26e",
+    "3f45ee333ce457bb",
+    "3f87aa90c0aaf83b",
+    "41148b8edaf1106e",
+    "412316e54e63beef",
+    "41fb9717f70d3037",
+    "42799014c183e1b3",
+    "42a062fe7f1ab253",
+    "446aa68e0c4c5083",
+    "44f1f74c09e634cb",
+    "4526aa14337a2b0c",
+    "454f266801ae2ae0",
+    "457fcec1ad31841f",
+    "463ecd64b7506048",
+    "4761b0a9b046cc66",
+    "4783477a22587a6e",
+    "4960693ee45f2ae6",
     "49bf2cf97dd4ca29",
-    "9776cff7d7ead7bb",
-    "69347104bc7e740a",
-    "818f62b997d143e3",
-    "bc17c54936484a90",
-    "824a8da5564344a3",
-    "fe223a9f01625d62",
+    "4a6100f1ddaef93b",
+    "4bede5d2af4cb11f",
+    "4c1ccde478ce063a",
     "4c3282bd1640fcbc",
-    "ee1c6dd97918d74a",
-    "ace99e269d2a1bd9",
+    "4d5cb991d2675a6a",
+    "4d776db86abecdda",
     "4f3bf2d41fc130df",
-    "ca5da2fe138af656",
-    "41fb9717f70d3037",
-    "de9a98c311683ef3",
-    "bc260c38202644f4",
-    "30120e4f3aa2f30a",
-    "e66a9b22df7fe32d",
+    "4f5f41ecf7517e63",
+    "4fd2ed9e6088176e",
+    "502645d74f607b06",
+    "536aac44b608f951",
+    "54273dd266fce692",
+    "54375fc2889bdfb4",
+    "543d800fd468f0fc",
+    "54dd28994604ee35",
     "559a5c3c4c23a9d3",
-    "3515801dee00995f",
-    "f20be2311365981c",
-    "e10bc3422afaec8a",
+    "5682e68143deea61",
     "5734ff91ea85611b",
-    "dabbb27e6faa13d5",
-    "7a8d4ed693d443d7",
-    "d94b13d95fd1c12a",
-    "e2e705cabd42e40e",
-    "b434dea9be2868db",
-    "e9e12b044b495c6f",
-    "a07548a64fac5794",
+    "598b6c745cdcd404",
+    "59f15180eddee8b2",
+    "5a12431e371f3ce9",
+    "5a66027d34aafc52",
+    "5bf4e37462a1b715",
+    "5c3806bf79eeab7f",
+    "5cc6ace5ef0d7525",
+    "5cdef03a3004a6ff",
+    "5deb0179858d78a1",
     "5df7b3241b307f8b",
-    "18e68a485e648ddc",
-    "01e8805514b4ef67",
-    "13d95239226aa537",
-    "3f87aa90c0aaf83b",
-    "f2dc875bd399a806",
-    "a83fe0ea2f391ee8",
-    "7793c695818500d2",
-    "a38fbb576027f6c6",
-    "2f418fb3ced3820a",
-    "b5b0e3205923e723",
+    "5e5d4b87195388c7",
+    "5ee0fb9826fec164",
+    "5f1bc82727f54376",
+    "5f3c4c189362c2e5",
+    "5f4b39feb2a82dac",
+    "5f7760ca5dd8b8ef",
+    "600e2a1d0f05afd9",
+    "608fe49756dc68cc",
+    "60b45a1573fc53e9",
+    "60f3bd10ee9be85b",
+    "610e118d804e4076",
+    "617810f97f3f0b4f",
+    "61832cb4291c343f",
+    "61bff064f593ecc0",
+    "623dee827372d496",
     "627bfd69c2a831cf",
-    "e0b122da55f1f002",
-    "f6579adcbf5faa99",
-    "b33eaac3aae2f1e7",
-    "e6e8add5e4db2d9d",
-    "b0b7b073ca2dfe5f",
-    "fa67fbbcfc0b9442",
-    "af61a31e3fd0d6f0",
-    "fe445dc1c13738d6",
-    "0d08a211bd29f963",
-    "f02cf5078e4548e0",
-    "a276bd58a15d5be6",
-    "9d0a023ecd4ad7c5",
-    "0ff9337564f4f742",
-    "fbb507bc72376d36",
-    "6ddd0d9c37f27706",
+    "6291bcc06184d125",
+    "62bf5de838cd3a7c",
+    "63d0bacbab451d2d",
+    "659f501baa5997da",
+    "65ab2fd5467ef78d",
+    "668b5c95eb0229c1",
+    "66e47f5d28e2f95b",
+    "678b2b51e429444a",
+    "680aeb0a20a9fed3",
+    "691119fe8a2f8e1c",
+    "69347104bc7e740a",
+    "6962629b904be54d",
     "6a2323c5267ae0b7",
-    "744a7a202d909323",
+    "6a5cb89e80f86ce9",
+    "6a999675c9422dfb",
+    "6b003152c5459126",
+    "6b249c38b49ec9f0",
+    "6c14f733fe7aa28e",
+    "6c222ea916b4f5e1",
+    "6ca1063d82ab795a",
+    "6d41460480570d22",
+    "6d6ade44dcc497dd",
+    "6dc47e3f5d35aedd",
+    "6ddd0d9c37f27706",
+    "6dfe436482047387",
+    "6e4de0c194952a21",
+    "6eb6b29c45f6355f",
+    "6f38b9bce48590e3",
+    "6f8d7a6a1dea6a3a",
     "714559f7e1bf956c",
-    "c92a196d0408a5ff",
-    "f869956d14ea1694",
-    "34370b67b5e675c3",
-    "388b6899fb38b762",
-    "dd81eae9478576f3",
-    "8975cbb58ab67846",
-    "055985699aee1e09",
-    "1416afc51d8ce022",
-    "eea87713e11a55c2",
-    "44f1f74c09e634cb",
-    "598b6c745cdcd404",
-    "e9b7e07dc5598afe",
-    "0b3e1cdf3ef9413d",
-    "610e118d804e4076",
-    "fb9ec9e846ea4e87",
-    "aa096ffde94b953b",
-    "887afbb0843425b6",
-    "e2275acc4f1c2d0d",
-    "b3b7ac7aae87793d",
-    "95b2a54b81b0bc6c",
+    "725b6b95cd47b601",
+    "727bce62137f97b7",
+    "732271484650d27a",
+    "73e4f65789159996",
+    "744a7a202d909323",
+    "751a324c3f87f58a",
+    "7574381766e293c7",
+    "76610bf4610246c7",
+    "768dda4099b6f2d9",
+    "76b289d10702db4d",
+    "775b1fdecba5cb81",
+    "7793c695818500d2",
     "77acb3a4a3f58e0b",
-    "ecfb020c517d37c1",
-    "9302a95ca3efba5c",
-    "ced6075dd1813122",
-    "ab3af998dff7a2ef",
-    "0f3900cdcd0c7f3e",
-    "d699ed2b1adc6be4",
-    "1e40bd733acd56bc",
-    "aea11fd4d2a7709d",
-    "cf7705e45048032e",
-    "1d89cece70c6cb64",
-    "f73d08f8cdfe276d",
-    "cfdacd53c732343c",
-    "9a94525307a1b13d",
-    "543d800fd468f0fc",
-    "25eade3365f92578",
-    "e886bfa3a4f4ffd4",
-    "b8437dbfeeaf0f04",
-    "05c03233d710cf54",
-    "18bb29b147e80caa",
-    "10107b83e7393c50",
-    "65ab2fd5467ef78d",
-    "10868774b23c5730",
-    "253e4d53f952a882",
-    "a47713a44e4c656b",
-    "f2e00a1fee27bdd7",
-    "4783477a22587a6e",
-    "15d935e36152eb15",
-    "4a6100f1ddaef93b",
-    "680aeb0a20a9fed3",
-    "14df54305cad83b4",
-    "188d3cda4e6f2d5d",
-    "fc373293fb7c56ac",
+    "79c729d80cb73939",
     "79f48541164e16ff",
-    "e9fc2f8b110e71a0",
-    "ce14eb1b2318f054",
-    "12fd6b1b6c4aecf3",
-    "5f4b39feb2a82dac",
-    "0f040d1ec658b8a5",
-    "09be49631855ae19",
-    "c8ff17063166050d",
-    "76b289d10702db4d",
-    "eac7be8886cb755b",
-    "84c868322995b179",
-    "a515a81bc4be3cd2",
-    "b2bab86575d21b44",
-    "0eff4063e68d7cd6",
-    "32431779b1c54c0a",
-    "c118ee1ff2d1bede",
-    "1b0a6ecee1f830fc",
-    "cd052c606f2f9124",
-    "5deb0179858d78a1",
-    "dd1472476dfe4a8b",
-    "1a191f3535ccaea3",
-    "a9d336b90e8b94a2",
-    "6c222ea916b4f5e1",
-    "c5a60dbdb668bc10",
-    "f3a461d6bf980196",
-    "977477f3f0bf1bfc",
-    "1ee7ecee9e7e131c",
-    "d1a87288683080c4",
-    "185aa4f76ff15d62",
-    "23ce630f1ab642b8",
+    "7a8d4ed693d443d7",
+    "7a9f073396eaec67",
+    "7d8dc0575933f142",
+    "7e24a5a087db6814",
+    "7e426c62da58175b",
+    "7e82a1ceb29e4019",
     "7e9dd6cb86b58f95",
-    "b30ef2c1aaec9236",
-    "a154a01ff4cf04c7",
-    "79c729d80cb73939",
-    "a9639521d8a5ab09",
-    "f32a1e7e2843c096",
+    "7f1f7bffe46d4c85",
+    "7f4d3b659fc17c09",
+    "7f9eb1b8217d15ad",
+    "815d67f8292ad7c3",
+    "818f62b997d143e3",
+    "824a8da5564344a3",
+    "82b7b915b7af9c17",
+    "845bb039719ac5d8",
+    "84a75b6fc33b669e",
+    "84c868322995b179",
+    "85054d3f47bee56c",
+    "850f5b2c312299d5",
+    "86255c61e33c2caf",
+    "882df28aee338320",
+    "887afbb0843425b6",
+    "890f788a91fc2fa1",
+    "8975cbb58ab67846",
+    "8ba22f6c9651edc3",
+    "8bb25a14b21fcce6",
+    "8bdb610131555c12",
     "8c21cafc192eeea5",
-    "59f15180eddee8b2",
-    "2ebb5bdbe7e00a78",
-    "bd1a21ea3d047c14",
-    "dbf15cc4f389196f",
-    "98f6f9dbb1540a31",
-    "cb203cf5d91b1ae3",
-    "f921d39906e6d93d",
-    "6c14f733fe7aa28e",
-    "f71fca5fc684df8b",
-    "7e82a1ceb29e4019",
-    "54273dd266fce692",
-    "536aac44b608f951",
-    "921e042759b30033",
-    "0167c92c0d8c8b47",
+    "8ecc2cf269498bee",
+    "8fbe28bc70a821bd",
+    "8ff91b407ccd4b18",
     "90581f56f6846be5",
-    "a91ca8347f800ee7",
-    "dc278166b5d14512",
     "90c290ed4ded479d",
-    "cee70006132a6f46",
-    "1357546ec01b1a65",
-    "5a66027d34aafc52",
-    "2c4caf71ca3735a8",
-    "0baac042dac42abc",
-    "54dd28994604ee35",
-    "60b45a1573fc53e9",
-    "6a999675c9422dfb",
-    "0d164fb0d72dfa00",
-    "b24f997d9358b324",
-    "df6728b8420ab3b4",
-    "d50ab8fd0c085fb6",
-    "9ae8bef18938d2df",
-    "19d14dec2d11fd96",
-    "f0ce40dfe333e918",
-    "3da6e0f0f97b3d2f",
-    "0d730f18ae9d5694",
-    "ecba01acba8df2ec",
-    "17a23970b35a4f44",
-    "1b8705a0486b3be1",
-    "1ca9b79a98ef45fa",
-    "66e47f5d28e2f95b",
-    "5f1bc82727f54376",
+    "921e042759b30033",
+    "9302a95ca3efba5c",
+    "9328c028356ab522",
+    "9529eb793550093c",
+    "95b2a54b81b0bc6c",
+    "95c69029642bdb41",
+    "9650ee69410c68ec",
     "9743174f44c5e2c9",
-    "ec84060a194fd9cc",
-    "845bb039719ac5d8",
-    "ea6d47427a2bb349",
-    "27d11629261d8806",
-    "6a5cb89e80f86ce9",
-    "29b0321ec6b75ebd",
-    "d355c778fcfee3a9",
-    "0ce083454814c1ba",
-    "c06d86db280f9b2b",
-    "b0e95af7c706ffa0",
-    "b9a973281ea252b3",
-    "d0705c9e6fbaeba3",
-    "a3210e75e10a3ba8",
-    "61832cb4291c343f",
-    "22dd52fbce13f27e",
+    "977477f3f0bf1bfc",
+    "9776cff7d7ead7bb",
+    "97d00a9f198a6e57",
+    "97d5f9e6e0a11e37",
     "985ffde955eee01a",
-    "882df28aee338320",
-    "b23babf61349db1a",
-    "1c9bd4db3d8388f7",
-    "1ff5d648a625f3c7",
-    "691119fe8a2f8e1c",
-    "c30de3192d2e8ec2",
-    "5bf4e37462a1b715",
-    "de23f9f77f9b49f0",
-    "dadd7dd2d4247627",
-    "36ed04f4b45e3ab9",
-    "6b249c38b49ec9f0",
+    "98f6f9dbb1540a31",
+    "99223aca5d9eb3b3",
+    "9a22a2bb323cfd0f",
+    "9a94525307a1b13d",
+    "9aa4b513807ea22c",
+    "9ad46ff913bb805f",
+    "9ae8bef18938d2df",
+    "9b8b9b7fcd97bc60",
+    "9c6440ecec046cc0",
+    "9d0a023ecd4ad7c5",
+    "9dbb21687510991c",
+    "9ef6037ebd29a9be",
+    "a07548a64fac5794",
+    "a154a01ff4cf04c7",
+    "a276bd58a15d5be6",
+    "a3210e75e10a3ba8",
+    "a38fbb576027f6c6",
+    "a3de319045809828",
+    "a47713a44e4c656b",
+    "a515a81bc4be3cd2",
+    "a52f807db0eb7fdb",
+    "a55ace0305e39654",
     "a69ec271f8651d1b",
-    "7f9eb1b8217d15ad",
-    "6b003152c5459126",
-    "6eb6b29c45f6355f",
-    "502645d74f607b06",
-    "6d6ade44dcc497dd",
-    "3ebe69af702fa116",
-    "29c65bb00fb9aa6c",
-    "b4a2999b439b1ba8",
-    "3b0c5666e54a2394",
-    "8bdb610131555c12",
+    "a6cdbfd0a7bef417",
+    "a6fe8fe0aaf2fa92",
+    "a7666a5157a31d32",
+    "a76d1c0d9964d583",
+    "a7d6e1aeb5e009f0",
+    "a83fe0ea2f391ee8",
+    "a867d281d34d34b7",
+    "a904d6cb272e8533",
+    "a91ca8347f800ee7",
+    "a92a031f28de3544",
+    "a9639521d8a5ab09",
+    "a9d336b90e8b94a2",
+    "aa096ffde94b953b",
+    "aabc3155ba9aec95",
+    "ab3af998dff7a2ef",
+    "ace99e269d2a1bd9",
+    "aea11fd4d2a7709d",
+    "af61a31e3fd0d6f0",
+    "b07adba37aa1b7e2",
+    "b0b7b073ca2dfe5f",
+    "b0e95af7c706ffa0",
     "b10ee0bb166cfd86",
-    "e676143c80802a59",
-    "15d68159595eae09",
-    "06b1a65aa87ffa78",
-    "5ee0fb9826fec164",
-    "2c15dfc106ebf8d8",
+    "b23babf61349db1a",
+    "b24f997d9358b324",
+    "b26c9f300c14239d",
+    "b2bab86575d21b44",
+    "b30ef2c1aaec9236",
+    "b33eaac3aae2f1e7",
+    "b3b7ac7aae87793d",
+    "b42009eb34746731",
+    "b434dea9be2868db",
+    "b4a2999b439b1ba8",
+    "b5b0e3205923e723",
+    "b61363e51ed90c7a",
+    "b8370ceababbca40",
+    "b8437dbfeeaf0f04",
+    "b9482c3bd09d03dd",
+    "b9a973281ea252b3",
+    "b9d8c4cb5cf8a4d6",
+    "baa8b8c5f52810af",
     "bab97726875c0f14",
-    "a867d281d34d34b7",
-    "eeb2e28f54484dc9",
-    "d753660b3c2bdd84",
-    "5f7760ca5dd8b8ef",
-    "8ba22f6c9651edc3",
-    "9650ee69410c68ec",
-    "608fe49756dc68cc",
-    "e7bfc160fd3c0d5d",
-    "335800e03030fb93",
-    "446aa68e0c4c5083",
-    "1e58fd6b31d72e2a",
-    "4c1ccde478ce063a",
-    "5cc6ace5ef0d7525",
-    "09b17cb9b3c8b1bb",
     "bbf0ceb1e0e4c6ee",
-    "c78ba3edcbd57788",
-    "ead1076787fdd7e4",
-    "8ecc2cf269498bee",
-    "d6d6b7c041820ab8",
-    "d837b4fde544a9f2",
-    "3ef6d59aeaa6d26e",
-    "fca2af05ee78ac5c",
-    "cf5cfcae47e5e3a1",
-    "baa8b8c5f52810af",
-    "3f45ee333ce457bb",
-    "7e24a5a087db6814",
-    "30f71b100c5cebae",
-    "9a22a2bb323cfd0f",
-    "84a75b6fc33b669e",
-    "21d71bac070403e0",
-    "0d4385899adc0781",
-    "05726637bd47a762",
-    "7a9f073396eaec67",
-    "5cdef03a3004a6ff",
-    "7d8dc0575933f142",
-    "890f788a91fc2fa1",
-    "5682e68143deea61",
-    "6dc47e3f5d35aedd",
-    "16bdbde359584f99",
-    "0bf7a19b22163465",
+    "bc17c54936484a90",
+    "bc260c38202644f4",
+    "bd1a21ea3d047c14",
+    "bda89f3b2429ab6a",
     "bdd16afe92166295",
-    "9ad46ff913bb805f",
-    "b07adba37aa1b7e2",
-    "3cdca4cc05854ade",
-    "41148b8edaf1106e",
-    "85054d3f47bee56c",
-    "aabc3155ba9aec95",
-    "dbaf9789160ca414",
-    "463ecd64b7506048",
-    "cef1c9957f4ff612",
-    "1ee8a6a4d8f26d65",
-    "a92a031f28de3544",
+    "c06d86db280f9b2b",
+    "c118ee1ff2d1bede",
+    "c175537338906951",
     "c1a87780d88d85d9",
-    "659f501baa5997da",
-    "95c69029642bdb41",
-    "e10d907c64d75e3a",
-    "4fd2ed9e6088176e",
-    "1eabf957a9401fab",
-    "cf8ac5972027dda5",
-    "623dee827372d496",
-    "f041e5b526e79ef4",
-    "0832672073492f50",
-    "0213bdb5ebb8dc19",
-    "d49988efd778ca9d",
+    "c2024b43eeaa96de",
     "c301c38902477230",
-    "376b8eb06fc0b40a",
-    "037577fc2019fcfa",
-    "14e053dfe9057aa8",
-    "28e70f46ba4394dd",
-    "b9482c3bd09d03dd",
-    "d2b0cb5a911b8ef8",
-    "05a8a84b60a30ceb",
-    "63d0bacbab451d2d",
-    "5f3c4c189362c2e5",
-    "c7d162e62f174cd5",
-    "1f9f4b843070fc1b",
-    "850f5b2c312299d5",
-    "b9d8c4cb5cf8a4d6",
-    "f6147b5c0a147123",
-    "3db023c27932ad93",
-    "42a062fe7f1ab253",
-    "a904d6cb272e8533",
-    "e86db251be2c8a60",
-    "5e5d4b87195388c7",
-    "9ef6037ebd29a9be",
-    "242fdec5d0ed5010",
-    "0bd759633c0ceec6",
-    "6f8d7a6a1dea6a3a",
-    "a55ace0305e39654",
-    "19f06ccd7136a2b4",
-    "2e2d56abe9dd6e22",
+    "c30de3192d2e8ec2",
+    "c32378a010479b33",
+    "c3549567b1ad7bdf",
     "c385c0ffaa17bd29",
     "c38734a9eeea0ff8",
-    "3897d8d16ba333e8",
+    "c39e6ef4060594e2",
+    "c5a60dbdb668bc10",
+    "c78ba3edcbd57788",
+    "c7d162e62f174cd5",
+    "c7f6cb518fb19151",
+    "c7fc146c5ccf4821",
+    "c8ff17063166050d",
+    "c92a196d0408a5ff",
+    "c96343baf608174d",
+    "c9b0cae8b5784d2d",
+    "ca5da2fe138af656",
+    "cacdfdafce47678c",
+    "cb203cf5d91b1ae3",
+    "cb862d0020c32547",
+    "cd052c606f2f9124",
+    "cd43abdf668dbb23",
+    "ce14eb1b2318f054",
+    "ce3ee1fa409676b3",
+    "ced6075dd1813122",
+    "cee70006132a6f46",
+    "cef1c9957f4ff612",
+    "cf32640f396edb87",
+    "cf5cfcae47e5e3a1",
+    "cf7705e45048032e",
+    "cf8ac5972027dda5",
+    "cfdacd53c732343c",
+    "d0705c9e6fbaeba3",
+    "d1a87288683080c4",
+    "d2a4b8cb9ae7fe8f",
+    "d2b0cb5a911b8ef8",
+    "d2d0b2521a1f186d",
+    "d355c778fcfee3a9",
+    "d49988efd778ca9d",
+    "d4e1874210b91dd4",
+    "d50ab8fd0c085fb6",
+    "d699ed2b1adc6be4",
+    "d6d6b7c041820ab8",
     "d7467eab308fb4a6",
-    "4761b0a9b046cc66",
-    "0ac0120f667a8dcf",
+    "d75282fdda80b6fd",
+    "d753660b3c2bdd84",
+    "d837b4fde544a9f2",
+    "d94b13d95fd1c12a",
+    "dabbb27e6faa13d5",
+    "dadd7dd2d4247627",
+    "db3b58c1836b220e",
+    "dba4b5a166f9456a",
+    "dbaf9789160ca414",
+    "dbbf02a1cd0ad33b",
+    "dbf15cc4f389196f",
+    "dc278166b5d14512",
+    "dd1472476dfe4a8b",
+    "dd81eae9478576f3",
+    "de23f9f77f9b49f0",
+    "de9a98c311683ef3",
+    "df6728b8420ab3b4",
+    "dfe9de5d26e08949",
+    "e0265f0fb8e196c0",
+    "e0b122da55f1f002",
+    "e10bc3422afaec8a",
+    "e10d907c64d75e3a",
+    "e2275acc4f1c2d0d",
+    "e2e705cabd42e40e",
+    "e66a9b22df7fe32d",
+    "e676143c80802a59",
+    "e6e8add5e4db2d9d",
+    "e7bfc160fd3c0d5d",
+    "e7e366f3619b5953",
+    "e86db251be2c8a60",
+    "e886bfa3a4f4ffd4",
+    "e8e5f42a4e4005c7",
+    "e9b7e07dc5598afe",
+    "e9e12b044b495c6f",
+    "e9fc2f8b110e71a0",
+    "ea6d47427a2bb349",
+    "eac7be8886cb755b",
+    "ead1076787fdd7e4",
+    "ec84060a194fd9cc",
+    "ecba01acba8df2ec",
+    "ecfb020c517d37c1",
     "ed10036ddfbda912",
-    "00efeb17bcaafce6",
-    "61bff064f593ecc0",
-    "c3549567b1ad7bdf",
-    "371dca18821f7714",
-    "a6cdbfd0a7bef417",
-    "c175537338906951",
-    "c7f6cb518fb19151",
-    "6291bcc06184d125",
-    "06e505abb8dad1b8",
-    "82b7b915b7af9c17",
-    "725b6b95cd47b601",
-    "4bede5d2af4cb11f",
-    "600e2a1d0f05afd9",
-    "03281d5d8a77be56",
-    "27b4366a0f1fcdef",
-    "42799014c183e1b3",
-    "9c6440ecec046cc0",
-    "b26c9f300c14239d",
-    "366786e98797ebf3",
-    "454f266801ae2ae0",
-    "fdbbd2b53cb2e1e3",
-    "76610bf4610246c7",
-    "09963e79b21cde12",
-    "457fcec1ad31841f",
-    "6962629b904be54d",
-    "a7d6e1aeb5e009f0",
-    "2674228d88c4f3d6",
-    "2bb4685a03857b2c",
-    "4d776db86abecdda",
-    "21c3236216f620ff",
-    "5a12431e371f3ce9",
-    "2d2ce200fea72359",
-    "9aa4b513807ea22c",
-    "4526aa14337a2b0c",
+    "ed22d4c0f09c3e3a",
+    "ee1c6dd97918d74a",
+    "eea87713e11a55c2",
+    "eeb2e28f54484dc9",
+    "f02cf5078e4548e0",
+    "f041e5b526e79ef4",
+    "f0ce40dfe333e918",
+    "f17fe9ceea628f62",
+    "f20be2311365981c",
+    "f2dc875bd399a806",
+    "f2e00a1fee27bdd7",
+    "f32a1e7e2843c096",
+    "f3a461d6bf980196",
+    "f6147b5c0a147123",
+    "f6579adcbf5faa99",
     "f659401e91439524",
-    "dbbf02a1cd0ad33b"
+    "f71fca5fc684df8b",
+    "f73d08f8cdfe276d",
+    "f869956d14ea1694",
+    "f87b5bc03e18d010",
+    "f921d39906e6d93d",
+    "fa67fbbcfc0b9442",
+    "fb9ec9e846ea4e87",
+    "fbb507bc72376d36",
+    "fc373293fb7c56ac",
+    "fca2af05ee78ac5c",
+    "fdbbd2b53cb2e1e3",
+    "fe223a9f01625d62",
+    "fe445dc1c13738d6"
 ]
\ No newline at end of file
diff --git a/data/reference_annotations_split/valid.json b/src/sec_certs/data/reference_annotations/split/valid.json
index 939924f0..939924f0 100644
--- a/data/reference_annotations_split/valid.json
+++ b/src/sec_certs/data/reference_annotations/split/valid.json
diff --git a/data/sar_correlations/all_certs_sar_cve_corr.csv b/src/sec_certs/data/sar_correlations/all_certs_sar_cve_corr.csv
index 9a459b6b..9a459b6b 100644
--- a/data/sar_correlations/all_certs_sar_cve_corr.csv
+++ b/src/sec_certs/data/sar_correlations/all_certs_sar_cve_corr.csv
diff --git a/data/sar_correlations/readme.md b/src/sec_certs/data/sar_correlations/readme.md
index 6695d4f1..6695d4f1 100644
--- a/data/sar_correlations/readme.md
+++ b/src/sec_certs/data/sar_correlations/readme.md
diff --git a/data/sar_correlations/vuln_rich_certs_sar_cve_corr.csv b/src/sec_certs/data/sar_correlations/vuln_rich_certs_sar_cve_corr.csv
index a5bc6df2..a5bc6df2 100644
--- a/data/sar_correlations/vuln_rich_certs_sar_cve_corr.csv
+++ b/src/sec_certs/data/sar_correlations/vuln_rich_certs_sar_cve_corr.csv
diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index 7eea6db7..04db4d7c 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -22,10 +22,12 @@ from sec_certs.dataset.cpe import CPEDataset
 from sec_certs.dataset.cve import CVEDataset
 from sec_certs.dataset.dataset import AuxiliaryDatasets, Dataset, logger
 from sec_certs.dataset.protection_profile import ProtectionProfileDataset
+from sec_certs.model import (
+    ReferenceFinder,
+    SARTransformer,
+    TransitiveVulnerabilityFinder,
+)
 from sec_certs.model.cc_matching import CCSchemeMatcher
-from sec_certs.model.reference_finder import ReferenceFinder
-from sec_certs.model.sar_transformer import SARTransformer
-from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder
 from sec_certs.sample.cc import CCCertificate
 from sec_certs.sample.cc_certificate_id import CertificateId
 from sec_certs.sample.cc_maintenance_update import CCMaintenanceUpdate
@@ -163,6 +165,10 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         return self.mu_dataset_dir / "maintenance_updates.json"
 
     @property
+    def reference_annotator_dir(self) -> Path:
+        return self.root_dir / "reference_annotator"
+
+    @property
     def scheme_dataset_path(self) -> Path:
         """
         Returns a path to the scheme dataset
diff --git a/src/sec_certs/model/__init__.py b/src/sec_certs/model/__init__.py
index 3c5ea459..69cbe194 100644
--- a/src/sec_certs/model/__init__.py
+++ b/src/sec_certs/model/__init__.py
@@ -17,4 +17,7 @@ __all__ = [
     "ReferenceFinder",
     "TransitiveVulnerabilityFinder",
     "SARTransformer",
+    "ReferenceAnnotator",
+    "ReferenceAnnotatorTrainer",
+    "ReferenceSegmentExtractor",
 ]
diff --git a/src/sec_certs/model/references_nlp/__init__.py b/src/sec_certs/model/references_nlp/__init__.py
new file mode 100644
index 00000000..d4e11e4d
--- /dev/null
+++ b/src/sec_certs/model/references_nlp/__init__.py
@@ -0,0 +1,13 @@
+# ruff: noqa: F401
+try:
+    import catboost
+    import optuna
+    import plotly.express
+    import setfit
+    import sklearn
+    import umap
+except ImportError as e:
+    print(e)
+    print(
+        "Requirements for ML annotation of references not met. Please run `pip install sec-certs[nlp]` or install `pip install -r requirements/nlp_requirements.txt."
+    )
diff --git a/src/sec_certs/model/references_nlp/annotator.py b/src/sec_certs/model/references_nlp/annotator.py
new file mode 100644
index 00000000..1cc816f5
--- /dev/null
+++ b/src/sec_certs/model/references_nlp/annotator.py
@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import json
+import logging
+from collections import Counter
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from setfit import SetFitModel
+
+from sec_certs.utils.nlp import softmax
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ReferenceAnnotator:
+    """
+    Class for annotating references. Its instances are supposed to by trained by `ReferenceAnnotatorTrainer`.
+    Can be serialized into a directory / load from a directory.
+    """
+
+    _model: Any
+    _label_mapping: dict[int, str]
+    _soft_voting_power: int = 2
+
+    @classmethod
+    def from_pretrained(cls, model_dir: str | Path) -> ReferenceAnnotator:
+        """
+        Loads classifier from directory, assuming that:
+          - the SetFitModel was dumped into that directory with model.save_pretrained(model_dir)
+          - json file label_mapping.json exists in model_dir
+
+        :param str | Path model_dir: path to directory to search for model and label mapping
+        :return RerefenceClassifier: classifier with SetFitModel and label mapping
+        """
+        logger.info(f"Loading pre-trained reference annotator from: {model_dir}")
+        model = SetFitModel.from_pretrained(str(model_dir))
+        with (Path(model_dir) / "label_mapping.json").open("r") as handle:
+            label_mapping = json.load(handle)
+            label_mapping = {int(k): v for k, v in label_mapping.items()}
+
+        return cls(model, label_mapping)
+
+    def save_pretrained(self, model_dir: str | Path):
+        """
+        Will dump _model and _label_mapping into a directory.
+        """
+        logger.info(f"Saving ReferenceAnnotator to {model_dir}")
+        model_dir = Path(model_dir)
+        model_dir.mkdir(exist_ok=True, parents=True)
+        logger.info
+        with (model_dir / "label_mapping.json").open("w") as handle:
+            json.dump(self._label_mapping, handle, indent=4)
+        self._model._save_pretrained(str(model_dir))
+
+    def train(self, train_dataset: pd.DataFrame):
+        raise NotImplementedError("ReferenceAnnotatorTrainer shall be used for training")
+
+    def predict(self, X: list[list[str]]) -> list[str]:
+        return [self._predict_single(x) for x in X]
+
+    def _predict_single(self, sample: list[str]) -> str:
+        # return self._predict_single_majority_vote(sample)
+        return self._label_mapping[int(np.argmax(self._predict_proba_single(sample)))]
+
+    def predict_proba(self, X: list[list[str]]) -> list[list[float]]:
+        return [self._predict_proba_single(x) for x in X]
+
+    def _predict_proba_single(self, sample: list[str]) -> list[float]:
+        """
+        1. Get predictions for each segment, convert pytorch tensor to numpy
+        2. Square every prediction to reward confidence
+        3. Sum probabilities for each label
+        4. softmax
+        """
+        return softmax(np.power(self._model.predict_proba(sample, as_numpy=True), self._soft_voting_power).sum(axis=0))
+
+    def _predict_single_majority_vote(self, sample: list[str]) -> str:
+        predictions = self._model.predict(sample)
+        most_common = Counter(predictions).most_common()
+        if len(most_common) > 1 and most_common[0][1] == most_common[1][1]:
+            return self._label_mapping[int(np.argmax(self._predict_proba_single(sample)))]
+        else:
+            return self._label_mapping[int(most_common[0][0])]
+
+    def predict_df(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        WIll read df.segments and populate the dataframe with predictions.
+        """
+        df_new = df.copy()
+        y_proba = self.predict_proba(df_new.segments)
+        df_new["y_proba"] = y_proba
+        df_new["y_pred"] = self.predict(df_new.segments)
+        df_new["correct"] = df_new.apply(
+            lambda row: row["y_pred"] == row["label"] if not pd.isnull(row["label"]) else np.NaN, axis=1
+        )
+        return df_new
diff --git a/src/sec_certs/model/references_nlp/annotator_trainer.py b/src/sec_certs/model/references_nlp/annotator_trainer.py
new file mode 100644
index 00000000..0f506456
--- /dev/null
+++ b/src/sec_certs/model/references_nlp/annotator_trainer.py
@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from functools import partial
+from typing import Final, Literal
+
+import pandas as pd
+from datasets import ClassLabel, Dataset, Features, NamedSplit, Value
+from sentence_transformers.losses import CosineSimilarityLoss
+from setfit import SetFitModel, SetFitTrainer
+from sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score
+
+from sec_certs.model.references_nlp.annotator import ReferenceAnnotator
+from sec_certs.utils.nlp import prepare_reference_annotations_df
+
+logger = logging.getLogger(__name__)
+
+
+class ReferenceAnnotatorTrainer:
+    METRIC_TO_USE: Final[dict[str, Callable]] = {
+        "accuracy": accuracy_score,
+        "balanced_accuracy": balanced_accuracy_score,
+        "f1": partial(f1_score, average="weighted", zero_division=0),
+    }
+
+    def __init__(
+        self,
+        train_dataset: pd.DataFrame,
+        eval_dataset: pd.DataFrame,
+        metric: Callable,
+        n_iterations: int = 20,
+        learning_rate: float = 2e-5,
+        n_epochs: int = 1,
+        batch_size: int = 16,
+        segmenter_metric: Literal["accuracy", "f1", "balanced_accuracy"] = "accuracy",
+        ensemble_soft_voting_power: int = 2,
+        show_progress_bar: bool = True,
+    ):
+        self._train_dataset = train_dataset
+        self._eval_dataset = eval_dataset
+        self._metric = metric
+        self.n_iterations = n_iterations
+        self.learning_rate = learning_rate
+        self.n_epochs = n_epochs
+        self.batch_size = batch_size
+        self.segmenter_metric = segmenter_metric
+        self.ensemble_soft_voting_power = ensemble_soft_voting_power
+        self.show_progress_bar = show_progress_bar
+
+        self._model, self._trainer, self.label_mapping = self._init_trainer()
+
+        self.clf = ReferenceAnnotator(
+            self._model,
+            self.label_mapping,
+            self.ensemble_soft_voting_power,
+        )
+
+    @classmethod
+    def from_df(
+        cls,
+        df: pd.DataFrame,
+        metric: Callable,
+        mode: Literal["training", "evaluation", "production", "cross-validation"] = "training",
+        n_iterations: int = 20,
+        learning_rate: float = 2e-5,
+        n_epochs: int = 1,
+        batch_size: int = 16,
+        segmenter_metric: Literal["accuracy", "f1", "balanced_accuracy"] = "accuracy",
+        ensemble_soft_voting_power: int = 2,
+        show_progress_bar: bool = True,
+    ):
+        df = prepare_reference_annotations_df(df)
+        dataset_generation_method = {
+            "training": ReferenceAnnotatorTrainer.split_df_for_training,
+            "evaluation": ReferenceAnnotatorTrainer.split_df_for_evaluation,
+            "production": ReferenceAnnotatorTrainer.split_df_for_production,
+            "cross-validation": ReferenceAnnotatorTrainer.split_df_for_training,
+        }
+
+        train_dataset, eval_dataset = dataset_generation_method[mode](df)
+        return cls(
+            train_dataset,
+            eval_dataset,
+            metric,
+            n_iterations,
+            learning_rate,
+            n_epochs,
+            batch_size,
+            segmenter_metric,
+            ensemble_soft_voting_power,
+            show_progress_bar,
+        )
+
+    @staticmethod
+    def split_df_for_training(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:
+        return df.loc[df.split == "train"].drop(columns="split"), df.loc[df.split == "valid"].drop(columns="split")
+
+    @staticmethod
+    def split_df_for_production(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:
+        logger.info("Splitting dataset for production, model can be trained, but not evaluated.")
+        return df.drop(columns="split"), df.drop(df.index)
+
+    @staticmethod
+    def split_df_for_evaluation(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:
+        df.split = df.split.map({"test": "test", "train": "train", "valid": "train"})
+        if df.loc[df.split == "test"].empty:
+            logger.warning("`test` split for annotator dataset is empty -> model can be trained, but not evaluated.")
+        return df.loc[df.split == "train"].drop(columns="split"), df.loc[df.split == "test"].drop(columns="split")
+
+    def _init_trainer(self):
+        model = SetFitModel.from_pretrained("paraphrase-multilingual-mpnet-base-v2")
+        # model = SetFitModel.from_pretrained("all-mpnet-base-v2")
+
+        train_dataset_relevant_cols = self._train_dataset[["dgst", "canonical_reference_keyword", "segments", "label"]]
+        eval_dataset_relevant_cols = self._eval_dataset[["dgst", "canonical_reference_keyword", "segments", "label"]]
+        internal_train_dataset = self._get_hugging_face_datasets_from_df(train_dataset_relevant_cols, "train")
+        internal_validation_dataset = self._get_hugging_face_datasets_from_df(eval_dataset_relevant_cols, "validation")
+
+        # Align labels alphabetically
+        labels_alphabetically = sorted(internal_train_dataset.features["label"].names)
+        label2id = {label: index for index, label in enumerate(labels_alphabetically)}
+        internal_train_dataset = internal_train_dataset.align_labels_with_mapping(label2id, "label")
+        internal_validation_dataset = internal_validation_dataset.align_labels_with_mapping(label2id, "label")
+
+        trainer = SetFitTrainer(
+            model=model,
+            train_dataset=internal_train_dataset,
+            eval_dataset=internal_validation_dataset,
+            loss_class=CosineSimilarityLoss,
+            metric=self.METRIC_TO_USE[self.segmenter_metric],
+            learning_rate=self.learning_rate,
+            batch_size=self.batch_size,
+            num_iterations=self.n_iterations,  # The number of text pairs to generate for contrastive learning
+            num_epochs=self.n_epochs,  # The number of epochs to use for contrastive learning
+            column_mapping={
+                "segment": "text",
+                "label": "label",
+            },  # Map dataset columns to text/label expected by trainer
+        )
+        return model, trainer, {index: label for label, index in label2id.items()}
+
+    @staticmethod
+    def _get_hugging_face_datasets_from_df(df: pd.DataFrame, split: NamedSplit) -> Dataset:
+        df_to_use = df.explode("segments").rename(columns={"segments": "segment"}).loc[df.label.notnull()]
+        features = Features(
+            {
+                "dgst": Value("string"),
+                "canonical_reference_keyword": Value("string"),
+                "segment": Value("string"),
+                "label": ClassLabel(names=list(df_to_use.label.unique())),
+            }
+        )
+        return Dataset.from_pandas(df_to_use, features=features, split=split, preserve_index=False)
+
+    def train(self):
+        self._trainer.train(show_progress_bar=self.show_progress_bar)
+
+    def evaluate(self):
+        print("Internal evaluation (of model working on individual segments)")
+        print(self._evaluate_raw())
+        print("Actual evaluation after ensemble soft voting")
+        print(self._evaluate_stacked())
+
+    def _evaluate_raw(self):
+        if self._eval_dataset.empty:
+            logger.error("Evaluation dataset is empty, cannot evaluate, returning.")
+            return
+        return self._trainer.evaluate()
+
+    def _evaluate_stacked(self):
+        y_pred = self.clf.predict(self._eval_dataset.segments)
+        y_true = self._eval_dataset.label
+        return self._metric(y_pred, y_true)
diff --git a/src/sec_certs/model/references_nlp/evaluation.py b/src/sec_certs/model/references_nlp/evaluation.py
new file mode 100644
index 00000000..ed3b2700
--- /dev/null
+++ b/src/sec_certs/model/references_nlp/evaluation.py
@@ -0,0 +1,80 @@
+import logging
+from pathlib import Path
+from typing import Literal
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import plotly.express as px
+from catboost import CatBoostClassifier
+from sklearn.dummy import DummyClassifier
+from sklearn.metrics import ConfusionMatrixDisplay, balanced_accuracy_score, classification_report
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate_model(
+    clf: DummyClassifier | CatBoostClassifier,
+    x_eval: np.ndarray,
+    y_eval: np.ndarray,
+    feature_cols: list[str],
+    output_path: Path | None = None,
+):
+    logger.info("Evaluating model.")
+    y_pred = clf.predict(x_eval)
+
+    print(classification_report(y_eval, y_pred))
+    print(f"Balanced accuracy score: {balanced_accuracy_score(y_eval, y_pred)}")
+
+    fig = ConfusionMatrixDisplay.from_predictions(
+        y_eval,
+        y_pred,
+        xticks_rotation=90,
+    )
+
+    if output_path:
+        report_dict = classification_report(y_eval, y_pred, output_dict=True)
+        report_df = pd.DataFrame(report_dict).transpose()
+        report_df.to_csv(output_path / "classification_report.csv")
+        fig.figure_.savefig(output_path / "confusion_matrix.png", bbox_inches="tight")
+        with Path(output_path / "balanced_accuracy_score.txt").open("w") as handle:
+            handle.write(str(balanced_accuracy_score(y_eval, y_pred)))
+
+    if isinstance(clf, CatBoostClassifier):
+        feature_importance = clf.get_feature_importance()
+        sorted_idx = np.argsort(feature_importance)
+        features = np.array(feature_cols)[sorted_idx]
+
+        fig_feature_importance = plt.figure(figsize=(10, 12))
+        plt.barh(features, feature_importance[sorted_idx], align="center")
+        plt.xlabel("Feature Importance")
+        plt.ylabel("Feature")
+        plt.title("Feature Importance in Gradient boosted trees classifier")
+        plt.tight_layout()
+        plt.show()
+
+        if output_path:
+            fig_feature_importance.savefig(output_path / "feature_importance.png")
+
+
+def display_dim_red_scatter(df: pd.DataFrame, dim_red: Literal["umap", "pca"]) -> None:
+    df_exploded = df.explode(["segments", dim_red]).reset_index()
+
+    x_col = dim_red + "_x"
+    y_col = dim_red + "_y"
+
+    df_exploded[x_col] = df_exploded[dim_red].map(lambda x: x[0])
+    df_exploded[y_col] = df_exploded[dim_red].map(lambda x: x[1])
+    df_exploded["wrapped_segment"] = df_exploded.segments.str.wrap(60).map(lambda x: x.replace("\n", "<br>"))
+
+    fig = px.scatter(
+        df_exploded,
+        x=x_col,
+        y=y_col,
+        color="label",
+        hover_data=["dgst", "canonical_reference_keyword", "wrapped_segment"],
+        width=1500,
+        height=1000,
+        title=f"{dim_red.upper()} projection of segment embeddings.",
+    )
+    fig.show()
diff --git a/src/sec_certs/model/references_nlp/feature_extraction.py b/src/sec_certs/model/references_nlp/feature_extraction.py
new file mode 100644
index 00000000..996972d7
--- /dev/null
+++ b/src/sec_certs/model/references_nlp/feature_extraction.py
@@ -0,0 +1,588 @@
+import itertools
+import logging
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+import spacy
+import umap
+import umap.plot
+from rapidfuzz import fuzz
+from scipy.spatial import ConvexHull, QhullError, distance_matrix
+from scipy.stats import kurtosis, skew
+from sklearn.decomposition import PCA
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+
+from sec_certs.constants import RANDOM_STATE, REF_ANNOTATION_MODES, REF_EMBEDDING_METHOD
+from sec_certs.dataset import CCDataset
+from sec_certs.model.references_nlp.annotator import ReferenceAnnotator
+from sec_certs.model.references_nlp.annotator_trainer import ReferenceAnnotatorTrainer
+from sec_certs.model.references_nlp.segment_extractor import ReferenceSegmentExtractor
+from sec_certs.utils.nlp import prec_recall_metric
+
+logger = logging.getLogger(__name__)
+
+nlp = spacy.load("en_core_web_sm")
+
+
+def strip_all(text: str, to_strip) -> str:
+    if pd.isna(to_strip):
+        return text
+    for i in to_strip:
+        text = text.replace(i, "")
+    return text
+
+
+def matches_recertification(segments: list[str]) -> bool:
+    regex_a = r"This is a re-?\s?certification based on (the\s){0,1}REFERENCED_CERTIFICATE_ID"
+    regex_b = r"Re-?\s?Zertifizierung basierend auf (the\s){0,1}REFERENCED_CERTIFICATE_ID"
+    return any(
+        re.search(regex_a, segment, re.IGNORECASE) or re.search(regex_b, segment, re.IGNORECASE) for segment in segments
+    )
+
+
+def compute_ngram_overlap_spacy(string1, string2, n):
+    doc1 = nlp(string1)
+    doc2 = nlp(string2)
+
+    ngrams1 = [" ".join([token.text for token in doc1[i : i + n]]) for i in range(len(doc1) - n + 1)]
+    ngrams2 = [" ".join([token.text for token in doc2[i : i + n]]) for i in range(len(doc2) - n + 1)]
+
+    overlap = sum((Counter(ngrams1) & Counter(ngrams2)).values())
+    return overlap
+
+
+def compute_character_ngram_overlap(str1, str2, n):
+    ngrams1 = [str1[i : i + n] for i in range(len(str1) - n + 1)]
+    ngrams2 = [str2[i : i + n] for i in range(len(str2) - n + 1)]
+    overlap = sum((Counter(ngrams1) & Counter(ngrams2)).values())
+    return overlap
+
+
+def compute_common_length(str1, str2, prefix=True):
+    length = 0
+    min_length = min(len(str1), len(str2))
+    if prefix:
+        for i in range(min_length):
+            if str1[i] == str2[i]:
+                length += 1
+            else:
+                break
+    else:
+        for i in range(1, min_length + 1):
+            if str1[-i] == str2[-i]:
+                length += 1
+            else:
+                break
+    return length
+
+
+def compute_numeric_token_overlap(str1, str2):
+    doc1 = nlp(str1)
+    doc2 = nlp(str2)
+
+    tokens1 = [token.text for token in doc1 if token.like_num]
+    tokens2 = [token.text for token in doc2 if token.like_num]
+
+    overlap = sum((Counter(tokens1) & Counter(tokens2)).values())
+    return overlap
+
+
+def get_lang_features(base_name: str, referenced_name: str) -> tuple:
+    common_numeric_words = compute_numeric_token_overlap(base_name, referenced_name)
+    common_words = compute_ngram_overlap_spacy(base_name, referenced_name, 1)
+    bigram_overlap = compute_ngram_overlap_spacy(base_name, referenced_name, 2)
+    trigram_overlap = compute_ngram_overlap_spacy(base_name, referenced_name, 3)
+    common_prefix_len = compute_common_length(base_name, referenced_name, True)
+    common_suffix_len = compute_common_length(base_name, referenced_name, False)
+    character_bigram_overlap = compute_character_ngram_overlap(base_name, referenced_name, 2)
+    character_trigram_overlap = compute_character_ngram_overlap(base_name, referenced_name, 3)
+    base_len = len(base_name)
+    referenced_len = len(referenced_name)
+    len_difference = abs(base_len - referenced_len)
+
+    return (
+        common_numeric_words,
+        common_words,
+        bigram_overlap,
+        trigram_overlap,
+        common_prefix_len,
+        common_suffix_len,
+        character_bigram_overlap,
+        character_trigram_overlap,
+        base_len,
+        referenced_len,
+        len_difference,
+    )
+
+
+def extract_segments(
+    cc_dset: CCDataset,
+    mode: REF_ANNOTATION_MODES,
+    n_sents_before: int = 2,
+    n_sents_after: int = 1,
+) -> pd.DataFrame:
+    logger.info("Extracting segments.")
+    df = ReferenceSegmentExtractor(n_sents_before, n_sents_after)(list(cc_dset.certs.values()))
+    if mode == "training":
+        return df.loc[(df.label.notnull()) & ((df.split == "train") | (df.split == "valid"))]
+    elif mode == "evaluation":
+        return df.loc[df.label.notnull()]
+    elif mode == "production":
+        return df
+    else:
+        raise ValueError(f"Unknown mode {mode}")
+
+
+def _build_transformer_embeddings(
+    segments: pd.DataFrame, mode: REF_ANNOTATION_MODES, model_path: Path | None = None
+) -> tuple[pd.DataFrame, ReferenceAnnotator]:
+    should_save_model = model_path is not None
+    annotator = None
+    logger.info("Building transformer embeddings.")
+    if model_path:
+        try:
+            annotator = ReferenceAnnotator.from_pretrained(model_path)
+            should_save_model = False
+        except Exception:
+            print(f"Failed to load ReferenceAnnotator from {model_path}.")
+            should_save_model = True
+
+    if not annotator:
+        print("Training ReferenceAnnotator from scratch.")
+        trainer = ReferenceAnnotatorTrainer.from_df(
+            segments,
+            prec_recall_metric,
+            mode=mode,
+            n_iterations=8,
+            n_epochs=2,
+            learning_rate=1.23e-5,
+            batch_size=16,
+            segmenter_metric="balanced_accuracy",
+            ensemble_soft_voting_power=2,
+            show_progress_bar=False,
+        )
+        trainer.train()
+        annotator = trainer.clf
+    assert annotator is not None
+
+    if should_save_model and model_path:
+        annotator.save_pretrained(model_path)
+
+    return (
+        segments.copy().assign(embeddings=lambda df_: df_.segments.map(annotator._model.model_body.encode)),
+        annotator,
+    )
+
+
+def _build_tf_idf_embeddings(segments: pd.DataFrame, mode: REF_ANNOTATION_MODES) -> pd.DataFrame:
+    def choose_values_to_fit(df_: pd.DataFrame) -> list[str]:
+        if mode == "training":
+            return df_.loc[df_.split == "train"].copy().explode("segments").segments.values
+        elif mode == "evaluation":
+            return df_.loc[df_.split != "test"].copy().explode("segments").segments.values
+        elif mode == "production":
+            return df_.copy().explode("segments").segments.values
+        else:
+            raise ValueError(f"Unknown mode {mode}")
+
+    logger.info("Building TF-IDF embeddings.")
+    tf_idf = TfidfVectorizer()
+    tf_idf = tf_idf.fit(choose_values_to_fit(segments))
+
+    return segments.copy().assign(
+        embeddings=lambda df_: df_.segments.map(lambda x: tf_idf.transform(x).toarray().tolist())
+    )
+
+
+def build_embeddings(
+    segments: pd.DataFrame,
+    mode: REF_ANNOTATION_MODES,
+    method: REF_EMBEDDING_METHOD,
+    model_path: Path | None = None,
+) -> tuple[pd.DataFrame, ReferenceAnnotator | None]:
+    if method == "transformer":
+        return _build_transformer_embeddings(segments, mode, model_path)
+    if method == "tf_idf":
+        return _build_tf_idf_embeddings(segments, mode), None
+    raise ValueError(f"Unknown embedding method {method}")
+
+
+def extract_language_features(df: pd.DataFrame, cc_dset: CCDataset) -> pd.DataFrame:
+    logger.info("Extracting language features.")
+    certs = list(cc_dset.certs.values())
+    dgst_to_cert_name = {x.dgst: x.name for x in certs}
+    cert_id_to_cert_name = {x.heuristics.cert_id: x.name for x in certs}
+    dgst_to_extracted_versions = {x.dgst: x.heuristics.extracted_versions for x in certs}
+    cert_id_to_extracted_versions = {x.heuristics.cert_id: x.heuristics.extracted_versions for x in certs}
+
+    df_lang = (
+        df.copy()
+        .assign(
+            cert_name=lambda df_: df_.dgst.map(dgst_to_cert_name),
+            referenced_cert_name=lambda df_: df_.canonical_reference_keyword.map(cert_id_to_cert_name),
+            cert_versions=lambda df_: df_.dgst.map(dgst_to_extracted_versions),
+            referenced_cert_versions=lambda df_: df_.canonical_reference_keyword.map(cert_id_to_extracted_versions),
+            cert_name_stripped_version=lambda df_: df_.apply(
+                lambda x: strip_all(x["cert_name"], x["cert_versions"]), axis=1
+            ),
+            referenced_cert_name_stripped_version=lambda df_: df_.apply(
+                lambda x: strip_all(x["referenced_cert_name"], x["referenced_cert_versions"]),
+                axis=1,
+            ),
+            lang_token_set_ratio=lambda df_: df_.apply(
+                lambda x: fuzz.token_set_ratio(
+                    x["cert_name_stripped_version"],
+                    x["referenced_cert_name_stripped_version"],
+                ),
+                axis=1,
+            ),
+            lang_partial_ratio=lambda df_: df_.apply(
+                lambda x: fuzz.partial_ratio(
+                    x["cert_name_stripped_version"],
+                    x["referenced_cert_name_stripped_version"],
+                ),
+                axis=1,
+            ),
+            lang_token_sort_ratio=lambda df_: df_.apply(
+                lambda x: fuzz.token_sort_ratio(
+                    x["cert_name_stripped_version"],
+                    x["referenced_cert_name_stripped_version"],
+                ),
+                axis=1,
+            ),
+            lang_n_segments=lambda df_: df_.segments.map(lambda x: len(x) if x else 0),
+            lang_matches_recertification=lambda df_: df_.segments.map(matches_recertification),
+        )
+        .assign(
+            lang_n_extracted_versions=lambda df_: df_.cert_versions.map(lambda x: len(x) if x else 0),
+            lang_n_intersection_versions=lambda df_: df_.apply(
+                lambda x: len(set(x["cert_versions"]).intersection(set(x["referenced_cert_versions"]))),
+                axis=1,
+            ),
+        )
+    )
+
+    df_lang_other_features = df_lang.apply(
+        lambda row: get_lang_features(row["cert_name"], row["referenced_cert_name"]),
+        axis=1,
+    ).apply(pd.Series)
+    lang_features = [
+        "common_numeric_words",
+        "common_words",
+        "bigram_overlap",
+        "trigram_overlap",
+        "common_prefix_len",
+        "common_suffix_len",
+        "character_bigram_overlap",
+        "character_trigram_overlap",
+        "base_len",
+        "referenced_len",
+        "len_difference",
+    ]
+    df_lang_other_features.columns = ["lang_" + x for x in lang_features]
+
+    df_lang = pd.concat([df_lang, df_lang_other_features], axis=1).assign(
+        lang_should_not_be_component=lambda df_: df_.apply(
+            lambda x: x.lang_len_difference < 5 and x.lang_token_set_ratio == 100,
+            axis=1,
+        ),
+    )
+    for col in df_lang.columns:
+        if col.startswith("pred_"):
+            df_lang[col] = df_lang[col] / df_lang.lang_n_segments
+
+    return df_lang
+
+
+def perform_dimensionality_reduction(
+    df: pd.DataFrame,
+    mode: REF_ANNOTATION_MODES,
+    umap_n_neighbors: int = 10,
+    umap_min_dist: float = 0.51026,
+    umap_metric: Literal["cosine", "euclidean", "manhattan"] = "cosine",
+) -> pd.DataFrame:
+    def choose_values_to_fit(df_: pd.DataFrame):
+        if mode == "training":
+            return df_.loc[df_.split == "train"].copy().embeddings.values
+        elif mode == "evaluation":
+            return df_.loc[df_.split != "test"].copy().embeddings.values
+        elif mode == "production":
+            return df_.copy().embeddings.values
+        else:
+            raise ValueError(f"Unknown mode {mode}")
+
+    def choose_labels_to_fit(df_: pd.DataFrame):
+        if mode == "training":
+            return df_.loc[df_.split == "train"].copy().label.values
+        elif mode == "evaluation":
+            return df_.loc[df_.split != "test"].copy().label.values
+        elif mode == "production":
+            return df_.copy().label.values
+        else:
+            raise ValueError(f"Unknown mode {mode}")
+
+    logger.info("Performing dimensionality reduction.")
+    df_exploded = df.copy().explode(["segments", "embeddings"]).reset_index(drop=True)
+    label_encoder = LabelEncoder()
+
+    embeddings_to_fit = np.vstack(choose_values_to_fit(df_exploded))
+    labels_to_fit = label_encoder.fit_transform(choose_labels_to_fit(df_exploded))
+
+    scaler = StandardScaler()
+    embeddings_to_fit_scaled = scaler.fit_transform(embeddings_to_fit)
+
+    # parallel UMAP not available with random state
+    umapper = umap.UMAP(
+        n_neighbors=umap_n_neighbors,
+        min_dist=umap_min_dist,
+        metric=umap_metric,
+        random_state=RANDOM_STATE,
+        n_jobs=1,
+    ).fit(embeddings_to_fit, y=labels_to_fit)
+    pca_mapper = PCA(n_components=2, random_state=RANDOM_STATE).fit(embeddings_to_fit_scaled, y=labels_to_fit)
+
+    all_embeddings = np.vstack(df.embeddings.values)
+    all_embeddings_scaled = scaler.transform(all_embeddings)
+
+    df_exploded["umap"] = umapper.transform(all_embeddings).tolist()
+    df_exploded["pca"] = pca_mapper.transform(all_embeddings_scaled).tolist()
+
+    return (
+        df_exploded.groupby(["dgst", "canonical_reference_keyword"])
+        .agg(
+            {
+                "segments": lambda x: x.tolist(),
+                "actual_reference_keywords": "first",
+                "label": "first",
+                "split": "first",
+                "embeddings": lambda x: x.tolist(),
+                "umap": lambda x: x.tolist(),
+                "pca": lambda x: x.tolist(),
+            }
+        )
+        .reset_index()
+    )
+
+
+def extract_prediction_features(df: pd.DataFrame, model) -> pd.DataFrame:
+    def get_setfit_prediction_numbers(val):
+        counter = Counter(val.tolist())
+        return [counter[x] for x in range(len(all_labels))]
+
+    logger.info("Extracting prediction features.")
+    df["annotator_predictions"] = df.segments.map(lambda x: model.predict(x))
+    all_labels = set(itertools.chain.from_iterable(x.tolist() for x in df.annotator_predictions.values))
+
+    df_features_pred = df.annotator_predictions.apply(get_setfit_prediction_numbers).apply(pd.Series)
+    feature_names = [f"pred_{x}" for x in range(len(all_labels))]
+    df_features_pred.columns = feature_names
+    return pd.concat([df, df_features_pred], axis=1)
+
+
+def extract_geometrical_features(df: pd.DataFrame) -> pd.DataFrame:
+    def extract_features(points):
+        # Convert list of points to a numpy array
+        points = np.array(points)
+        xs = points[:, 0]
+        ys = points[:, 1]
+
+        # Basic Descriptive Statistics
+        mean_x, mean_y = np.mean(xs), np.mean(ys)
+        var_x, var_y = np.var(xs), np.var(ys)
+        std_x, std_y = np.std(xs), np.std(ys)
+        if len(points) > 1:
+            skew_x, skew_y = skew(xs), skew(ys)
+            kurt_x, kurt_y = kurtosis(xs), kurtosis(ys)
+        else:
+            skew_x, skew_y = 0, 0
+            kurt_x, kurt_y = 0, 0
+
+        # Spatial Spread
+        range_x, range_y = np.ptp(xs), np.ptp(ys)
+        cov_xy = np.cov(xs, ys)[0, 1] if len(points) > 1 else 0
+        median_x, median_y = np.median(xs), np.median(ys)
+
+        # Distance-based Features
+        centroid = [mean_x, mean_y]
+        distances_to_centroid = np.linalg.norm(points - centroid, axis=1) if len(points) > 1 else [0]
+        mean_distance = np.mean(distances_to_centroid)
+        max_distance = np.max(distances_to_centroid)
+        min_distance = np.min(distances_to_centroid)
+        std_distance = np.std(distances_to_centroid)
+        max_min_distance = max_distance - min_distance
+
+        sorted_points = points[np.argsort(distances_to_centroid)]
+        total_distance = np.sum(np.linalg.norm(sorted_points[1:] - sorted_points[:-1], axis=1))
+
+        # Geometric Features
+        hull_area, hull_perimeter = (0, 0)
+        if len(points) > 2:  # ConvexHull needs at least 3 points
+            try:
+                hull = ConvexHull(points)
+                hull_area = hull.volume
+                hull_perimeter = hull.area
+            except QhullError:
+                pass
+
+        pairwise_distances = distance_matrix(points, points) if len(points) > 1 else np.array([[0]])
+        mean_pairwise_distance = np.mean(pairwise_distances)
+        max_pairwise_distance = np.max(pairwise_distances)
+
+        if len(points) > 1:
+            min_coords = np.min(points, axis=0)
+            max_coords = np.max(points, axis=0)
+            bounding_box_width = max_coords[0] - min_coords[0]
+            bounding_box_height = max_coords[1] - min_coords[1]
+            bounding_box_area = bounding_box_width * bounding_box_height
+
+            aspect_ratio = bounding_box_width / bounding_box_height if bounding_box_height != 0 else 1
+            point_density = len(points) / bounding_box_area
+        else:
+            aspect_ratio = 0
+            point_density = 0
+
+        # Gather all features into a list
+        features = [
+            mean_x,
+            mean_y,
+            var_x,
+            var_y,
+            std_x,
+            std_y,
+            skew_x,
+            skew_y,
+            kurt_x,
+            kurt_y,
+            range_x,
+            range_y,
+            cov_xy,
+            median_x,
+            median_y,
+            mean_distance,
+            max_distance,
+            min_distance,
+            max_min_distance,
+            std_distance,
+            total_distance,
+            hull_area,
+            hull_perimeter,
+            mean_pairwise_distance,
+            max_pairwise_distance,
+            aspect_ratio,
+            point_density,
+        ]
+
+        return features
+
+    feature_names = [
+        "mean_x",
+        "mean_y",
+        "var_x",
+        "var_y",
+        "std_x",
+        "std_y",
+        "skew_x",
+        "skew_y",
+        "kurt_x",
+        "kurt_y",
+        "range_x",
+        "range_y",
+        "cov_xy",
+        "median_x",
+        "median_y",
+        "mean_distance_to_centroid",
+        "max_distance_to_centroid",
+        "min_distance_to_centroid",
+        "max_min_distance_to_centroid",
+        "std_distance_to_centroid",
+        "total_distances_to_centroid",
+        "hull_area",
+        "hull_perimeter",
+        "mean_pairwise_distance",
+        "max_pairwise_distance",
+        "aspect_ratio",
+        "point_density",
+    ]
+
+    logger.info("Extracting geometrical features.")
+    df_features_pca = df.pca.apply(extract_features).apply(pd.Series)
+    feature_names_pca = ["pca_" + x for x in feature_names]
+    df_features_pca.columns = feature_names_pca
+
+    df_features_umap = df.umap.apply(extract_features).apply(pd.Series)
+    feature_names_umap = ["umap_" + x for x in feature_names]
+    df_features_umap.columns = feature_names_umap
+
+    return pd.concat([df, df_features_pca, df_features_umap], axis=1)
+
+
+def _choose_feature_columns(
+    df: pd.DataFrame, use_pca: bool = True, use_umap: bool = True, use_lang: bool = True, use_pred: bool = True
+) -> list[str]:
+    feature_columns = []
+    if not use_pca and not use_umap and not use_lang and not use_pred:
+        raise ValueError("At least one of PCA, UMAP or language features must be used.")
+    if use_pca:
+        feature_columns.extend([x for x in df.columns if x.startswith("pca_")])
+    if use_umap:
+        feature_columns.extend([x for x in df.columns if x.startswith("umap_")])
+    if use_lang:
+        feature_columns.extend([x for x in df.columns if x.startswith("lang_")])
+    if use_pred:
+        feature_columns.extend([x for x in df.columns if x.startswith("pred_")])
+    return feature_columns
+
+
+def _split_df(df: pd.DataFrame, mode: REF_ANNOTATION_MODES) -> tuple[pd.DataFrame, pd.DataFrame | None]:
+    if mode == "training":
+        train_df = df.loc[df.split == "train"].copy()
+        eval_df = df.loc[df.split == "valid"].copy()
+    elif mode == "evaluation":
+        train_df = df.loc[df.split != "test"].copy()
+        eval_df = df.loc[df.split == "test"].copy()
+    elif mode == "production":
+        train_df = df.copy()
+        eval_df = df.copy()
+    elif mode == "cross-validation":
+        train_df = df.loc[df.split != "test"].copy()
+        eval_df = None
+    else:
+        raise ValueError(f"Unknown mode {mode}")
+    return train_df, eval_df
+
+
+def dataframe_to_training_arrays(
+    df: pd.DataFrame,
+    mode: REF_ANNOTATION_MODES,
+    use_pca: bool = True,
+    use_umap: bool = True,
+    use_lang: bool = True,
+    use_pred: bool = True,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray | None, np.ndarray | None, list[str]]:
+    feature_columns = _choose_feature_columns(df, use_pca, use_umap, use_lang, use_pred)
+    train_df, eval_df = _split_df(df.loc[df.label.notnull()].copy(), mode)
+
+    x_train, y_train = (
+        np.vstack(train_df[feature_columns].values),
+        train_df.label.values,
+    )
+    if eval_df is not None:
+        x_valid, y_valid = (
+            np.vstack(eval_df[feature_columns].values),
+            eval_df.label.values,
+        )
+    else:
+        x_valid, y_valid = None, None
+
+    return (
+        x_train,
+        y_train,
+        x_valid,
+        y_valid,
+        feature_columns,
+    )
diff --git a/src/sec_certs/model/references_nlp/segment_extractor.py b/src/sec_certs/model/references_nlp/segment_extractor.py
new file mode 100644
index 00000000..d63208ec
--- /dev/null
+++ b/src/sec_certs/model/references_nlp/segment_extractor.py
@@ -0,0 +1,368 @@
+from __future__ import annotations
+
+import itertools
+import json
+import logging
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from importlib.resources import files
+from pathlib import Path
+from typing import Any, Literal
+
+# import langdetect
+import numpy as np
+import pandas as pd
+import spacy
+
+from sec_certs.sample.cc import CCCertificate
+from sec_certs.sample.cc_certificate_id import CertificateId
+from sec_certs.utils import parallel_processing
+
+nlp = spacy.load("en_core_web_sm")
+logger = logging.getLogger(__name__)
+
+
+def swap_and_filter_dict(dct: dict[str, Any], filter_to_keys: set[str]):
+    new_dct: dict[str, set[str]] = {}
+    for key, val in dct.items():
+        if val in new_dct:
+            new_dct[val].add(key)
+        else:
+            new_dct[val] = {key}
+
+    return {key: frozenset(val) for key, val in new_dct.items() if key in filter_to_keys}
+
+
+def fill_reference_segments(record: ReferenceRecord, n_sent_before: int = 2, n_sent_after: int = 1) -> ReferenceRecord:
+    """
+    Compute indices of the sentences containing the reference keyword, take their surrounding sentences and join them.
+    """
+
+    def compute_surroundings(hit_index: int, max_index: int, n_before: int, n_after: int):
+        """
+        Computes indices of sentences to join into a coherent paragraph based on their location in text.
+        Ideally we would like to take (hit_index - n_before, hit_index + n_after), but we need to make sure
+        that we do not go out of bounds.
+        """
+        lower = max(0, hit_index - n_before)
+        upper = min(max_index, hit_index + n_after)
+        return range(lower, upper + 1)
+
+    with record.processed_data_source_path.open("r") as handle:
+        data = handle.read()
+
+    sents = [sent.text for sent in nlp(data).sents]
+    indices_of_relevant_sents = [sents.index(x) for x in sents if any(y in x for y in record.actual_reference_keywords)]
+
+    if not indices_of_relevant_sents:
+        record.segments = None
+        return record
+
+    sequences_to_take = [
+        compute_surroundings(x, len(sents) - 1, n_sent_before, n_sent_after) for x in indices_of_relevant_sents
+    ]
+    record.segments = {"".join([sents[y] for y in x]) for x in sequences_to_take}
+
+    return record
+
+
+def preprocess_data_source(record: ReferenceRecord) -> ReferenceRecord:
+    # TODO: There's some space for improvement, the preprocessing is acutally run twice.
+
+    with record.raw_data_source_path.open("r") as handle:
+        data = handle.read()
+
+    processed_data = preprocess_txt_func(data, record.actual_reference_keywords)
+
+    with record.processed_data_source_path.open("w") as handle:
+        handle.write(processed_data)
+
+    return record
+
+
+def find_bracket_pattern(sentences: set[str], actual_reference_keywords: frozenset[str]):
+    patterns = [r"(\[.+?\])(?=.*" + x + r")" for x in actual_reference_keywords]
+    res: list[tuple[str, str]] = []
+
+    for sent in sentences:
+        for pattern, keyword in zip(patterns, actual_reference_keywords):
+            matches = re.findall(pattern, sent, flags=re.MULTILINE | re.UNICODE | re.DOTALL)
+            if matches:
+                res.append((matches[-1], keyword))
+    return res
+
+
+def preprocess_txt_func(data: str, actual_reference_keywords: frozenset[str]) -> str:
+    data = replace_acronyms(data)
+    data = replace_citation_identifiers(data, actual_reference_keywords)
+    return data
+
+
+def replace_citation_identifiers(data: str, actual_reference_keywords: frozenset[str]) -> str:
+    segments = {sent.text for sent in nlp(data).sents if any(x in sent.text for x in actual_reference_keywords)}
+    patterns_to_replace = find_bracket_pattern(segments, actual_reference_keywords)
+    for x in patterns_to_replace:
+        data = data.replace(x[0], x[1])
+    return data
+
+
+def replace_acronyms(text: str) -> str:
+    acronym_replacements = {
+        "TOE": "target of evaluation",
+        "CC": "certification framework",
+        "PP": "protection profile",
+        "ST": "security target",
+        "SFR": "security Functional Requirement",
+        "SFRs": "security Functional Requirements",
+        "IC": "integrated circuit",
+        "MRTD": "machine readable travel document",
+        "TSF": "security functions of target of evaluation",
+        "PACE": "password authenticated connection establishment",
+    }
+
+    for acronym, replacement in acronym_replacements.items():
+        pattern = rf"(?<!\S){re.escape(acronym)}(?!\S)"
+        text = re.sub(pattern, replacement, text)
+
+    return text
+
+
+@dataclass
+class ReferenceRecord:
+    """
+    Data structure to hold objects when extracting text segments from txt files relevant for reference annotations.
+    """
+
+    certificate_dgst: str
+    raw_data_source_path: Path
+    processed_data_source_path: Path
+    canonical_reference_keyword: str
+    actual_reference_keywords: frozenset[str]
+    source: str
+    segments: set[str] | None = None
+
+    def to_pandas_tuple(self) -> tuple[str, str, frozenset[str], str, set[str] | None]:
+        return (
+            self.certificate_dgst,
+            self.canonical_reference_keyword,
+            self.actual_reference_keywords,
+            self.source,
+            self.segments,
+        )
+
+
+class ReferenceSegmentExtractor:
+    """
+    Class to process list of certificates into a dataframe that holds reference segments.
+    Should be only called with ReferenceSegmentExtractor()(list_of_certificates)
+    """
+
+    def __init__(self, n_sents_before: int = 1, n_sents_after: int = 0):
+        self.n_sents_before = n_sents_before
+        self.n_sents_after = n_sents_after
+
+    def __call__(self, certs: Iterable[CCCertificate]) -> pd.DataFrame:
+        return self._prepare_df_from_cc_dset(certs)
+
+    def _prepare_df_from_cc_dset(self, certs: Iterable[CCCertificate]) -> pd.DataFrame:
+        """
+        Prepares processed DataFrame for reference annotator training from a list of certificates. This method:
+        - Extracts text segments relevant for each reference out of the certificates, forms dataframe from those
+        - Loads data splits into train/valid/test (unseen certificates are put into test set)
+        - Loads manually annotated samples
+        - Combines all of that into single dataframe
+        """
+        target_certs = [x for x in certs if x.heuristics.st_references.directly_referencing and x.state.st_txt_path]
+        report_certs = [
+            x for x in certs if x.heuristics.report_references.directly_referencing and x.state.report_txt_path
+        ]
+        df_targets = self._build_df(target_certs, "target")
+        df_reports = self._build_df(report_certs, "report")
+        print(f"df_targets shape: {df_targets.shape}")
+        print(f"df_reports shape: {df_reports.shape}")
+
+        return ReferenceSegmentExtractor._process_df(pd.concat([df_targets, df_reports]), certs)
+
+    def _build_records(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> list[ReferenceRecord]:
+        def get_cert_records(cert: CCCertificate, source: Literal["target", "report"]) -> list[ReferenceRecord]:
+            canonical_ref_var = {
+                "target": "st_references",
+                "report": "report_references",
+            }
+            actual_ref_var = {"target": "st_keywords", "report": "report_keywords"}
+            raw_source_var = {"target": "st_txt_path", "report": "report_txt_path"}
+
+            canonical_references = getattr(cert.heuristics, canonical_ref_var[source]).directly_referencing
+            actual_references = getattr(cert.pdf_data, actual_ref_var[source])["cc_cert_id"]
+            actual_references = {
+                inner_key: CertificateId(outer_key, inner_key).canonical
+                for outer_key, val in actual_references.items()
+                for inner_key in val
+            }
+            actual_references = swap_and_filter_dict(actual_references, canonical_references)
+
+            raw_source_dir = getattr(cert.state, raw_source_var[source]).parent
+            processed_source_dir = raw_source_dir.parent / "txt_processed"
+
+            return [
+                ReferenceRecord(
+                    cert.dgst,
+                    raw_source_dir / f"{cert.dgst}.txt",
+                    processed_source_dir / f"{cert.dgst}.txt",
+                    key,
+                    val,
+                    source,
+                )
+                for key, val in actual_references.items()
+            ]
+
+        (certs[0].state.report_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True)
+        (certs[0].state.st_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True)
+        return list(itertools.chain.from_iterable(get_cert_records(cert, source) for cert in certs))
+
+    def _build_df(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> pd.DataFrame:
+        records = self._build_records(certs, source)
+
+        records = parallel_processing.process_parallel(
+            preprocess_data_source,
+            records,
+            use_threading=False,
+            progress_bar=True,
+            progress_bar_desc="Preprocessing data",
+        )
+        records_with_args = [(x, self.n_sents_before, self.n_sents_after) for x in records]
+
+        results = parallel_processing.process_parallel(
+            fill_reference_segments,
+            records_with_args,
+            unpack=True,
+            use_threading=False,
+            progress_bar=True,
+            progress_bar_desc="Recovering reference segments",
+        )
+
+        print(f"I now have {len(results)} in {source} mode")
+        return pd.DataFrame.from_records(
+            [x.to_pandas_tuple() for x in results],
+            columns=[
+                "dgst",
+                "canonical_reference_keyword",
+                "actual_reference_keywords",
+                "source",
+                "segments",
+            ],
+        )
+
+    @staticmethod
+    def _get_split_dict() -> dict[str, str]:
+        """
+        Returns dictionary that maps dgst: split, where split in `train`, `valid`, `test`
+        """
+
+        def get_single_dct(pth: Path, split_name: str) -> dict[str, str]:
+            with pth.open("r") as handle:
+                return dict.fromkeys(json.load(handle), split_name)
+
+        split_directory = Path(str(files("sec_certs.data") / "reference_annotations/split/"))
+        return {
+            **get_single_dct(split_directory / "train.json", "train"),
+            **get_single_dct(split_directory / "valid.json", "valid"),
+            **get_single_dct(split_directory / "test.json", "test"),
+        }
+
+    @staticmethod
+    def _get_annotations_dict() -> dict[tuple[str, str], str]:
+        """
+        Returns dictionary mapping tuples `(dgst, canonical_reference_keyword) -> label`
+        """
+
+        def load_single_df(pth: Path, split_name: str) -> pd.DataFrame:
+            return (
+                pd.read_csv(
+                    pth,
+                    na_values=["None"],
+                    dtype={
+                        "dgst": str,
+                        "canonical_reference_keyword": str,
+                        "source": str,
+                        "label": str,
+                        "comment": str,
+                    },
+                )
+                .dropna(subset="label")
+                .assign(
+                    label=lambda df_: df_.label.str.replace(" ", "_").str.upper(),
+                    split=split_name,
+                )
+            )
+
+        annotations_directory = Path(str(files("sec_certs.data") / "reference_annotations/final/"))
+        df_annot = pd.concat(
+            [
+                load_single_df(annotations_directory / "train.csv", "train"),
+                load_single_df(annotations_directory / "valid.csv", "valid"),
+                load_single_df(annotations_directory / "test.csv", "test"),
+            ]
+        )
+
+        return (
+            df_annot[["dgst", "canonical_reference_keyword", "label"]]
+            .set_index(["dgst", "canonical_reference_keyword"])
+            .label.to_dict()
+        )
+
+    @staticmethod
+    def _process_df(df: pd.DataFrame, certs: Iterable[CCCertificate]) -> pd.DataFrame:
+        def process_segment(segment: str, actual_reference_keywords: frozenset[str]) -> str:
+            segment = " ".join(segment.split())
+            for ref_id in actual_reference_keywords:
+                segment = segment.replace(ref_id, "REFERENCED_CERTIFICATE_ID")
+            return segment
+
+        def unique_elements(series):
+            combined = [item for sublist in series for item in sublist]
+            return list(set(combined))
+
+        """
+        Fully processes the dataframe.
+        """
+        annotations_dict = ReferenceSegmentExtractor._get_annotations_dict()
+        split_dct = ReferenceSegmentExtractor._get_split_dict()
+        logger.info(f"Deleting {df.loc[df.segments.isnull()].shape[0]} rows with no segments.")
+
+        df_new = df.copy()
+        df_new["full_key"] = df_new.apply(lambda x: (x["dgst"], x["canonical_reference_keyword"]), axis=1)
+        to_delete = len(df_new.loc[df_new.segments.isnull()].full_key.unique())
+        print(
+            f"Deleting records for {to_delete} unique (dgst, referenced_id) pairs, not necessarily labeled ones. These have empty segments."
+        )
+
+        df_processed = (
+            df.loc[df.segments.notnull()]
+            .explode("segments")
+            # .assign(lang=lambda df_: df_.segments.map(langdetect.detect))
+            # .loc[lambda df_: df_.lang.isin({"en", "fr", "de"})]  # This could get disabled possibly.
+            .groupby(
+                ["dgst", "canonical_reference_keyword"],
+                as_index=False,
+                dropna=False,
+            )
+            .agg({"segments": list, "actual_reference_keywords": unique_elements})
+            .assign(
+                actual_reference_keywords=lambda df_: df_.actual_reference_keywords.map(list),
+                label=lambda df_: [
+                    annotations_dict.get(x) for x in zip(df_["dgst"], df_["canonical_reference_keyword"])
+                ],
+                split=lambda df_: df_.dgst.map(split_dct),
+            )
+            .assign(
+                label=lambda df_: df_.label.map(lambda x: x if x is not None else np.nan),
+                split=lambda df_: df_.split.map(lambda x: "test" if pd.isnull(x) else x),
+            )
+        )
+        df_processed.segments = df_processed.apply(
+            lambda row: [process_segment(x, row.actual_reference_keywords) for x in row.segments],
+            axis=1,
+        )
+        return df_processed
diff --git a/src/sec_certs/model/references_nlp/training.py b/src/sec_certs/model/references_nlp/training.py
new file mode 100644
index 00000000..e0abe94f
--- /dev/null
+++ b/src/sec_certs/model/references_nlp/training.py
@@ -0,0 +1,95 @@
+import logging
+import os
+
+import numpy as np
+import pandas as pd
+from catboost import CatBoostClassifier, Pool
+from sklearn.dummy import DummyClassifier
+from sklearn.metrics import balanced_accuracy_score
+from sklearn.model_selection import KFold
+
+from sec_certs.constants import RANDOM_STATE, REF_ANNOTATION_MODES
+from sec_certs.model.references_nlp.feature_extraction import dataframe_to_training_arrays
+
+logger = logging.getLogger(__name__)
+
+
+def _train_model(
+    mode: REF_ANNOTATION_MODES,
+    x_train: np.ndarray,
+    y_train: np.ndarray,
+    x_eval: np.ndarray | None = None,
+    y_eval: np.ndarray | None = None,
+    learning_rate: float = 0.03,
+    depth: int = 6,
+    l2_leaf_reg: float = 3,
+):
+    # In production mode, we don't have early stopping on validation set. Hence we use number of iterations that worked during evaluation.
+    n_iters = 20 if mode == "production" else 1000
+    clf = CatBoostClassifier(
+        learning_rate=learning_rate,
+        depth=depth,
+        l2_leaf_reg=l2_leaf_reg,
+        task_type="GPU",
+        devices=os.environ["CUDA_VISIBLE_DEVICES"],
+        random_seed=RANDOM_STATE,
+        iterations=n_iters,
+    )
+
+    train_pool = Pool(x_train, y_train)
+    eval_pool = Pool(x_eval, y_eval) if x_eval is not None else None
+    clf.fit(
+        train_pool,
+        eval_set=eval_pool,
+        verbose=False,
+        plot=True,
+        early_stopping_rounds=100,
+        use_best_model=True,
+    )
+    return clf
+
+
+def train_model(
+    mode: REF_ANNOTATION_MODES,
+    x_train: np.ndarray,
+    y_train: np.ndarray,
+    x_eval: np.ndarray | None = None,
+    y_eval: np.ndarray | None = None,
+    train_baseline: bool = False,
+    learning_rate: float = 0.079573,
+    depth: int = 10,
+    l2_leaf_reg: float = 7.303517,
+) -> DummyClassifier | CatBoostClassifier:
+    logger.info(f"Training model with baselne={train_baseline}")
+
+    if train_baseline:
+        clf = DummyClassifier(random_state=RANDOM_STATE)
+        clf.fit(x_train, y_train)
+    else:
+        clf = _train_model(
+            mode,
+            x_train,
+            y_train,
+            x_eval,
+            y_eval,
+            learning_rate,
+            depth,
+            l2_leaf_reg,
+        )
+    return clf
+
+
+def cross_validate_model(
+    mode: REF_ANNOTATION_MODES, df: pd.DataFrame, learning_rate: float = 0.03, depth: int = 6, l2_leaf_reg: int = 3
+) -> float:
+    logger.info("Cross-validating model")
+    X_train, y_train, _, _, _ = dataframe_to_training_arrays(df, "cross-validation", True, True, True, True)
+    kf = KFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)
+    scores = []
+    for train_index, test_index in kf.split(X_train):
+        X_train_, X_test_ = X_train[train_index], X_train[test_index]
+        y_train_, y_test_ = y_train[train_index], y_train[test_index]
+        clf = _train_model(mode, X_train_, y_train_, X_test_, y_test_, learning_rate, depth, l2_leaf_reg)
+        scores.append(balanced_accuracy_score(y_test_, clf.predict(X_test_)))
+
+    return np.mean(scores)
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index f2087eeb..32c875c8 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -434,6 +434,10 @@ class CCCertificate(
         cert_id: str | None = field(default=None)
         st_references: References = field(default_factory=References)
         report_references: References = field(default_factory=References)
+
+        # Contains direct outward references merged from both st, and report sources, annotated with ReferenceAnnotator
+        # TODO: Reference meanings as Enum if we work with it further.
+        annotated_references: dict[str, str] | None = field(default=None)
         extracted_sars: set[SAR] | None = field(default=None)
         direct_transitive_cves: set[str] | None = field(default=None)
         indirect_transitive_cves: set[str] | None = field(default=None)
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index d6e49718..254c8e05 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -14,6 +14,13 @@ class CertificateId:
     raw: str
 
     def _canonical_fr(self) -> str:
+        def pad_last_segment_with_zero(id_str: str) -> str:
+            splitted = id_str.split("/")
+            if len(splitted) > 1:
+                num = splitted[-1].zfill(2)
+                return f"{''.join(splitted[:-1])}/{num}"
+            return id_str
+
         new_cert_id = self.clean
         rules = [
             "(?:Rapport de certification|Certification Report) ([0-9]+[/-_][0-9]+(?:[vV][1-9])?(?:[_/-][MSR][0-9]+)?)",
@@ -22,7 +29,7 @@ class CertificateId:
         ]
         for rule in rules:
             if match := re.match(rule, new_cert_id):
-                return "ANSSI-CC-" + match.group(1).replace("_", "/").replace("V", "v")
+                return pad_last_segment_with_zero("ANSSI-CC-" + match.group(1).replace("_", "/").replace("V", "v"))
 
         return new_cert_id
 
diff --git a/src/sec_certs/utils/nlp.py b/src/sec_certs/utils/nlp.py
new file mode 100644
index 00000000..1e8dc911
--- /dev/null
+++ b/src/sec_certs/utils/nlp.py
@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from ast import literal_eval
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import precision_score, recall_score
+
+
+def prec_recall_metric(y_pred, y_true):
+    return {
+        "precision": precision_score(y_true, y_pred, zero_division="warn", average="weighted"),
+        "recall": recall_score(y_true, y_pred, zero_division="warn", average="weighted"),
+    }
+
+
+def softmax(x):
+    return np.exp(x - np.max(x)) / np.exp(x - np.max(x)).sum()
+
+
+def eval_strings_if_necessary(series: pd.Series) -> pd.Series:
+    return series.map(literal_eval) if isinstance(series.iloc[0], str) else series
+
+
+def filter_short_sentences(sentences, actual_reference_keywords):
+    return [x for x in sentences if len(x) > min(len(x) for x in actual_reference_keywords) + 20]
+
+
+def prepare_reference_annotations_df(df: pd.DataFrame):
+    if df.loc[(df.label != "SELF") & (df.label.notnull())].empty:
+        raise ValueError("No expert annotations found in the dataset of references.")
+    df = df.loc[lambda df_: (df_.label != "SELF") & (df_.label.notnull())].assign(
+        segments=lambda df_: eval_strings_if_necessary(df_.segments)
+    )
+    df.segments = df.apply(
+        lambda row: filter_short_sentences(row["segments"], row["actual_reference_keywords"]), axis=1
+    )
+    df = df.loc[lambda df_: df_.segments.map(len) > 0]
+    return df
diff --git a/tests/data/cc/analysis/cc_full_dataset.json b/tests/data/cc/analysis/cc_full_dataset.json
index 05e75c55..3b07f9c5 100644
--- a/tests/data/cc/analysis/cc_full_dataset.json
+++ b/tests/data/cc/analysis/cc_full_dataset.json
@@ -551,6 +551,7 @@
                     "directly_referencing": null,
                     "indirectly_referencing": null
                 },
+                "annotated_references": null,
                 "extracted_sars": {
                     "_type": "Set",
                     "elements": [
@@ -726,4 +727,4 @@
             }
         }
     ]
-}
\ No newline at end of file
+}
diff --git a/tests/data/cc/analysis/reference_dataset.json b/tests/data/cc/analysis/reference_dataset.json
index 57ae81f1..00ab6674 100644
--- a/tests/data/cc/analysis/reference_dataset.json
+++ b/tests/data/cc/analysis/reference_dataset.json
@@ -508,6 +508,7 @@
             ]
           }
         },
+        "annotated_references": null,
         "extracted_sars": {
           "_type": "Set",
           "elements": [
@@ -1105,6 +1106,7 @@
             ]
           }
         },
+        "annotated_references": null,
         "extracted_sars": {
           "_type": "Set",
           "elements": [
@@ -1736,6 +1738,7 @@
             ]
           }
         },
+        "annotated_references": null,
         "extracted_sars": {
           "_type": "Set",
           "elements": [
@@ -1841,4 +1844,4 @@
       }
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/tests/data/cc/analysis/transitive_vulnerability_dataset.json b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
index da2348bf..fb45efde 100644
--- a/tests/data/cc/analysis/transitive_vulnerability_dataset.json
+++ b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
@@ -1144,6 +1144,7 @@
             ]
           }
         },
+        "annotated_references": null,
         "extracted_sars": {
           "_type": "Set",
           "elements": [
@@ -2084,6 +2085,7 @@
             ]
           }
         },
+        "annotated_references": null,
         "extracted_sars": {
           "_type": "Set",
           "elements": [
@@ -3436,6 +3438,7 @@
             ]
           }
         },
+        "annotated_references": null,
         "extracted_sars": {
           "_type": "Set",
           "elements": [
@@ -3611,4 +3614,4 @@
       }
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/tests/data/cc/analysis/vulnerable_dataset.json b/tests/data/cc/analysis/vulnerable_dataset.json
index 1afb42bb..1b23ef0d 100644
--- a/tests/data/cc/analysis/vulnerable_dataset.json
+++ b/tests/data/cc/analysis/vulnerable_dataset.json
@@ -63,6 +63,7 @@
                     "8.2"
                 ],
                 "cpe_matches": null,
+                "annotated_references": null,
                 "verified_cpe_matches": null,
                 "related_cves": null,
                 "cert_lab": null,
@@ -118,6 +119,7 @@
                     "8.2"
                 ],
                 "cpe_matches": null,
+                "annotated_references": null,
                 "verified_cpe_matches": null,
                 "related_cves": null,
                 "cert_lab": null,
@@ -125,4 +127,4 @@
             }
         }
     ]
-}
\ No newline at end of file
+}
diff --git a/tests/data/cc/certificate/fictional_cert.json b/tests/data/cc/certificate/fictional_cert.json
index 7cf04d2d..ff95795c 100644
--- a/tests/data/cc/certificate/fictional_cert.json
+++ b/tests/data/cc/certificate/fictional_cert.json
@@ -73,6 +73,7 @@
         "related_cves": null,
         "cert_lab": null,
         "cert_id": null,
+        "annotated_references": null,
         "extracted_sars": null,
         "direct_transitive_cves": null,
         "indirect_transitive_cves": null,
@@ -95,4 +96,4 @@
     "report_link": "https://path.to/report/link",
     "st_link": "https://path.to/st/link",
     "cert_link": "https://path.to/cert/link"
-}
\ No newline at end of file
+}
diff --git a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
index 8a087e2b..4c1ff4c7 100644
--- a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
+++ b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
@@ -68,6 +68,7 @@
                     "directly_referencing": null,
                     "indirectly_referencing": null
                 },
+                "annotated_references": null,
                 "extracted_sars": null,
                 "direct_transitive_cves": null,
                 "indirect_transitive_cves": null,
@@ -77,4 +78,4 @@
             "maintenance_date": "2019-08-26"
         }
     ]
-}
\ No newline at end of file
+}
diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json
index b425235e..e32cffa4 100644
--- a/tests/data/cc/dataset/toy_dataset.json
+++ b/tests/data/cc/dataset/toy_dataset.json
@@ -77,6 +77,7 @@
                 "related_cves": null,
                 "cert_lab": null,
                 "cert_id": null,
+                "annotated_references": null,
                 "extracted_sars": null,
                 "direct_transitive_cves": null,
                 "indirect_transitive_cves": null,
@@ -165,6 +166,7 @@
                 "related_cves": null,
                 "cert_lab": null,
                 "cert_id": null,
+                "annotated_references": null,
                 "extracted_sars": null,
                 "direct_transitive_cves": null,
                 "indirect_transitive_cves": null,
@@ -261,6 +263,7 @@
                 "related_cves": null,
                 "cert_lab": null,
                 "cert_id": null,
+                "annotated_references": null,
                 "extracted_sars": null,
                 "direct_transitive_cves": null,
                 "indirect_transitive_cves": null,
@@ -282,4 +285,4 @@
             }
         }
     ]
-}
\ No newline at end of file
+}