bump references

1 files changed, 2 insertions, 29 deletions

diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md
index b10a2e30..8521eead 100644
--- a/src/sec_certs/data/reference_annotations/readme.md
+++ b/src/sec_certs/data/reference_annotations/readme.md
@@ -54,35 +54,8 @@ These can be further merged into the following super-categories:
 The inter-annotator agreement is measured both with Cohen's Kappa and with percentage. The results are as follows:
 
 | Cohen's Kappa | Percentage |
-|---------------|------------|
+| ------------- | ---------- |
 | 0.71          | 0.82       |
 
-The code used to measure the agreement is:
+The code used to measure the agreement is stored in `notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb`.
 
-```python
-import pandas as pd
-from pathlib import Path
-from sklearn.metrics import cohen_kappa_score
-
-def load_all_dataframes(base_folder: Path) -> pd.DataFrame:
-    splits = ["train", "valid", "test"]
-
-    df_train, df_valid, df_test = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
-    for split in splits:
-        df = pd.read_csv(base_folder / f"{split}.csv")
-        if split == "train":
-            df_train = df
-        elif split == "valid":
-            df_valid = df
-        else:
-            df_test = df
-
-    return pd.concat([df_train, df_valid, df_test])
-
-adam_df = load_all_dataframes(Path("./src/sec_certs/data/reference_annotations/adam"))
-jano_df = load_all_dataframes(Path("./src/sec_certs/data/reference_annotations/jano"))
-agreement_series = adam_df.label == jano_df.label
-
-print(f"Cohen's Kappa: {cohen_kappa_score(adam_df.label, jano_df.label)}")
-print(f"Percentage agreement: {agreement_series.loc[agreement_series == True].count() / agreement_series.count()}")
-```