From 76d032e53e4e84c1c0036daaadeab2821be0730a Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Fri, 17 Feb 2023 14:11:57 +0100 Subject: more work on PoC reference annotator --- .../reference_annotations/data_preprocessing.ipynb | 57 +- .../cc/reference_annotations/prediction.ipynb | 719 ++++++--------------- 2 files changed, 220 insertions(+), 556 deletions(-) (limited to 'notebooks/cc/reference_annotations') diff --git a/notebooks/cc/reference_annotations/data_preprocessing.ipynb b/notebooks/cc/reference_annotations/data_preprocessing.ipynb index 5e478205..7e6e05ba 100644 --- a/notebooks/cc/reference_annotations/data_preprocessing.ipynb +++ b/notebooks/cc/reference_annotations/data_preprocessing.ipynb @@ -11,22 +11,15 @@ "\n", "1. Recover text segments that surround certificate ID for all references in CC dataset\n", "2. Create a DataFrame `(dgst, cert_id, label, text_segments)` out of the objects\n", - " - two versions, in fact: one with single row per segment, second with all segments from all sources (target, report) merged into single row\n", "3. Clean and dump into csv\n", "4. Check for label noise" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [] - } - ], + "outputs": [], "source": [ "from __future__ import annotations\n", "\n", @@ -37,8 +30,6 @@ "from sec_certs.utils.parallel_processing import process_parallel\n", "import pandas as pd\n", "import json\n", - "from tqdm import tqdm\n", - "\n", "\n", "nlp = spacy.load(\"en_core_web_sm\")\n", "from pathlib import Path\n", @@ -89,7 +80,7 @@ " Builds dataframe with [dgst,cert_id,location,reason,sentences] with references from list of ReferenceRecords.\n", " Reason set to None if not defined. \n", " \"\"\"\n", - " results = process_parallel(ReferenceRecord.get_cert_references_with_sentences, records, max_workers=200, use_threading=False, progress_bar=True)\n", + " results = process_parallel(ReferenceRecord.get_cert_references_with_sentences, records, use_threading=False, progress_bar=True)\n", " return pd.DataFrame.from_records([x.to_pandas_tuple() for x in results], columns=[\"dgst\", \"cert_id\", \"location\", \"label\", \"sentences\"])" ] }, @@ -103,17 +94,16 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " 69%|██████▉ | 654/944 [22:33<10:00, 2.07s/it]\n", - "100%|██████████| 58/58 [00:07<00:00, 8.26it/s]\n", - "100%|██████████| 944/944 [01:04<00:00, 14.59it/s]\n", - "100%|██████████| 2259/2259 [00:30<00:00, 75.10it/s]\n" + "100%|██████████| 58/58 [00:07<00:00, 8.27it/s]\n", + "100%|██████████| 944/944 [01:06<00:00, 14.12it/s]\n", + "100%|██████████| 2259/2259 [00:32<00:00, 69.22it/s]\n" ] } ], @@ -147,17 +137,21 @@ ] }, { - "cell_type": "code", - "execution_count": 49, + "attachments": {}, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "df = df_labeled.copy()" + "## Process Dataframes and dump two versions into csv\n", + "\n", + "1. Version with `dgst, cert_id, location, single_sentence` as `*_exploded.csv`\n", + "2. Version where all sentences tied to `(dgst, cert_id)` key are merged into `sentences`. Saved as `*_grouped.csv`\n", + "\n", + "*Note*: So far don't work with test dataset" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -170,28 +164,27 @@ "\n", "split_dct = {**dict.fromkeys(train_digests, \"train\"), **dict.fromkeys(valid_digests, \"valid\")}\n", "\n", - "\n", "# Apply filtering\n", - "df = df.loc[df.sentences.notnull()] # TODO: We should investigate the cases when we match no sentence\n", - "df[\"split\"] = df.dgst.map(split_dct) # Annotate with splits\n", + "# TODO: We should investigate the cases when we match no sentence\n", + "df = df.loc[df.sentences.notnull()] \n", + "df[\"split\"] = df.dgst.map(split_dct)\n", "df = df.loc[df.split.notnull()] # Discard test samples\n", - "df.explode(\"sentences\").to_csv(REPO_ROOT / \"datasets/reference_classification_dataset_exploded.csv\", sep=';', index=False)\n", "\n", "# TODO: Add language detection\n", "\n", "# Aggregate sentences from different sources (target, report) into one row\n", - "df_grouped = df.groupby([\"dgst\", \"cert_id\", \"label\", \"split\"], as_index=False)[\"sentences\"].agg({\"sentences\": lambda x: set.union(*x)})\n", - "df_grouped.to_csv(REPO_ROOT / \"datasets/reference_classification_dataset_merrged.csv\", sep=';', index=False)" + "df = df.groupby([\"dgst\", \"cert_id\", \"label\", \"split\"], as_index=False)[\"sentences\"].agg({\"sentences\": lambda x: set.union(*x)})\n", + "df.to_csv(REPO_ROOT / \"datasets/reference_classification_dataset_merrged.csv\", sep=';', index=False)" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - "# Check for label noise\n", - "duplicates_df = df_grouped[df_grouped.duplicated(subset=[\"dgst\", \"cert_id\"], keep=False)]\n", + "# Check for label noise, i.e., search for instances that have different label of a reference.\n", + "duplicates_df = df[df.duplicated(subset=[\"dgst\", \"cert_id\"], keep=False)]\n", "if not duplicates_df.empty:\n", " print(\"Warning, label noise in dataset. I.e. tuples (dgst, cert_id) with inconsistent reason. See `duplicates_df` frame.\")" ] @@ -218,7 +211,7 @@ "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "a5b8c5b127d2cfe5bc3a1c933e197485eb9eba25154c3661362401503b4ef9d4" + "hash": "a2ed43df31f510d0b358bd0625493376557b0c4d37aa99c09b398809f951b6a5" } } }, diff --git a/notebooks/cc/reference_annotations/prediction.ipynb b/notebooks/cc/reference_annotations/prediction.ipynb index 435fe684..29b63523 100644 --- a/notebooks/cc/reference_annotations/prediction.ipynb +++ b/notebooks/cc/reference_annotations/prediction.ipynb @@ -1,8 +1,20 @@ { "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prediction of reference annotations in CC Certificates (Raw)\n", + "\n", + "This notebook:\n", + "- loads dataframe of a dataset with `(dgst, cert_id, sentences, label)`\n", + "- Trains a model to classify the sentences related to certificate reference to their common sentiment (meaning of reference)" + ] + }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -14,7 +26,7 @@ } ], "source": [ - "# It is important to first set CUDA_VISIBLE_DEVICES environment variable directly from notebook\n", + "# When on Aura, it is important to first set CUDA_VISIBLE_DEVICES environment variable directly from notebook\n", "# For available GPUs, see https://www.fi.muni.cz/tech/unix/aura.html.cs\n", "\n", "import os\n", @@ -22,140 +34,38 @@ "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"MIG-a5459e6a-b26d-5985-874c-528458a7728b\"\n", "print(os.getenv(\"CUDA_VISIBLE_DEVICES\"))\n", "\n", - "# import spacy\n", - "# from spacy_cld import LanguageDetector\n", - "\n", - "# nlp = spacy.load(\"en_core_web_sm\")\n", - "# language_detector = LanguageDetector()\n", - "# nlp.add_pipe(language_detector)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "# installed packages: setfit\n", - "\n", "import pandas as pd\n", - "from datasets import Dataset, DatasetDict, ClassLabel, Features, Value, Sequence\n", - "from sentence_transformers.losses import CosineSimilarityLoss\n", + "from sec_certs.utils.nlp import prec_recall_metric\n", "from ast import literal_eval\n", - "from setfit import SetFitModel, SetFitTrainer, sample_dataset\n", - "\n", - "from sklearn.metrics import precision_score, recall_score\n", "from pathlib import Path\n", + "from sec_certs.model.reference_classification import ReferenceClassifierTrainer\n", + "import numpy as np\n", "\n", "REPO_ROOT = Path(\"../../../\").resolve()\n", "\n", - "def prec_recall_metric(y_pred, y_test):\n", - " return {\"precision\": precision_score(y_test, y_pred, zero_division=\"warn\", average=\"micro\"), \"recall\": recall_score(y_test, y_pred, zero_division=\"warn\", average=\"micro\")}\n", "\n", - "def predict_and_fill_df(model, df, train_dataset) -> pd.DataFrame:\n", - " label_mapping = {index: x for index, x in enumerate(train_dataset.features[\"label\"].names)}\n", + "def predict_and_fill_df(clf, df, label_mapping):\n", + " \"\"\"\n", + " Given the classifier, dataframe and label mapping, will populate dataframe with predictions for simple inspection.\n", + " \"\"\"\n", " df_new = df.copy()\n", - "\n", - " y_train_proba = model.predict_proba(df_new.sentences.tolist())\n", - " df_new[\"y_proba\"] = y_train_proba.tolist()\n", - " df_new[\"y_pred\"] = df_new.y_proba.map(lambda x: label_mapping[x.index(max(x))])\n", - " df_new[\"correct\"] = df_new.y_pred == df_new.label\n", - " \n", - " return df_new\n" + " y_proba = clf.predict_proba(df_new.sentences)\n", + " df_new[\"y_proba\"] = y_proba\n", + " df_new[\"y_pred\"] = df_new.y_proba.map(lambda x: label_mapping[np.argmax(x)])\n", + " df_new[\"correct\"] = df_new.label == df_new.y_pred\n", + " return df_new" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "# EXPLODED variant\n", - "# # Prepare dataset\n", - "\n", - "# df = pd.read_csv(REPO_ROOT / \"datasets/reference_classification_dataset_exploded.csv\", sep=\";\")\n", - "# df = df.loc[(df.label.notnull()) & ((df.location == \"report\"))]\n", - "\n", - "# # # Get language of the sentence, quite unreliable for the moment\n", - "# # df[\"lang\"] = df.sentence.map(lambda x: nlp(x)._.languages)\n", - "# # df[\"is_en\"] = df.lang.map(lambda x: x == [\"en\"])\n", - "\n", - "# # # Take suitable subset of the dataframe\n", - "# # df = df.loc[df.is_en] # only english\n", - "\n", - "# df = df.loc[df.label.isin({\"COMPONENT_USED\", \"BASIS_OF_RECERTIFICATION\", \"BASIS_FOR\"})] # only the most popular labels\n", - "\n", - "# # Split into train/valid\n", - "# df_train = df.loc[df.split == \"train\"].drop(columns=\"split\")\n", - "# df_valid = df.loc[df.split == \"valid\"].drop(columns=\"split\")\n", - "\n", - "# dataset_features = Features(\n", - "# {\n", - "# \"dgst\": Value(\"string\"),\n", - "# \"cert_id\": Value(\"string\"),\n", - "# \"location\": Value(\"string\"),\n", - "# \"sentences\": Value(\"string\"),\n", - "# \"label\": ClassLabel(names=list(df.label.unique())),\n", - "# }\n", - "# )\n", - "# train_dataset = Dataset.from_pandas(df_train, features=dataset_features, split=\"train\", preserve_index=False)\n", - "# valid_dataset = Dataset.from_pandas(df_valid, features=dataset_features, split=\"validation\", preserve_index=False)\n", - "\n", - "# dataset = DatasetDict()\n", - "# dataset['train'] = train_dataset\n", - "# dataset['validation'] = valid_dataset\n", - "\n", - "# train_dataset = sample_dataset(dataset[\"train\"], label_column=\"label\", num_samples=10)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "099029c554974aa1abdd9660eb8e2ece", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/1 [00:00dgst\n", " cert_id\n", " label\n", - " split\n", " sentences\n", " \n", " \n", " \n", " \n", - " 0\n", - " 0c7ef6c32cbdee47\n", - " ANSSI-CC-2017/61\n", + " 12\n", + " 99223aca5d9eb3b3\n", + " DCSSI-2009/11\n", " COMPONENT_USED\n", - " valid\n", - " {'Elixir-2 Project, Certification ID ANSSI-CC-...\n", + " [Toolbox Certificate DCSSI-2009/11\\nTable 1:]\n", " \n", " \n", - " 1\n", - " 0c7ef6c32cbdee47\n", - " BSI-DSZ-CC-1074-2019\n", - " BASIS_FOR\n", - " valid\n", - " {'The BAC+PACE configuration is subject of the...\n", + " 4\n", + " 0f3900cdcd0c7f3e\n", + " BSI-DSZ-CC-1072-V4-2021-MA-01\n", + " COMPONENT_USED\n", + " [Certification Report NXP Secure Smart Card Co...\n", " \n", " \n", - " 2\n", - " 0e22fe4e4e58faf4\n", - " BSI-DSZ-CC-1052-V4-2021\n", + " 9\n", + " 6d6ade44dcc497dd\n", + " BSI-DSZ-CC-0227-2004\n", " BASIS_OF_RECERTIFICATION\n", - " valid\n", - " {'basierend auf BSI-DSZ-CC-1052-V4-2021.'}\n", + " [This is a\\nre-certification based on BSI-DSZ-...\n", " \n", " \n", - " 3\n", + " 5\n", " 0f3900cdcd0c7f3e\n", - " BSI-DSZ-CC-1072-V4-2021\n", + " NSCIB-CC-66030-CR5\n", " COMPONENT_USED\n", - " train\n", - " {'BSI-DSZ-CC-1072-V4-2021 and the Crypto Libra...\n", + " [certificate identification NSCIB-CC-66030-CR5...\n", " \n", " \n", - " 4\n", - " 0f3900cdcd0c7f3e\n", - " BSI-DSZ-CC-1072-V4-2021-MA-01\n", + " 6\n", + " 1fb1564dfb0f0b04\n", + " ANSSI-CC-2020/34\n", " COMPONENT_USED\n", - " train\n", - " {'Certification Report NXP Secure Smart Card C...\n", + " [[CER_IC] Rapport de certification ANSSI-CC-20...\n", " \n", " \n", "\n", "" ], "text/plain": [ - " dgst cert_id label \\\n", - "0 0c7ef6c32cbdee47 ANSSI-CC-2017/61 COMPONENT_USED \n", - "1 0c7ef6c32cbdee47 BSI-DSZ-CC-1074-2019 BASIS_FOR \n", - "2 0e22fe4e4e58faf4 BSI-DSZ-CC-1052-V4-2021 BASIS_OF_RECERTIFICATION \n", - "3 0f3900cdcd0c7f3e BSI-DSZ-CC-1072-V4-2021 COMPONENT_USED \n", - "4 0f3900cdcd0c7f3e BSI-DSZ-CC-1072-V4-2021-MA-01 COMPONENT_USED \n", + " dgst cert_id label \\\n", + "12 99223aca5d9eb3b3 DCSSI-2009/11 COMPONENT_USED \n", + "4 0f3900cdcd0c7f3e BSI-DSZ-CC-1072-V4-2021-MA-01 COMPONENT_USED \n", + "9 6d6ade44dcc497dd BSI-DSZ-CC-0227-2004 BASIS_OF_RECERTIFICATION \n", + "5 0f3900cdcd0c7f3e NSCIB-CC-66030-CR5 COMPONENT_USED \n", + "6 1fb1564dfb0f0b04 ANSSI-CC-2020/34 COMPONENT_USED \n", "\n", - " split sentences \n", - "0 valid {'Elixir-2 Project, Certification ID ANSSI-CC-... \n", - "1 valid {'The BAC+PACE configuration is subject of the... \n", - "2 valid {'basierend auf BSI-DSZ-CC-1052-V4-2021.'} \n", - "3 train {'BSI-DSZ-CC-1072-V4-2021 and the Crypto Libra... \n", - "4 train {'Certification Report NXP Secure Smart Card C... " + " sentences \n", + "12 [Toolbox Certificate DCSSI-2009/11\\nTable 1:] \n", + "4 [Certification Report NXP Secure Smart Card Co... \n", + "9 [This is a\\nre-certification based on BSI-DSZ-... \n", + "5 [certificate identification NSCIB-CC-66030-CR5... \n", + "6 [[CER_IC] Rapport de certification ANSSI-CC-20... " ] }, - "execution_count": 12, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df.head()" + "df_train.head()" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -297,16 +187,16 @@ "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n", "Applying column mapping to training dataset\n", "***** Running training *****\n", - " Num examples = 640\n", + " Num examples = 1760\n", " Num epochs = 1\n", - " Total optimization steps = 40\n", + " Total optimization steps = 110\n", " Total train batch size = 16\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a7118a8525ee47f78f29b74fb5e62117", + "model_id": "96e469ad1f984bf6ba2c819884a1c231", "version_major": 2, "version_minor": 0 }, @@ -320,12 +210,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "96f5685713554e05af4ff7b7ec4e2f31", + "model_id": "a9ef28c8c0314e7f831e6e35c2af75db", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Iteration: 0%| | 0/40 [00:00\n", " dgst\n", " cert_id\n", - " location\n", " label\n", " sentences\n", " y_proba\n", @@ -422,374 +295,171 @@ " \n", " \n", " 9\n", - " 0e22fe4e4e58faf4\n", - " BSI-DSZ-CC-1052-V4-2021\n", - " report\n", + " 6d6ade44dcc497dd\n", + " BSI-DSZ-CC-0227-2004\n", " BASIS_OF_RECERTIFICATION\n", - " basierend auf BSI-DSZ-CC-1052-V4-2021.\n", - " [0.9006353496264655, 0.0993646503735345]\n", + " [This is a\\nre-certification based on BSI-DSZ-...\n", + " [0.5461188093773812, 0.45388119062261884]\n", " COMPONENT_USED\n", " False\n", " \n", " \n", - " 10\n", - " 29964f32c68b0ce8\n", - " BSI-DSZ-CC-0519-V3-2021\n", - " report\n", + " 19\n", + " ca5da2fe138af656\n", + " BSI-DSZ-CC-0413-2007\n", " BASIS_OF_RECERTIFICATION\n", - " Specific results from\\nthe evaluation process ...\n", - " [0.9018767488720597, 0.09812325112794029]\n", + " [This is a re-certification based on\\nBSI-DSZ-...\n", + " [0.5465589745598575, 0.4534410254401425]\n", " COMPONENT_USED\n", " False\n", " \n", - " \n", - " 11\n", - " 29964f32c68b0ce8\n", - " BSI-DSZ-CC-0519-V3-2021\n", - " report\n", - " BASIS_OF_RECERTIFICATION\n", - " This is a re-certification based on BSI-DSZ-CC...\n", - " [0.694838178998959, 0.30516182100104094]\n", - " COMPONENT_USED\n", - " False\n", + " \n", + "\n", + "" + ], + "text/plain": [ + " dgst cert_id label \\\n", + "9 6d6ade44dcc497dd BSI-DSZ-CC-0227-2004 BASIS_OF_RECERTIFICATION \n", + "19 ca5da2fe138af656 BSI-DSZ-CC-0413-2007 BASIS_OF_RECERTIFICATION \n", + "\n", + " sentences \\\n", + "9 [This is a\\nre-certification based on BSI-DSZ-... \n", + "19 [This is a re-certification based on\\nBSI-DSZ-... \n", + "\n", + " y_proba y_pred correct \n", + "9 [0.5461188093773812, 0.45388119062261884] COMPONENT_USED False \n", + "19 [0.5465589745598575, 0.4534410254401425] COMPONENT_USED False " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_train.loc[~df_train.correct]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", "
dgstcert_idlabelsentencesy_probay_predcorrect
1229964f32c68b0ce8BSI-DSZ-CC-0519-V3-2021reportBASIS_OF_RECERTIFICATIONAs the evaluation work performed for this cert...[0.8982407935537878, 0.1017592064462122]10c7ef6c32cbdee47BSI-DSZ-CC-1074-2019BASIS_FOR[The BAC+PACE configuration is subject of the ...[0.9330686268852108, 0.06693137311478929]COMPONENT_USEDFalse
22c1d88ce9dadd7d2dBSI-DSZ-CC-0312-2005reportCOMPONENT_USED[13] Certification Report BSI-DSZ-CC-0312-2005...[0.8947294027870802, 0.1052705972129198]COMPONENT_USEDTrue
23c1d88ce9dadd7d2dBSI-DSZ-CC-0312-2005reportCOMPONENT_USEDP5CT072V0N refer to the certification report B...[0.900841506996804, 0.09915849300319597]COMPONENT_USEDTrue
28238f8edc5eda1358BSI-DSZ-CC-0222-2003report20e22fe4e4e58faf4BSI-DSZ-CC-1052-V4-2021BASIS_OF_RECERTIFICATIONThis is a re-\\ncertification based on BSI-DSZ-...[0.6922405130591214, 0.3077594869408786][basierend auf BSI-DSZ-CC-1052-V4-2021.][0.7070543956916182, 0.2929456043083818]COMPONENT_USEDFalse
297238f8edc5eda1358BSI-DSZ-CC-0222-2003reportBASIS_OF_RECERTIFICATIONThis certification is a re-certification of BS...[0.7146859285928153, 0.2853140714071847]COMPONENT_USEDFalse
31a6fac58198296194BSI-DSZ-CC-0555-2009reportBASIS_OF_RECERTIFICATIONSpecific results from the evaluation process\\n...[0.9003949217128049, 0.09960507828719516][This certification is a re-certification of B...[0.5998535578550636, 0.4001464421449364]COMPONENT_USEDFalse
32a6fac58198296194BSI-DSZ-CC-0555-2009report829964f32c68b0ce8BSI-DSZ-CC-0519-V3-2021BASIS_OF_RECERTIFICATIONAs the evaluation work performed for this cert...[0.8962471709210816, 0.10375282907891832][This is a re-certification based on BSI-DSZ-C...[0.8727371952470533, 0.12726280475294682]COMPONENT_USEDFalse
3313a6fac58198296194BSI-DSZ-CC-0555-2009reportBASIS_OF_RECERTIFICATIONThis is a re-certification\\nbased on BSI-DSZ-C...[0.6928894855518116, 0.3071105144481884]COMPONENT_USEDFalse
400c7ef6c32cbdee47BSI-DSZ-CC-1074-2019reportBASIS_FORThe BAC+PACE configuration is subject of the s...[0.8997694315622796, 0.1002305684377204][Specific results from the evaluation process\\...[0.8670438280210987, 0.13295617197890133]COMPONENT_USEDFalse
410c7ef6c32cbdee47BSI-DSZ-CC-1074-2019reportBASIS_FORThe further security mechanism Basic Access Co...[0.8923570789488144, 0.10764292105118559]COMPONENT_USEDFalse
420c7ef6c32cbdee47BSI-DSZ-CC-1074-2019reportBASIS_FOR[14] Certification Report BSI-DSZ-CC-1074-2019...[0.9007422537659316, 0.09925774623406838]COMPONENT_USEDFalse
430c7ef6c32cbdee47ANSSI-CC-2017/61reportCOMPONENT_USEDFor details\\nconcerning the CC evaluation of t...[0.8933002244736725, 0.10669977552632742]COMPONENT_USEDTrue
440c7ef6c32cbdee47ANSSI-CC-2017/61reportCOMPONENT_USEDFor development and production sites regarding...[0.9000976545847758, 0.0999023454152242]COMPONENT_USEDTrue
450c7ef6c32cbdee47ANSSI-CC-2017/61reportCOMPONENT_USEDElixir-2 Project, Certification ID ANSSI-CC-20...[0.8997425549440855, 0.10025744505591448]COMPONENT_USEDTrue
460c7ef6c32cbdee47ANSSI-CC-2017/61reportCOMPONENT_USED® X – Security Target for\\ncomposition; STMicr...[0.8976230487254611, 0.10237695127453893]COMPONENT_USEDTrue
470c7ef6c32cbdee47ANSSI-CC-2017/61reportCOMPONENT_USED[12] Rapport de certification ANSSI-CC-2017/61...[0.8939784357781945, 0.10602156422180552]COMPONENT_USEDTrue
\n", "
" ], "text/plain": [ - " dgst cert_id location \\\n", - "9 0e22fe4e4e58faf4 BSI-DSZ-CC-1052-V4-2021 report \n", - "10 29964f32c68b0ce8 BSI-DSZ-CC-0519-V3-2021 report \n", - "11 29964f32c68b0ce8 BSI-DSZ-CC-0519-V3-2021 report \n", - "12 29964f32c68b0ce8 BSI-DSZ-CC-0519-V3-2021 report \n", - "22 c1d88ce9dadd7d2d BSI-DSZ-CC-0312-2005 report \n", - "23 c1d88ce9dadd7d2d BSI-DSZ-CC-0312-2005 report \n", - "28 238f8edc5eda1358 BSI-DSZ-CC-0222-2003 report \n", - "29 238f8edc5eda1358 BSI-DSZ-CC-0222-2003 report \n", - "31 a6fac58198296194 BSI-DSZ-CC-0555-2009 report \n", - "32 a6fac58198296194 BSI-DSZ-CC-0555-2009 report \n", - "33 a6fac58198296194 BSI-DSZ-CC-0555-2009 report \n", - "40 0c7ef6c32cbdee47 BSI-DSZ-CC-1074-2019 report \n", - "41 0c7ef6c32cbdee47 BSI-DSZ-CC-1074-2019 report \n", - "42 0c7ef6c32cbdee47 BSI-DSZ-CC-1074-2019 report \n", - "43 0c7ef6c32cbdee47 ANSSI-CC-2017/61 report \n", - "44 0c7ef6c32cbdee47 ANSSI-CC-2017/61 report \n", - "45 0c7ef6c32cbdee47 ANSSI-CC-2017/61 report \n", - "46 0c7ef6c32cbdee47 ANSSI-CC-2017/61 report \n", - "47 0c7ef6c32cbdee47 ANSSI-CC-2017/61 report \n", - "\n", - " label \\\n", - "9 BASIS_OF_RECERTIFICATION \n", - "10 BASIS_OF_RECERTIFICATION \n", - "11 BASIS_OF_RECERTIFICATION \n", - "12 BASIS_OF_RECERTIFICATION \n", - "22 COMPONENT_USED \n", - "23 COMPONENT_USED \n", - "28 BASIS_OF_RECERTIFICATION \n", - "29 BASIS_OF_RECERTIFICATION \n", - "31 BASIS_OF_RECERTIFICATION \n", - "32 BASIS_OF_RECERTIFICATION \n", - "33 BASIS_OF_RECERTIFICATION \n", - "40 BASIS_FOR \n", - "41 BASIS_FOR \n", - "42 BASIS_FOR \n", - "43 COMPONENT_USED \n", - "44 COMPONENT_USED \n", - "45 COMPONENT_USED \n", - "46 COMPONENT_USED \n", - "47 COMPONENT_USED \n", + " dgst cert_id label \\\n", + "1 0c7ef6c32cbdee47 BSI-DSZ-CC-1074-2019 BASIS_FOR \n", + "2 0e22fe4e4e58faf4 BSI-DSZ-CC-1052-V4-2021 BASIS_OF_RECERTIFICATION \n", + "7 238f8edc5eda1358 BSI-DSZ-CC-0222-2003 BASIS_OF_RECERTIFICATION \n", + "8 29964f32c68b0ce8 BSI-DSZ-CC-0519-V3-2021 BASIS_OF_RECERTIFICATION \n", + "13 a6fac58198296194 BSI-DSZ-CC-0555-2009 BASIS_OF_RECERTIFICATION \n", "\n", " sentences \\\n", - "9 basierend auf BSI-DSZ-CC-1052-V4-2021. \n", - "10 Specific results from\\nthe evaluation process ... \n", - "11 This is a re-certification based on BSI-DSZ-CC... \n", - "12 As the evaluation work performed for this cert... \n", - "22 [13] Certification Report BSI-DSZ-CC-0312-2005... \n", - "23 P5CT072V0N refer to the certification report B... \n", - "28 This is a re-\\ncertification based on BSI-DSZ-... \n", - "29 This certification is a re-certification of BS... \n", - "31 Specific results from the evaluation process\\n... \n", - "32 As the evaluation work performed for this cert... \n", - "33 This is a re-certification\\nbased on BSI-DSZ-C... \n", - "40 The BAC+PACE configuration is subject of the s... \n", - "41 The further security mechanism Basic Access Co... \n", - "42 [14] Certification Report BSI-DSZ-CC-1074-2019... \n", - "43 For details\\nconcerning the CC evaluation of t... \n", - "44 For development and production sites regarding... \n", - "45 Elixir-2 Project, Certification ID ANSSI-CC-20... \n", - "46 ® X – Security Target for\\ncomposition; STMicr... \n", - "47 [12] Rapport de certification ANSSI-CC-2017/61... \n", + "1 [The BAC+PACE configuration is subject of the ... \n", + "2 [basierend auf BSI-DSZ-CC-1052-V4-2021.] \n", + "7 [This certification is a re-certification of B... \n", + "8 [This is a re-certification based on BSI-DSZ-C... \n", + "13 [Specific results from the evaluation process\\... \n", "\n", " y_proba y_pred correct \n", - "9 [0.9006353496264655, 0.0993646503735345] COMPONENT_USED False \n", - "10 [0.9018767488720597, 0.09812325112794029] COMPONENT_USED False \n", - "11 [0.694838178998959, 0.30516182100104094] COMPONENT_USED False \n", - "12 [0.8982407935537878, 0.1017592064462122] COMPONENT_USED False \n", - "22 [0.8947294027870802, 0.1052705972129198] COMPONENT_USED True \n", - "23 [0.900841506996804, 0.09915849300319597] COMPONENT_USED True \n", - "28 [0.6922405130591214, 0.3077594869408786] COMPONENT_USED False \n", - "29 [0.7146859285928153, 0.2853140714071847] COMPONENT_USED False \n", - "31 [0.9003949217128049, 0.09960507828719516] COMPONENT_USED False \n", - "32 [0.8962471709210816, 0.10375282907891832] COMPONENT_USED False \n", - "33 [0.6928894855518116, 0.3071105144481884] COMPONENT_USED False \n", - "40 [0.8997694315622796, 0.1002305684377204] COMPONENT_USED False \n", - "41 [0.8923570789488144, 0.10764292105118559] COMPONENT_USED False \n", - "42 [0.9007422537659316, 0.09925774623406838] COMPONENT_USED False \n", - "43 [0.8933002244736725, 0.10669977552632742] COMPONENT_USED True \n", - "44 [0.9000976545847758, 0.0999023454152242] COMPONENT_USED True \n", - "45 [0.8997425549440855, 0.10025744505591448] COMPONENT_USED True \n", - "46 [0.8976230487254611, 0.10237695127453893] COMPONENT_USED True \n", - "47 [0.8939784357781945, 0.10602156422180552] COMPONENT_USED True " - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_valid" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "first = np.array([0.1,0.2,0.7])\n", - "second = np.array([0.2, 0.4, 0.4])\n", - "third = np.array([0.5, 0.5, 0])\n", - "preds = np.array([first, second, third])\n", - "\n", - "preds = np.power(preds, 2)\n", - "preds = preds.sum(axis=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([0.3 , 0.45, 0.65])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "preds" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2" + "1 [0.9330686268852108, 0.06693137311478929] COMPONENT_USED False \n", + "2 [0.7070543956916182, 0.2929456043083818] COMPONENT_USED False \n", + "7 [0.5998535578550636, 0.4001464421449364] COMPONENT_USED False \n", + "8 [0.8727371952470533, 0.12726280475294682] COMPONENT_USED False \n", + "13 [0.8670438280210987, 0.13295617197890133] COMPONENT_USED False " ] }, - "execution_count": 12, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "np.argmax(preds)" + "df_valid.loc[~df_valid.correct]" ] } ], @@ -811,9 +481,10 @@ "pygments_lexer": "ipython3", "version": "3.8.13" }, + "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "a5b8c5b127d2cfe5bc3a1c933e197485eb9eba25154c3661362401503b4ef9d4" + "hash": "a2ed43df31f510d0b358bd0625493376557b0c4d37aa99c09b398809f951b6a5" } } }, -- cgit v1.3.1