1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
from __future__ import annotations
import html
import logging
from datetime import date
import numpy as np
import pandas as pd
from bs4 import NavigableString
logger = logging.getLogger(__name__)
def sanitize_navigable_string(string: NavigableString | str | None) -> str | None:
if not string:
return None
return str(string).strip().replace("\xad", "").replace("\xa0", "")
def sanitize_link(record: str | None) -> str | None:
if not record:
return None
return record.replace(":443", "").replace(" ", "%20").replace("http://", "https://")
def sanitize_date(record: pd.Timestamp | date | np.datetime64) -> date | None:
if pd.isnull(record):
return None
if isinstance(record, pd.Timestamp):
return record.date()
if isinstance(record, date | type(None)):
return record
raise ValueError("Unsupported type given as input")
def sanitize_string(record: str) -> str:
# There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I & J' that has to be unescaped twice
string = html.unescape(html.unescape(record)).replace("\n", "")
return " ".join(string.split())
def sanitize_security_levels(record: str | set[str]) -> set[str]:
if isinstance(record, str):
record = set(record.split(","))
return record - {"Basic", "ND-PP", "PP\xa0Compliant", "None", "Medium"}
def sanitize_protection_profiles(record: str) -> list:
if not record:
return []
return record.split(",")
|
