diff options
| author | Adam Janovsky | 2023-05-18 07:28:05 +0200 |
|---|---|---|
| committer | Adam Janovsky | 2023-05-18 07:28:05 +0200 |
| commit | b0ce6e50649bdc94a8bf98af827436b115f6b7eb (patch) | |
| tree | d4560d2bb065217266951e9b030dc32eefbc192b | |
| parent | b808490207666abf0e4afc97305d05ca9facc0fa (diff) | |
| download | sec-certs-b0ce6e50649bdc94a8bf98af827436b115f6b7eb.tar.gz sec-certs-b0ce6e50649bdc94a8bf98af827436b115f6b7eb.tar.zst sec-certs-b0ce6e50649bdc94a8bf98af827436b115f6b7eb.zip | |
coerce problematic datetime values in cert csvs/dfs/htmls
| -rw-r--r-- | src/sec_certs/dataset/cc.py | 8 | ||||
| -rw-r--r-- | src/sec_certs/dataset/fips.py | 4 |
2 files changed, 6 insertions, 6 deletions
diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py index f5237d42..0db196c9 100644 --- a/src/sec_certs/dataset/cc.py +++ b/src/sec_certs/dataset/cc.py @@ -79,8 +79,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=CCCertificate.pandas_columns) df = df.set_index("dgst") - df.not_valid_before = pd.to_datetime(df.not_valid_before, infer_datetime_format=True) - df.not_valid_after = pd.to_datetime(df.not_valid_after, infer_datetime_format=True) + df.not_valid_before = pd.to_datetime(df.not_valid_before, infer_datetime_format=True, errors="coerce") + df.not_valid_after = pd.to_datetime(df.not_valid_after, infer_datetime_format=True, errors="coerce") df = df.astype( {"category": "category", "status": "category", "scheme": "category", "cert_lab": "category"} ).fillna(value=np.nan) @@ -361,7 +361,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable df[["not_valid_before", "not_valid_after", "maintenance_date"]] = df[ ["not_valid_before", "not_valid_after", "maintenance_date"] - ].apply(pd.to_datetime) + ].apply(pd.to_datetime, errors="coerce") df["dgst"] = df.apply(lambda row: helpers.get_first_16_bytes_sha256(_get_primary_key_str(row)), axis=1) @@ -897,7 +897,7 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType): df = df.set_index("dgst") df.index.name = "dgst" - df.maintenance_date = pd.to_datetime(df.maintenance_date, infer_datetime_format=True) + df.maintenance_date = pd.to_datetime(df.maintenance_date, infer_datetime_format=True, errors="coerce") return df.fillna(value=np.nan) @classmethod diff --git a/src/sec_certs/dataset/fips.py b/src/sec_certs/dataset/fips.py index 3c057266..24536b9b 100644 --- a/src/sec_certs/dataset/fips.py +++ b/src/sec_certs/dataset/fips.py @@ -329,8 +329,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=FIPSCertificate.pandas_columns) df = df.set_index("dgst") - df.date_validation = pd.to_datetime(df.date_validation, infer_datetime_format=True) - df.date_sunset = pd.to_datetime(df.date_sunset, infer_datetime_format=True) + df.date_validation = pd.to_datetime(df.date_validation, infer_datetime_format=True, errors="coerce") + df.date_sunset = pd.to_datetime(df.date_sunset, infer_datetime_format=True, errors="coerce") # Manually delete one certificate with bad embodiment (seems to have many blank fields) df = df.loc[~(df.embodiment == "*")] |