diff options
| -rw-r--r-- | cc_oop_demo.py | 17 | ||||
| -rw-r--r-- | sec_certs/helpers.py | 5 |
2 files changed, 12 insertions, 10 deletions
diff --git a/cc_oop_demo.py b/cc_oop_demo.py index 49729400..e9d913a5 100644 --- a/cc_oop_demo.py +++ b/cc_oop_demo.py @@ -22,27 +22,26 @@ def main(): dset = CCDataset({}, Path('./debug_dataset'), 'sample_dataset', 'sample dataset description') # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web(to_download=False) - logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') + dset.get_certs_from_web(to_download=True) - # # Dump dataset into JSON - dset.to_json('./debug_dataset/cc_full_dataset.json') + # Dump dataset into JSON + dset.to_json('./debug_dataset/parsed_meta.json') # Load dataset from JSON - new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json') - - assert dset == new_dset + dset = CCDataset.from_json('./debug_dataset/parsed_meta.json') + # assert dset == new_dset # Download pdfs dset.download_all_pdfs() + dset.to_json('./debug_dataset/downloaded_pdfs.json') # Convert pdfs to text dset.convert_all_pdfs() + dset.to_json('./debug_dataset/converted_pdfs.json') # Extract data from txt files dset.extract_data() - - dset.to_json('./debug_dataset/cc_full_dataset.json') + dset.to_json('./debug_dataset/extracted_pdfs.json') end = datetime.now() logger.info(f'The computation took {(end-start)} seconds.') diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 20d4fde3..22d2e272 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -34,6 +34,9 @@ def download_file(url: str, output: Path) -> int: return r.status_code except requests.exceptions.Timeout: return requests.codes.timeout + except Exception as e: + logger.error(f'Failed to download from {url}; {e}') + return constants.RETURNCODE_NOK def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Sequence[Tuple[str, int]]: @@ -176,9 +179,9 @@ def convert_pdf_file(pdf_path: Path, txt_path: Path, options): def extract_pdf_metadata(filepath: Path): metadata = dict() - metadata['pdf_file_size_bytes'] = filepath.stat().st_size try: + metadata['pdf_file_size_bytes'] = filepath.stat().st_size with filepath.open('rb') as handle: pdf = PdfFileReader(handle) |