diff options
| author | Adam Janovsky | 2021-02-21 15:06:21 +0100 |
|---|---|---|
| committer | Adam Janovsky | 2021-02-21 15:06:21 +0100 |
| commit | 97791842b8729e93eb32e343306c917d67faeb26 (patch) | |
| tree | 9489e626e765340167b69cd37c82766cc5af9913 | |
| parent | 5a86997d59f15675903c4d402e56fcb0f2fca528 (diff) | |
| download | sec-certs-97791842b8729e93eb32e343306c917d67faeb26.tar.gz sec-certs-97791842b8729e93eb32e343306c917d67faeb26.tar.zst sec-certs-97791842b8729e93eb32e343306c917d67faeb26.zip | |
more cautious exception handling
| -rw-r--r-- | cc_oop_demo.py | 17 | ||||
| -rw-r--r-- | sec_certs/helpers.py | 5 |
2 files changed, 12 insertions, 10 deletions
diff --git a/cc_oop_demo.py b/cc_oop_demo.py index 49729400..e9d913a5 100644 --- a/cc_oop_demo.py +++ b/cc_oop_demo.py @@ -22,27 +22,26 @@ def main(): dset = CCDataset({}, Path('./debug_dataset'), 'sample_dataset', 'sample dataset description') # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web(to_download=False) - logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') + dset.get_certs_from_web(to_download=True) - # # Dump dataset into JSON - dset.to_json('./debug_dataset/cc_full_dataset.json') + # Dump dataset into JSON + dset.to_json('./debug_dataset/parsed_meta.json') # Load dataset from JSON - new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json') - - assert dset == new_dset + dset = CCDataset.from_json('./debug_dataset/parsed_meta.json') + # assert dset == new_dset # Download pdfs dset.download_all_pdfs() + dset.to_json('./debug_dataset/downloaded_pdfs.json') # Convert pdfs to text dset.convert_all_pdfs() + dset.to_json('./debug_dataset/converted_pdfs.json') # Extract data from txt files dset.extract_data() - - dset.to_json('./debug_dataset/cc_full_dataset.json') + dset.to_json('./debug_dataset/extracted_pdfs.json') end = datetime.now() logger.info(f'The computation took {(end-start)} seconds.') diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 20d4fde3..22d2e272 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -34,6 +34,9 @@ def download_file(url: str, output: Path) -> int: return r.status_code except requests.exceptions.Timeout: return requests.codes.timeout + except Exception as e: + logger.error(f'Failed to download from {url}; {e}') + return constants.RETURNCODE_NOK def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Sequence[Tuple[str, int]]: @@ -176,9 +179,9 @@ def convert_pdf_file(pdf_path: Path, txt_path: Path, options): def extract_pdf_metadata(filepath: Path): metadata = dict() - metadata['pdf_file_size_bytes'] = filepath.stat().st_size try: + metadata['pdf_file_size_bytes'] = filepath.stat().st_size with filepath.open('rb') as handle: pdf = PdfFileReader(handle) |