Add final run notebook.

1 files changed, 201 insertions, 0 deletions

diff --git a/notebooks/final_run.ipynb b/notebooks/final_run.ipynb
new file mode 100644
index 00000000..6a1b817b
--- /dev/null
+++ b/notebooks/final_run.ipynb
@@ -0,0 +1,201 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from sec_certs.dataset import CCDataset\n",
+    "from pathlib import Path\n",
+    "import shutil\n",
+    "import subprocess"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# A directory with a dump of the documents that will enrich the dataset.\n",
+    "doc_dump_dir = Path(\"cc_certs_03_10_2022\")\n",
+    "\n",
+    "# An output directory for the dataset.\n",
+    "dataset_dir = Path(\"cc_03_10_2022\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Print tool versions\n",
+    "subprocess.run([\"pdftotext\", \"-v\"])\n",
+    "subprocess.run([\"tesseract\", \"-v\"]);"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Create the dataset\n",
+    "dset = CCDataset(root_dir=dataset_dir, name=\"cc\", description=\"Final run on 03.10.2022\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Get the metadata\n",
+    "dset.get_certs_from_web()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Enrich the dataset with the pre-downloaded PDFs (collected over multiple runs to fill in the timeouts and disappearances).\n",
+    "for cert in dset:\n",
+    "    report_pdf = doc_dump_dir / \"report\" / \"pdf\" / f\"{cert.dgst}.pdf\"\n",
+    "    if report_pdf.exists():\n",
+    "        shutil.copy(report_pdf, cert.state.report_pdf_path)\n",
+    "        cert.state.report_download_ok = True\n",
+    "    target_pdf = doc_dump_dir / \"target\" / \"pdf\" / f\"{cert.dgst}.pdf\"\n",
+    "    if target_pdf.exists():\n",
+    "        shutil.copy(target_pdf, cert.state.target_pdf_path)\n",
+    "        cert.state.target_download_ok = True"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Try to download last missing (or ones not present in the pre-downloaded)\n",
+    "dset.download_all_pdfs(fresh=False)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Convert all\n",
+    "dset.convert_all_pdfs()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Analyze all\n",
+    "dset.analyze_certificates()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# And do maintenance updates as well\n",
+    "dset.process_maintenance_updates()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# Finally, dump it all\n",
+    "dset.to_json()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file