Add OCR notebook.

2 files changed, 349 insertions, 1 deletions

diff --git a/notebooks/ocr.ipynb b/notebooks/ocr.ipynb
new file mode 100644
index 00000000..c67cfac3
--- /dev/null
+++ b/notebooks/ocr.ipynb
@@ -0,0 +1,346 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "import glob\n",
+    "from pathlib import Path\n",
+    "from matplotlib import pyplot as plt"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# Exploration of :garbage: PDFs"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "bad_reports = [\n",
+    "    \"c5e25f90c7006546\", # ANSSI spacing\n",
+    "    \"03bce905b71945aa\", # ANSSI spacing\n",
+    "    \"3477723044183b31\", # ANSSI empty?\n",
+    "    \"7e32023021d5aad2\", # empty?\n",
+    "    \"4c9468f20fdb04f7\", # empty?\n",
+    "    \"82c24f729c2e0092\", # ANSSI spacing\n",
+    "    \"e1daa354ae5a61fd\", # ANSSI spacing\n",
+    "    \"c80801f9a71b030e\", # ANSSI spacing\n",
+    "]\n",
+    "\n",
+    "good_reports = [\n",
+    "    \"2544ffa2d8eef431\", # Japan, short but OK\n",
+    "    \"a0aa53cad9c5d049\", # Korea, OK, but low avg\n",
+    "    \"10f1399a27470345\", # Korea, OK, but low avg\n",
+    "    \"60c49ab7f7d33501\", # Korea, OK, but low avg\n",
+    "    \"e133881d7203a6e4\", # Spain, OK\n",
+    "    \"4ff70fb16691d53c\", # India, OK\n",
+    "]"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def average_line_length(text: str) -> float:\n",
+    "    length = 0\n",
+    "    lines = 0\n",
+    "    for line in text.splitlines():\n",
+    "        length += len(line)\n",
+    "        lines += 1\n",
+    "    if lines:\n",
+    "        return length/lines\n",
+    "    else:\n",
+    "        return 0\n",
+    "\n",
+    "def overall_size(text: str) -> float:\n",
+    "    return len(text)\n",
+    "\n",
+    "def num_lines(text: str) -> float:\n",
+    "    return len(text.splitlines())\n",
+    "\n",
+    "def every_second_char(text: str) -> float:\n",
+    "    c = 0\n",
+    "    for line in text.splitlines():\n",
+    "        if len(set(line[1::2])) > 1:\n",
+    "            c += 1\n",
+    "    return c\n",
+    "\n",
+    "def alpha_chars(text: str) -> float:\n",
+    "    tl = len(text)\n",
+    "    if tl == 0:\n",
+    "        return 0\n",
+    "    return len(\"\".join(filter(str.isalpha, text))) / tl"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "LINES_THRESHOLD = 30\n",
+    "SIZE_THRESHOLD = 1000\n",
+    "AVG_LLEN_THRESHOLD = 10\n",
+    "EVERY_SECOND_CHAR_THRESHOLD = 15\n",
+    "ALPHA_CHARS_THRESHOLD = 0.5\n",
+    "\n",
+    "def garbage(text: str) -> bool:\n",
+    "    size = len(text)\n",
+    "    content_len = 0\n",
+    "    lines = 0\n",
+    "    every_second = 0\n",
+    "    alpha_len = len(\"\".join(filter(str.isalpha, text)))\n",
+    "    for line in text.splitlines():\n",
+    "        content_len += len(line)\n",
+    "        lines += 1\n",
+    "        if len(set(line[1::2])) > 1:\n",
+    "            every_second += 1\n",
+    "\n",
+    "    if lines:\n",
+    "        avg_line_len = content_len / lines\n",
+    "    else:\n",
+    "        avg_line_len = 0\n",
+    "    if size:\n",
+    "        alpha = alpha_len / size\n",
+    "    else:\n",
+    "        alpha = 0\n",
+    "\n",
+    "    # If number of lines is small, this is garbage.\n",
+    "    if lines < LINES_THRESHOLD:\n",
+    "        return True\n",
+    "    # If the file size is small, this is garbage.\n",
+    "    if size < SIZE_THRESHOLD:\n",
+    "        return True\n",
+    "    # If the average length of a line is small, this is garbage.\n",
+    "    if avg_line_len < AVG_LLEN_THRESHOLD:\n",
+    "        return True\n",
+    "    # If there a small amount of lines that have more than one character at every second character, this is garbage.\n",
+    "    # This detects the ANSSI spacing issues.\n",
+    "    if every_second < EVERY_SECOND_CHAR_THRESHOLD:\n",
+    "        return True\n",
+    "    # If there is a small ratio of alphanumeric chars to all chars, this is garbage.\n",
+    "    if alpha < ALPHA_CHARS_THRESHOLD:\n",
+    "        return True\n",
+    "    return False\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "avgs = {}\n",
+    "sizes = {}\n",
+    "line_counts = {}\n",
+    "schars = {}\n",
+    "alphas = {}\n",
+    "bad = set()\n",
+    "for fname in glob.glob(\"../cc_dset/certs/reports/txt/*\"):\n",
+    "    path = Path(fname)\n",
+    "\n",
+    "    with path.open(\"r\") as f:\n",
+    "        text = f.read()\n",
+    "    dgst = path.stem\n",
+    "\n",
+    "    avg = average_line_length(text)\n",
+    "    size = overall_size(text)\n",
+    "    nlines = num_lines(text)\n",
+    "    schar = every_second_char(text)\n",
+    "    alpha = alpha_chars(text)\n",
+    "\n",
+    "    avgs[dgst] = avg\n",
+    "    sizes[dgst] = size\n",
+    "    line_counts[dgst] = nlines\n",
+    "    schars[dgst] = schar\n",
+    "    alphas[dgst] = alpha\n",
+    "\n",
+    "    if nlines < 30:\n",
+    "        print(f\"{dgst}:  nlines: {nlines:.2f}\")\n",
+    "        bad.add(dgst)\n",
+    "    if size < 1000:\n",
+    "        print(f\"{dgst}:    size: {size:.2f}\")\n",
+    "        bad.add(dgst)\n",
+    "    if avg < 10:\n",
+    "        print(f\"{dgst}:     avg: {avg:.2f}\")\n",
+    "        bad.add(dgst)\n",
+    "    if schar < 15:\n",
+    "        print(f\"{dgst}:   schar: {schar:.2f}\")\n",
+    "        bad.add(dgst)\n",
+    "    if alpha < 0.5:\n",
+    "        print(f\"{dgst}:   alpha: {alpha:.2f}\")\n",
+    "        bad.add(dgst)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "print(len(bad))\n",
+    "print(\"                  nlines,   size,   lavg, schar, alpha\")\n",
+    "for b in bad:\n",
+    "    print(f\"{b}: {line_counts[b]:>6}, {sizes[b]:>7}, {avgs[b]:>5.02f}, {schars[b]:>5}, {alphas[b]:>5.02f}\")\n",
+    "for b in bad_reports:\n",
+    "    print(b in bad)\n",
+    "\n",
+    "for b in good_reports:\n",
+    "    print(b not in bad)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "plt.hist(line_counts.values(), bins=30);"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "plt.hist(sizes.values(), bins=30);"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "plt.hist(avgs.values(), bins=30);"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "plt.hist(schars.values(), bins=30);"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "plt.hist(alphas.values(), bins=30);"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "for pdf_name in glob.glob(\"../cc_dset/certs/reports/pdf/*.pdf\"):\n",
+    "    pdf_path = Path(pdf_name)\n",
+    "    dgst = pdf_path.stem\n",
+    "\n",
+    "    txt_path = Path(\"../cc_dset/certs/reports/txt\") / (dgst + \".txt\")\n",
+    "    if not txt_path.exists():\n",
+    "        print(dgst)\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/sec_certs/utils/helpers.py b/sec_certs/utils/helpers.py
index b1a17275..faaaef0d 100644
--- a/sec_certs/utils/helpers.py
+++ b/sec_certs/utils/helpers.py
@@ -222,6 +222,8 @@ def warn_if_missing_tesseract() -> None:
     """
     try:
         if not pkgconfig.installed("tesseract", ">=5.0.0"):
-            logger.warning("Attempting to run pipeline with pdf->txt conversion, that requires tesseract, but tesseract was not found.")
+            logger.warning(
+                "Attempting to run pipeline with pdf->txt conversion, that requires tesseract, but tesseract was not found."
+            )
     except EnvironmentError:
         logger.warning("Attempting to find tesseract, but pkg-config was not found.")