Merge branch 'dev' into fips

17 files changed, 199 insertions, 40 deletions

diff --git a/.github/workflows/GA_CI.yml b/.github/workflows/GA_CI.yml
new file mode 100644
index 00000000..3239468f
--- /dev/null
+++ b/.github/workflows/GA_CI.yml
@@ -0,0 +1,22 @@
+name: tests
+on: [push]
+
+    
+jobs:
+  run-test:
+    runs-on: ubuntu-latest
+    steps:
+      - run: sudo apt-get install poppler-utils
+      - uses: actions/checkout@v2
+      - name : Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install python dependencies
+        run: pip install -r requirements.txt
+      - name : Install pytest and run scripts like Travis does
+        run: |
+          pip install pytest
+          pip install pytest-cov
+          pip install ".[dev,test]"
+          pytest test
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
new file mode 100644
index 00000000..fc0ca7f0
--- /dev/null
+++ b/.github/workflows/docker-image.yml
@@ -0,0 +1,30 @@
+# code created with help of the github documentation : https://docs.github.com/en/actions/guides/publishing-docker-images
+
+name: Docker Image CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: seccerts/sec-certs:latest
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
new file mode 100644
index 00000000..782ec7aa
--- /dev/null
+++ b/.github/workflows/python-publish.yml
@@ -0,0 +1,43 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+          
+      - name: Build and publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} #Same as Docker, need to add an account in the secrets
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*
+      
+      - name: Build and publish to PyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} #Same as Docker, need to add an account in the secrets
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload --repository testpypi dist/*
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index e67232e0..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-os: linux
-language: python
-dist: xenial
-python: "3.8"
-
-before_install:
-  - sudo apt-get -y install poppler-utils
-
-install:
-  - pip install ".[dev,test]"
-
-script:
-  - pytest test
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5a06ada3..7184033e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -8,3 +8,7 @@ You contribution is warmly welcomed. You can help by:
  3. Perform additional analysis with extracted data (analyze_certificates.py)
  3. Improve the code (TODO: Follow Github contribution guidelines, ideally contact us first about your plan)
 
+## Branches
+
+- `dev` is the default branch against which all pull requests are to be made. 
+- `master` is protected branch where stable releases reside. Each push or PR to master should be properly tagged by [semantic version](https://semver.org/) as it will invoke actions to publish docker image and PyPi package. 
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..e891e9a5
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,33 @@
+FROM ubuntu
+
+#installing dependencies
+RUN apt-get update
+RUN apt-get install python3 -y
+RUN apt-get install python3-pip -y
+RUN apt-get install python3-venv -y
+RUN apt-get install git -y
+#------installing the needed thing for PyPDF2 and pdftotext-------------------------------------
+RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
+RUN apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev poppler-utils -y
+RUN apt-get install libqpdf-dev -y
+RUN apt-get install pkg-config -y
+#-----------------------------------------------------------------------------------------------
+
+
+RUN git clone https://github.com/crocs-muni/sec-certs.git /opt/sec-certs
+
+#creating the venv in a way that works, the activation script doesn't work in Docker, we do manualy what it does
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Install dependencies:
+RUN cp /opt/sec-certs/requirements.txt .
+RUN pip install wheel
+RUN pip install -r requirements.txt
+#just to be sure that pdftotext is in $PATH
+ENV PATH /usr/bin/pdftotext:${PATH}
+
+
+# Run the application:
+CMD ["python3", "/opt/sec-certs/examples/cc_oop_demo.py"]
diff --git a/README.md b/README.md
index dc231d9e..320dc65c 100644
--- a/README.md
+++ b/README.md
@@ -4,10 +4,29 @@ Tool for analysis of security certificates and their security targets (Common Cr
 
 This project is developed by the [Centre for Research On Cryptography and Security](https://crocs.fi.muni.cz) at Faculty of Informatics, Masaryk University.
 
+[![Website](https://img.shields.io/website?down_color=red&down_message=offline&style=flat-square&up_color=SpringGreen&up_message=online&url=https%3A%2F%2Fseccerts.org)](https://seccerts.org)
+[![PyPI](https://img.shields.io/pypi/v/sec-certs?style=flat-square)](https://pypi.org/project/sec-certs/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sec-certs?label=Python%20versions&style=flat-square)](https://pypi.org/project/sec-certs/)
+[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/crocs-muni/sec-certs/tests?style=flat-square)](https://github.com/crocs-muni/sec-certs/actions/workflows/GA_CI.yml)
+[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/crocs-muni/sec-certs/Docker%20Image%20CI?label=Docker%20build&style=flat-square)](https://hub.docker.com/repository/docker/seccerts/sec-certs)
+
 ## Installation (CC)
 
 The tool requires several Python packages as well as the `pdftotext` binary somewhere on the `PATH`.
-The easiest way to setup the tool is to install it in a virtual environment, e.g.:
+[
+The stable release is published on [PyPi](https://pypi.org/project/sec-certs/) as well as on [DockerHub](https://hub.docker.com/repository/docker/seccerts/sec-certs), you can install it with:
+
+```
+pip install -U sec-certs
+```
+
+or
+
+```
+docker pull seccerts/sec-certs
+```
+
+Alternatively, you can setup the tool for development in a virtual environment, e.g.:
 Install Python virtual environment (if not yet):
 ```
 python3 -m pip install --upgrade pip
@@ -51,3 +70,9 @@ The analysis can be extended in several ways:
  1. Additional keywords can be extracted from PDF files (modify `cert_rules.py`)
  2. Data from `certificate_data_complete.json` can be analyzed in a novel way - this is why this project was concieved at the first place.
  3. Help to fix problems in data extraction - some PDF files are corrupted, there are many typos even in certificate IDs...
+
+## How to run the application with a Docker container
+
+ 1. pull the image from the DockerHub repository : `docker pull seccerts/sec-certs`
+ 2. run `docker run --volume ./processed_data:/opt/sec-certs/examples/debug_dataset -it seccerts/sec-certs`
+ 3. All processed data will be in the `~/processed_data` directory
diff --git a/examples/readme.md b/examples/readme.md
index 66f43661..0e6fd2a9 100644
--- a/examples/readme.md
+++ b/examples/readme.md
@@ -19,9 +19,9 @@ dset.manually_verify_cpe_matches()
 For each of the certificates, the user is then prompted for an expert knowledge, see example below:
 
 ```
-[0/1512] Vendor: NetIQ Corporation, Name: NetIQ Identity Manager 4.7
-	- [0]: CPE(uri='cpe:2.3:a:netiq:sentinel:-:*:*:*:*:*:*:*', title='NetIQ Sentinel', version='-', vendor='netiq', item_name='sentinel')
-	- [1]: CPE(uri='cpe:2.3:a:netiq:sentinel_agent_manager:-:*:*:*:*:*:*:*', title='NetIQ Sentinel Agent Manager', version='-', vendor='netiq', item_name='sentinel agent manager')
+[0/1516] Vendor: NetIQ Corporation, Name: NetIQ Identity Manager 4.7
+	- [0]: netiq NetIQ Sentinel CPE-URI: cpe:2.3:a:netiq:sentinel:-:*:*:*:*:*:*:*
+	- [1]: netiq NetIQ Sentinel Agent Manager CPE-URI: cpe:2.3:a:netiq:sentinel_agent_manager:-:*:*:*:*:*:*:*
 	- [A]: All are fitting
 	- [X]: No fitting match
 Select fitting CPE matches (split with comma if choosing more):
@@ -30,17 +30,23 @@ Select fitting CPE matches (split with comma if choosing more):
 Here, one should type `X` (case insensitive) and press enter, since all guesses are false positives. In different case
 
 ```
-[1/1512] Vendor: NetIQ, Incorporated, Name: NetIQ Access Manager 4.5
-	- [0]: CPE(uri='cpe:2.3:a:netiq:access_manager:4.5:hotfix1:*:*:*:*:*:*', title='NetIQ Access Manager 4.5 Hotfix 1', version='4.5', vendor='netiq', item_name='access manager')
-	- [1]: CPE(uri='cpe:2.3:a:netiq:access_manager:4.5:sp1:*:*:*:*:*:*', title='NetIQ Access Manager 4.5 Service Pack 1', version='4.5', vendor='netiq', item_name='access manager')
-	- [2]: CPE(uri='cpe:2.3:a:netiq:access_manager:4.5:-:*:*:*:*:*:*', title='NetIQ Access Manager 4.5', version='4.5', vendor='netiq', item_name='access manager')
+[1/1516] Vendor: NetIQ, Incorporated, Name: NetIQ Access Manager 4.5
+	- [0]: netiq NetIQ Access Manager 4.5 CPE-URI: cpe:2.3:a:netiq:access_manager:4.5:-:*:*:*:*:*:*
+	- [1]: netiq NetIQ Access Manager 4.5 Service Pack 1 CPE-URI: cpe:2.3:a:netiq:access_manager:4.5:sp1:*:*:*:*:*:*
+	- [2]: netiq NetIQ Access Manager 4.5 Hotfix 1 CPE-URI: cpe:2.3:a:netiq:access_manager:4.5:hotfix1:*:*:*:*:*:*
 	- [A]: All are fitting
 	- [X]: No fitting match
-Select fitting CPE matches (split with comma if choosing more):
 ```
 
-one may answer with `0,1,2` as all CPEs may be releated to the certificate. 
+one may answer with `0,1,2` or simply `A` as all CPEs may be releated to the certificate. 
 
 The progress of the expert is periodically saved. Currently, there's no way to gracefully exit the process, just do keyboard interrupt if you want to stop. The json will be updated and next time you will get prompted only for the unlabeled certificates. 
 
 We strongly suggest you try the process with `dset.manually_verify_cpe_matches(update_json=False)` to experiment with correct inputs/outputs. While you will get prompted again if the input is recognized incorrect, the `update_json=False` will not store the results so you can experiment with the tool without loosing your results or creating bad labels.
+
+Would you want to exit in the middle and return back after a while, just exit with CTRL+C. You can then label the rest of the unlabeled certificates with
+
+```
+dset = CCDataset.from_json('./my_debug_datset/cc_full_dataset.json')
+dset.manually_verify_cpe_matches()
+```
diff --git a/requirements.txt b/requirements.txt
index c417d3f2..d5c7df10 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+wheel==0.34.2
 beautifulsoup4==4.9.3
 certifi==2020.11.8
 chardet==3.0.4
@@ -25,4 +26,4 @@ tabulate==0.8.7
 tqdm==4.51.0
 urllib3==1.25.11
 rapidfuzz==1.1.1
-pyyaml
\ No newline at end of file
+pyyaml
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index 12cb6adc..85ab5344 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -905,20 +905,23 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         cpe_matches: Optional[List[Tuple[float, CPE]]]
         verified_cpe_matches: Optional[List[CPE]]
         related_cves: Optional[List[str]]
+        labeled: bool
 
         def __init__(self,
                      extracted_versions: Optional[List[str]] = None,
                      cpe_matches: Optional[List[str]] = None,
                      verified_cpe_matches: Optional[List[str]] = None,
-                     related_cves: Optional[List[CVE]] = None):
+                     related_cves: Optional[List[CVE]] = None,
+                     labeled: bool = False):
             self.extracted_versions = extracted_versions
             self.cpe_matches = cpe_matches
             self.cpe_candidate_vendors = None
             self.verified_cpe_matches = verified_cpe_matches
             self.related_cves = related_cves
+            self.labeled = labeled
 
         def to_dict(self):
-            return {'extracted_versions': self.extracted_versions, 'cpe_matches': self.cpe_matches, 'verified_cpe_matches': self.verified_cpe_matches, 'related_cves': self.related_cves}
+            return {'extracted_versions': self.extracted_versions, 'cpe_matches': self.cpe_matches, 'verified_cpe_matches': self.verified_cpe_matches, 'related_cves': self.related_cves, 'labeled': self.labeled}
 
         @classmethod
         def from_dict(cls, dct: Dict[str, str]):
diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index 4a4cf577..99d81640 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -7,7 +7,7 @@ RETURNCODE_NOK = 'nok'
 REQUEST_TIMEOUT = 10
 
 CPE_MATCHING_THRESHOLD = 70
-CPE_MAX_MATCHES = 10
+CPE_MAX_MATCHES = 20
 
 MIN_CORRECT_CERT_SIZE = 5000
 
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index b80b25da..7369cdf1 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -735,7 +735,7 @@ class CCDataset(Dataset, ComplexSerializableType):
             for i, x in enumerate(certificates_to_verify):
                 print(f'\n[{i}/{n_certs_to_verify}] Vendor: {x.manufacturer}, Name: {x.name}')
                 for index, c in enumerate(x.heuristics.cpe_matches):
-                    print(f'\t- {[index]}: {c[1]}')
+                    print(f'\t- {[index]}: {c[1].vendor} {c[1].title} CPE-URI: {c[1].uri}')
                 print(f'\t- [A]: All are fitting')
                 print(f'\t- [X]: No fitting match')
                 inpts = input('Select fitting CPE matches (split with comma if choosing more):').strip().split(',')
@@ -757,12 +757,12 @@ class CCDataset(Dataset, ComplexSerializableType):
                         matches = [x.heuristics.cpe_matches[y][1] for y in inpts]
                         self[x.dgst].heuristics.verified_cpe_matches = matches
 
-                        if i != 0 and not i % 10 and update_json:
-                            print(f'Saving progress.')
-                            self.to_json()
+                if i != 0 and not i % 10 and update_json:
+                    print(f'Saving progress.')
+                    self.to_json()
+                self[x.dgst].heuristics.labeled = True
 
-        certs_to_verify: List[CommonCriteriaCert] = [x for x in self if (
-                x.heuristics.cpe_matches and not x.heuristics.verified_cpe_matches)]
+        certs_to_verify: List[CommonCriteriaCert] = [x for x in self if (x.heuristics.cpe_matches and not x.heuristics.labeled)]
         logger.info('Manually verifying CPE matches')
         time.sleep(0.05)  # easier than flushing the logger
         verify_certs(certs_to_verify)
diff --git a/fips_certificates.py b/sec_certs/entrypoints/fips_certificates.py
index eb147d48..eb147d48 100755
--- a/fips_certificates.py
+++ b/sec_certs/entrypoints/fips_certificates.py
diff --git a/process_certificates.py b/sec_certs/entrypoints/process_certificates.py
index 5272fc67..5272fc67 100755..100644
--- a/process_certificates.py
+++ b/sec_certs/entrypoints/process_certificates.py
diff --git a/setup.py b/setup.py
index cf6f3e5a..d3a80ce0 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,8 @@ setup(
     name='sec-certs',
     author='Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky',
     author_email='svenda@fi.muni.cz',
-    version='0.0.0',
+    version_config=True,
+    setup_requires=['setuptools-git-versioning'],
     packages=find_packages(),
     license='MIT',
     description="Tool for analysis of security certificates",
@@ -19,7 +20,8 @@ setup(
         "License :: OSI Approved :: MIT License",
         "Topic :: Security",
         "Topic :: Security :: Cryptography",
-        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Intended Audience :: Developers",
         "Intended Audience :: Science/Research"
     ],
@@ -31,7 +33,7 @@ setup(
     },
     entry_points="""
         [console_scripts]
-        process-certs=process_certificates:main
-        fips-certs=fips_certificates:main
+        process-certs=sec_certs.entrypoints.process_certificates:main
+        fips-certs=sec_certs.entrypoints.fips_certificates:main
     """
 )
diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json
index 3856548d..9a01b8eb 100644
--- a/test/data/test_cc_oop/fictional_cert.json
+++ b/test/data/test_cc_oop/fictional_cert.json
@@ -56,6 +56,7 @@
         "extracted_versions": null,
         "cpe_matches": null,
         "verified_cpe_matches": null,
-        "related_cves": null
+        "related_cves": null,
+        "labeled": false
     }
 }
\ No newline at end of file
diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json
index 62ea04de..7f1ce7f7 100644
--- a/test/data/test_cc_oop/toy_dataset.json
+++ b/test/data/test_cc_oop/toy_dataset.json
@@ -59,7 +59,8 @@
                 "extracted_versions": null,
                 "cpe_matches": null,
                 "verified_cpe_matches": null,
-                "related_cves": null
+                "related_cves": null,
+                "labeled": false
             }
         },
         {
@@ -110,7 +111,8 @@
                 "extracted_versions": null,
                 "cpe_matches": null,
                 "verified_cpe_matches": null,
-                "related_cves": null
+                "related_cves": null,
+                "labeled": false
             }
         }
     ]