From fa6a62ac2dae2d63873c9479e66661e3d2e34a44 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 16 Apr 2021 09:25:58 +0200 Subject: Update README.md adding instructions about how to build and run the docker image, next step is to push the final image to DockerHub.--- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 552750ab..41636b1d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,13 @@ Tool for analysis of security certificates and their security targets (Common Cr This project is developed by the [Centre for Research On Cryptography and Security](https://crocs.fi.muni.cz) at Faculty of Informatics, Masaryk University. +## Usage (Dockerfile) + +1. Run `sudo apt install docker.io` +2. Go into the directory where the Dockerfile is +3. Run `sudo docker build . -t NAME_OF_YOUR_IMAGE` +4. The image is now created in your local Docker repo, use `sudo docker run NAME_OF_YOUR_IMAGE:latest` to run the container + ## Usage (CC) The tool requires several Python packages as well as the `pdftotext` binary somewhere on the `PATH`. -- cgit v1.3.1 From 90fb4b213aa9a2516a25a5754a3a32bd781dba53 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 16 Apr 2021 09:36:56 +0200 Subject: Create Dockerfile Adding the Dockerfile to the repo, we can build the container with it, and run the demo script (/!\reminder: 3 hours long to run) Next step is to push the image to DockerHub, so we won't need to build the image anymore.--- Dockerfile | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3ac77ff2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu + +#installing dependencies +RUN apt-get update +RUN apt-get install python3 -y +RUN apt-get install python3-pip -y +RUN apt-get install python3-venv -y +RUN apt-get install git -y +#------installing the needed thing for PyPDF2 and pdftotext------------------------------------- +RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata +RUN apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev poppler-utils -y +RUN apt-get install libqpdf-dev -y +RUN apt-get install pkg-config -y +#----------------------------------------------------------------------------------------------- + + +RUN git clone https://github.com/crocs-muni/sec-certs.git /opt/sec-certs + +#creating the venv in a way that works, the activation script doesn't work in Docker, we do manualy what it does +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Install dependencies: +RUN cp /opt/sec-certs/requirements.txt . +RUN pip install wheel +RUN pip install -r requirements.txt +#just to be sure that pdftotext is in $PATH +ENV PATH /usr/bin/pdftotext:${PATH} + + +# Run the application: +CMD ["python3", "/opt/sec-certs/cc_oop_demo.py"] -- cgit v1.3.1 From df2da49efb917da2b3466cb8d508b1ca6fc55f54 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 16 Apr 2021 14:52:19 +0200 Subject: Create docker-image.yml First try for a yaml file to automatize the build and push of a docker image--- .github/workflows/docker-image.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/docker-image.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 00000000..c0298502 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,25 @@ +# code created with help of the github documentation : https://docs.github.com/en/actions/guides/publishing-docker-images + +name: Docker Image CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build-and-push: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Push to Docker Hub + uses: docker/build-push-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} #we can set these secrets with admin rights, so the code will adapt to the dockerHub account used + password: ${{ secrets.DOCKER_PASSWORD }} + repository: keleran/sec-certs #for now I wrote my repo + tag_with_ref: true -- cgit v1.3.1 From 27486822202b50b2dd1cdf2dc95d143671b1ae25 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 16 Apr 2021 15:17:37 +0200 Subject: Update docker-image.yml switch to v2 of build-push action as asked by the warning--- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c0298502..7500554d 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -17,7 +17,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v2 - name: Push to Docker Hub - uses: docker/build-push-action@v1 + uses: docker/build-push-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} #we can set these secrets with admin rights, so the code will adapt to the dockerHub account used password: ${{ secrets.DOCKER_PASSWORD }} -- cgit v1.3.1 From e1fa06edf8f3157fb7c90bc036ce1a80a31b9eb7 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 16 Apr 2021 15:37:47 +0200 Subject: Create python-publish.yml Attempt to add a github action to publish a PyPI package on each release.--- .github/workflows/python-publish.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..088bc13d --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,31 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} #Same as Docker, need to add an account in the secrets + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* -- cgit v1.3.1 From f5d2eebfe806761f74ac5ac00b8bc74c12ce1de3 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Mon, 19 Apr 2021 14:20:32 +0200 Subject: Update python-publish.yml calls the publisher on each push--- .github/workflows/python-publish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 088bc13d..0f442307 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -3,9 +3,7 @@ name: Upload Python Package -on: - release: - types: [created] +on: [push] jobs: deploy: -- cgit v1.3.1 From 3d85388d517ba4ab12dfb7ab563c66d9be74df1c Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Mon, 19 Apr 2021 16:49:38 +0200 Subject: Update requirements.txt add wheel requirement in order to build PyPDF2 properly.--- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 11ca661e..3236104f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +wheel==0.34.2 beautifulsoup4==4.9.3 certifi==2020.11.8 chardet==3.0.4 @@ -25,4 +26,4 @@ tabulate==0.8.7 tqdm==4.51.0 urllib3==1.25.11 rapidfuzz==1.1.1 -pyyaml \ No newline at end of file +pyyaml -- cgit v1.3.1 -- cgit v1.3.1 From 56aba3e8a8668c7b5caba116a73420d2afed663a Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 19 Apr 2021 17:48:35 +0200 Subject: improve interface of manual cpe labeling --- examples/readme.md | 24 +++++++++++++++--------- sec_certs/dataset.py | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/examples/readme.md b/examples/readme.md index 66f43661..0e6fd2a9 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -19,9 +19,9 @@ dset.manually_verify_cpe_matches() For each of the certificates, the user is then prompted for an expert knowledge, see example below: ``` -[0/1512] Vendor: NetIQ Corporation, Name: NetIQ Identity Manager 4.7 - - [0]: CPE(uri='cpe:2.3:a:netiq:sentinel:-:*:*:*:*:*:*:*', title='NetIQ Sentinel', version='-', vendor='netiq', item_name='sentinel') - - [1]: CPE(uri='cpe:2.3:a:netiq:sentinel_agent_manager:-:*:*:*:*:*:*:*', title='NetIQ Sentinel Agent Manager', version='-', vendor='netiq', item_name='sentinel agent manager') +[0/1516] Vendor: NetIQ Corporation, Name: NetIQ Identity Manager 4.7 + - [0]: netiq NetIQ Sentinel CPE-URI: cpe:2.3:a:netiq:sentinel:-:*:*:*:*:*:*:* + - [1]: netiq NetIQ Sentinel Agent Manager CPE-URI: cpe:2.3:a:netiq:sentinel_agent_manager:-:*:*:*:*:*:*:* - [A]: All are fitting - [X]: No fitting match Select fitting CPE matches (split with comma if choosing more): @@ -30,17 +30,23 @@ Select fitting CPE matches (split with comma if choosing more): Here, one should type `X` (case insensitive) and press enter, since all guesses are false positives. In different case ``` -[1/1512] Vendor: NetIQ, Incorporated, Name: NetIQ Access Manager 4.5 - - [0]: CPE(uri='cpe:2.3:a:netiq:access_manager:4.5:hotfix1:*:*:*:*:*:*', title='NetIQ Access Manager 4.5 Hotfix 1', version='4.5', vendor='netiq', item_name='access manager') - - [1]: CPE(uri='cpe:2.3:a:netiq:access_manager:4.5:sp1:*:*:*:*:*:*', title='NetIQ Access Manager 4.5 Service Pack 1', version='4.5', vendor='netiq', item_name='access manager') - - [2]: CPE(uri='cpe:2.3:a:netiq:access_manager:4.5:-:*:*:*:*:*:*', title='NetIQ Access Manager 4.5', version='4.5', vendor='netiq', item_name='access manager') +[1/1516] Vendor: NetIQ, Incorporated, Name: NetIQ Access Manager 4.5 + - [0]: netiq NetIQ Access Manager 4.5 CPE-URI: cpe:2.3:a:netiq:access_manager:4.5:-:*:*:*:*:*:* + - [1]: netiq NetIQ Access Manager 4.5 Service Pack 1 CPE-URI: cpe:2.3:a:netiq:access_manager:4.5:sp1:*:*:*:*:*:* + - [2]: netiq NetIQ Access Manager 4.5 Hotfix 1 CPE-URI: cpe:2.3:a:netiq:access_manager:4.5:hotfix1:*:*:*:*:*:* - [A]: All are fitting - [X]: No fitting match -Select fitting CPE matches (split with comma if choosing more): ``` -one may answer with `0,1,2` as all CPEs may be releated to the certificate. +one may answer with `0,1,2` or simply `A` as all CPEs may be releated to the certificate. The progress of the expert is periodically saved. Currently, there's no way to gracefully exit the process, just do keyboard interrupt if you want to stop. The json will be updated and next time you will get prompted only for the unlabeled certificates. We strongly suggest you try the process with `dset.manually_verify_cpe_matches(update_json=False)` to experiment with correct inputs/outputs. While you will get prompted again if the input is recognized incorrect, the `update_json=False` will not store the results so you can experiment with the tool without loosing your results or creating bad labels. + +Would you want to exit in the middle and return back after a while, just exit with CTRL+C. You can then label the rest of the unlabeled certificates with + +``` +dset = CCDataset.from_json('./my_debug_datset/cc_full_dataset.json') +dset.manually_verify_cpe_matches() +``` diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 5845e4d3..0539181e 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -732,7 +732,7 @@ class CCDataset(Dataset, ComplexSerializableType): for i, x in enumerate(certificates_to_verify): print(f'\n[{i}/{n_certs_to_verify}] Vendor: {x.manufacturer}, Name: {x.name}') for index, c in enumerate(x.heuristics.cpe_matches): - print(f'\t- {[index]}: {c[1]}') + print(f'\t- {[index]}: {c[1].vendor} {c[1].title} CPE-URI: {c[1].uri}') print(f'\t- [A]: All are fitting') print(f'\t- [X]: No fitting match') inpts = input('Select fitting CPE matches (split with comma if choosing more):').strip().split(',') -- cgit v1.3.1 From 902638c5ed0e3aed572925c4ef9124d8733612fb Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 20 Apr 2021 10:28:03 +0200 Subject: increase max number of cpe matches --- sec_certs/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sec_certs/constants.py b/sec_certs/constants.py index 610e4cc9..645f632d 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -7,7 +7,7 @@ RETURNCODE_NOK = 'nok' REQUEST_TIMEOUT = 10 CPE_MATCHING_THRESHOLD = 70 -CPE_MAX_MATCHES = 10 +CPE_MAX_MATCHES = 100 MIN_CORRECT_CERT_SIZE = 5000 -- cgit v1.3.1 From 9028613aac7441c8eaf1030800481b9130ef312c Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 20 Apr 2021 11:00:41 +0200 Subject: fix in cpe labeling progress save --- sec_certs/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 0539181e..b3a836af 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -753,9 +753,9 @@ class CCDataset(Dataset, ComplexSerializableType): matches = [x.heuristics.cpe_matches[y][1] for y in inpts] self[x.dgst].heuristics.verified_cpe_matches = matches - if i != 0 and not i % 10 and update_json: - print(f'Saving progress.') - self.to_json() + if i != 0 and not i % 10 and update_json: + print(f'Saving progress.') + self.to_json() certs_to_verify: List[CommonCriteriaCert] = [x for x in self if (x.heuristics.cpe_matches and not x.heuristics.verified_cpe_matches)] logger.info('Manually verifying CPE matches') -- cgit v1.3.1 From 748561994627b5fe7862d92b1220dfe589240d32 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 20 Apr 2021 11:12:14 +0200 Subject: fix labeling error progress save --- sec_certs/certificate.py | 7 +++++-- sec_certs/dataset.py | 9 +++++---- test/data/test_cc_oop/fictional_cert.json | 3 ++- test/data/test_cc_oop/toy_dataset.json | 6 ++++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index b873ac37..7b0770de 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -878,20 +878,23 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): cpe_matches: Optional[List[Tuple[float, CPE]]] verified_cpe_matches: Optional[List[CPE]] related_cves: Optional[List[str]] + labeled: bool def __init__(self, extracted_versions: Optional[List[str]] = None, cpe_matches: Optional[List[str]] = None, verified_cpe_matches: Optional[List[str]] = None, - related_cves: Optional[List[CVE]] = None): + related_cves: Optional[List[CVE]] = None, + labeled: bool = False): self.extracted_versions = extracted_versions self.cpe_matches = cpe_matches self.cpe_candidate_vendors = None self.verified_cpe_matches = verified_cpe_matches self.related_cves = related_cves + self.labeled = labeled def to_dict(self): - return {'extracted_versions': self.extracted_versions, 'cpe_matches': self.cpe_matches, 'verified_cpe_matches': self.verified_cpe_matches, 'related_cves': self.related_cves} + return {'extracted_versions': self.extracted_versions, 'cpe_matches': self.cpe_matches, 'verified_cpe_matches': self.verified_cpe_matches, 'related_cves': self.related_cves, 'labeled': self.labeled} @classmethod def from_dict(cls, dct: Dict[str, str]): diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index b3a836af..c1ac4f35 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -753,11 +753,12 @@ class CCDataset(Dataset, ComplexSerializableType): matches = [x.heuristics.cpe_matches[y][1] for y in inpts] self[x.dgst].heuristics.verified_cpe_matches = matches - if i != 0 and not i % 10 and update_json: - print(f'Saving progress.') - self.to_json() + if i != 0 and not i % 10 and update_json: + print(f'Saving progress.') + self.to_json() + self[x.dgst].heuristics.labeled = True - certs_to_verify: List[CommonCriteriaCert] = [x for x in self if (x.heuristics.cpe_matches and not x.heuristics.verified_cpe_matches)] + certs_to_verify: List[CommonCriteriaCert] = [x for x in self if (x.heuristics.cpe_matches and not x.heuristics.labeled)] logger.info('Manually verifying CPE matches') time.sleep(0.05) # easier than flushing the logger verify_certs(certs_to_verify) diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json index 3856548d..9a01b8eb 100644 --- a/test/data/test_cc_oop/fictional_cert.json +++ b/test/data/test_cc_oop/fictional_cert.json @@ -56,6 +56,7 @@ "extracted_versions": null, "cpe_matches": null, "verified_cpe_matches": null, - "related_cves": null + "related_cves": null, + "labeled": false } } \ No newline at end of file diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json index 62ea04de..7f1ce7f7 100644 --- a/test/data/test_cc_oop/toy_dataset.json +++ b/test/data/test_cc_oop/toy_dataset.json @@ -59,7 +59,8 @@ "extracted_versions": null, "cpe_matches": null, "verified_cpe_matches": null, - "related_cves": null + "related_cves": null, + "labeled": false } }, { @@ -110,7 +111,8 @@ "extracted_versions": null, "cpe_matches": null, "verified_cpe_matches": null, - "related_cves": null + "related_cves": null, + "labeled": false } } ] -- cgit v1.3.1 From 4770f7d4aa81235d6053ede19e46a90004af61ca Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Tue, 20 Apr 2021 11:19:14 +0200 Subject: Create GA_CI.yml First attempt to use GitHub actions instead of Travis CI--- .github/workflows/GA_CI.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/GA_CI.yml diff --git a/.github/workflows/GA_CI.yml b/.github/workflows/GA_CI.yml new file mode 100644 index 00000000..1a3231e5 --- /dev/null +++ b/.github/workflows/GA_CI.yml @@ -0,0 +1,24 @@ +name: tests +on: + push: + branches: + - master + - dev + +jobs: + name: Set env up + runs-on: ubuntu-latest + steps: + - run: apt-get install poppler-utils + - uses: actions/checkout@v2 + - name : Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install python dependencies + run: pip install -r requirements.txt + - name : Install pytest and run scripts like Travis does + run: | + pip install pytest + pip install pytest-cov + pytest test -- cgit v1.3.1 From d0252bad41d78877b0912dd070677b6fde2daacc Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Tue, 20 Apr 2021 11:21:22 +0200 Subject: Update GA_CI.yml Corrected syntax errors--- .github/workflows/GA_CI.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/GA_CI.yml b/.github/workflows/GA_CI.yml index 1a3231e5..22c334c5 100644 --- a/.github/workflows/GA_CI.yml +++ b/.github/workflows/GA_CI.yml @@ -6,19 +6,19 @@ on: - dev jobs: - name: Set env up - runs-on: ubuntu-latest - steps: - - run: apt-get install poppler-utils - - uses: actions/checkout@v2 - - name : Setup python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install python dependencies - run: pip install -r requirements.txt - - name : Install pytest and run scripts like Travis does - run: | - pip install pytest - pip install pytest-cov - pytest test + run-test: + runs-on: ubuntu-latest + steps: + - run: apt-get install poppler-utils + - uses: actions/checkout@v2 + - name : Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install python dependencies + run: pip install -r requirements.txt + - name : Install pytest and run scripts like Travis does + run: | + pip install pytest + pip install pytest-cov + pytest test -- cgit v1.3.1 From 00e0152bd1a8ed79153f6f3c2263dd90ff387c66 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 20 Apr 2021 11:29:23 +0200 Subject: adjust cpe matching constant --- sec_certs/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sec_certs/constants.py b/sec_certs/constants.py index 645f632d..f19919a6 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -7,7 +7,7 @@ RETURNCODE_NOK = 'nok' REQUEST_TIMEOUT = 10 CPE_MATCHING_THRESHOLD = 70 -CPE_MAX_MATCHES = 100 +CPE_MAX_MATCHES = 20 MIN_CORRECT_CERT_SIZE = 5000 -- cgit v1.3.1 From 11078d51591f5a9ce344915e8178fede502f6dfe Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Wed, 21 Apr 2021 10:23:27 +0200 Subject: Update GA_CI.yml Modify the event that triggers the workflow, now it will run on each push everywhere--- .github/workflows/GA_CI.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/GA_CI.yml b/.github/workflows/GA_CI.yml index 22c334c5..25f9c7b8 100644 --- a/.github/workflows/GA_CI.yml +++ b/.github/workflows/GA_CI.yml @@ -1,9 +1,6 @@ name: tests -on: - push: - branches: - - master - - dev +on: [push] + jobs: run-test: -- cgit v1.3.1 From 5a9841c2668a6e09617b6d20fe07a822cc18042d Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Wed, 21 Apr 2021 10:34:08 +0200 Subject: Update GA_CI.yml sudo might be needed to install an apt package--- .github/workflows/GA_CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/GA_CI.yml b/.github/workflows/GA_CI.yml index 25f9c7b8..e26b2562 100644 --- a/.github/workflows/GA_CI.yml +++ b/.github/workflows/GA_CI.yml @@ -6,7 +6,7 @@ jobs: run-test: runs-on: ubuntu-latest steps: - - run: apt-get install poppler-utils + - run: sudo apt-get install poppler-utils - uses: actions/checkout@v2 - name : Setup python uses: actions/setup-python@v2 -- cgit v1.3.1 From 806d5cc35ecd2bfc3abd6f234bcabc145ac4da70 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Wed, 21 Apr 2021 11:00:08 +0200 Subject: Update setup.py Attempt to add a package to setup.py to set the version number according to a commit tag (e.g. v1.0.0 or 1.0.0), link to the repo used : https://github.com/dolfinus/setuptools-git-versioning--- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cf6f3e5a..5aae20b2 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,8 @@ setup( name='sec-certs', author='Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky', author_email='svenda@fi.muni.cz', - version='0.0.0', + version_config=True, + setup_requires=['setuptools-git-versioning'], packages=find_packages(), license='MIT', description="Tool for analysis of security certificates", -- cgit v1.3.1 From 1b637cbb3b287e5607e78a2be4153eee6b8aa846 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Wed, 21 Apr 2021 11:13:20 +0200 Subject: Update python-publish.yml The build seems to work well, modify the rules to publish the package when a push or PR on master occurs--- .github/workflows/python-publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 0f442307..fb2b2323 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -3,7 +3,11 @@ name: Upload Python Package -on: [push] +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] jobs: deploy: -- cgit v1.3.1 From 0558f03b36b4e9f93d99c450860c061c2dce0186 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Wed, 21 Apr 2021 11:16:00 +0200 Subject: Update GA_CI.yml pytest didn't find the sec_certs code, adding a line that was in travis.yml--- .github/workflows/GA_CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/GA_CI.yml b/.github/workflows/GA_CI.yml index e26b2562..d4b00ff6 100644 --- a/.github/workflows/GA_CI.yml +++ b/.github/workflows/GA_CI.yml @@ -18,4 +18,5 @@ jobs: run: | pip install pytest pip install pytest-cov + pip install ".[dev,test]" pytest test -- cgit v1.3.1 From 0d5882cc8c2a09601111659cd85ddca8b6d31d31 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Wed, 21 Apr 2021 11:55:09 +0200 Subject: Update docker-image.yml Revision of the code thanks to the reviews, we'll now only use the official method.--- .github/workflows/docker-image.yml | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 7500554d..d8b8f5a0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -7,19 +7,27 @@ on: branches: [ master ] pull_request: branches: [ master ] - + jobs: - - build-and-push: - name: Push Docker image to Docker Hub + docker: runs-on: ubuntu-latest steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Push to Docker Hub + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_LOGIN }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + id: docker_build uses: docker/build-push-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} #we can set these secrets with admin rights, so the code will adapt to the dockerHub account used - password: ${{ secrets.DOCKER_PASSWORD }} - repository: keleran/sec-certs #for now I wrote my repo - tag_with_ref: true + push: true + tags: user/app:latest -- cgit v1.3.1 From 24443ae5e16fde5323177bde3fedbd73d90e61fc Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Thu, 22 Apr 2021 09:47:40 +0200 Subject: Delete .travis.yml Deleting the Travis conf file since it is not useful anymore--- .travis.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e67232e0..00000000 --- a/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -os: linux -language: python -dist: xenial -python: "3.8" - -before_install: - - sudo apt-get -y install poppler-utils - -install: - - pip install ".[dev,test]" - -script: - - pytest test -- cgit v1.3.1 From ce479e95c1f22eb5223d8ea25eed996f0c018add Mon Sep 17 00:00:00 2001 From: dependabot[bot] Date: Thu, 22 Apr 2021 15:19:47 +0000 Subject: Bump pikepdf from 2.0.0 to 2.10.0 Bumps [pikepdf](https://github.com/pikepdf/pikepdf) from 2.0.0 to 2.10.0. - [Release notes](https://github.com/pikepdf/pikepdf/releases) - [Changelog](https://github.com/pikepdf/pikepdf/blob/master/docs/release_notes.rst) - [Commits](https://github.com/pikepdf/pikepdf/compare/v2.0.0...v2.10.0) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 356e3e83..c417d3f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ lxml==4.6.3 matplotlib==3.3.2 numpy==1.19.4 pandas==1.1.4 -pikepdf==2.0.0 +pikepdf==2.10.0 Pillow==8.1.1 pyparsing==2.4.7 PyPDF2==1.26.0 -- cgit v1.3.1 From 4652dde511aace0a89f8a9631fe7644921e4e45a Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 22 Apr 2021 17:23:49 +0200 Subject: default branch change in cotributing --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a06ada3..7184033e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,3 +8,7 @@ You contribution is warmly welcomed. You can help by: 3. Perform additional analysis with extracted data (analyze_certificates.py) 3. Improve the code (TODO: Follow Github contribution guidelines, ideally contact us first about your plan) +## Branches + +- `dev` is the default branch against which all pull requests are to be made. +- `master` is protected branch where stable releases reside. Each push or PR to master should be properly tagged by [semantic version](https://semver.org/) as it will invoke actions to publish docker image and PyPi package. \ No newline at end of file -- cgit v1.3.1 From 88707e8abf413a9c22d3bf4191354777225d0403 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 23 Apr 2021 11:08:48 +0200 Subject: Update GA_CI.yml Forcing 3.8 version of Python to ensure compatibility--- .github/workflows/GA_CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/GA_CI.yml b/.github/workflows/GA_CI.yml index d4b00ff6..3239468f 100644 --- a/.github/workflows/GA_CI.yml +++ b/.github/workflows/GA_CI.yml @@ -11,7 +11,7 @@ jobs: - name : Setup python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.8' - name: Install python dependencies run: pip install -r requirements.txt - name : Install pytest and run scripts like Travis does -- cgit v1.3.1 From 8d3b03c02fa550d6c73e5993797ffce4efc66afa Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 23 Apr 2021 11:11:36 +0200 Subject: Update python-publish.yml Forcing 3.8 for compatibility purpose--- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index fb2b2323..22f85fe2 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.8' - name: Install dependencies run: | python -m pip install --upgrade pip -- cgit v1.3.1 From ed69fdd0c83160653ee363ef2b3ebfe0783db88e Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Fri, 23 Apr 2021 11:31:54 +0200 Subject: Update Dockerfile Modified the path of the demo script.--- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3ac77ff2..e891e9a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,4 +30,4 @@ ENV PATH /usr/bin/pdftotext:${PATH} # Run the application: -CMD ["python3", "/opt/sec-certs/cc_oop_demo.py"] +CMD ["python3", "/opt/sec-certs/examples/cc_oop_demo.py"] -- cgit v1.3.1 From f61d3db18d7b9803d648281d1444b915cdcc8484 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 11:18:52 +0200 Subject: test manual dispatch of GitHub Actions --- .github/workflows/docker-image.yml | 21 +++++++++------------ README.md | 5 ++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d8b8f5a0..d76ffc16 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -4,28 +4,25 @@ name: Docker Image CI on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] - + branches: [master] + workflow_dispatch: + jobs: docker: runs-on: ubuntu-latest steps: - - - name: Set up QEMU + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_LOGIN }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push + - name: Build and push id: docker_build uses: docker/build-push-action@v2 with: diff --git a/README.md b/README.md index b956b44d..35078d09 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,13 @@ Tool for analysis of security certificates and their security targets (Common Cr This project is developed by the [Centre for Research On Cryptography and Security](https://crocs.fi.muni.cz) at Faculty of Informatics, Masaryk University. +[![Website seccerts.org](https://img.shields.io/website-up-down-green-red/http/monip.org.svg)](http://seccerts.org/) +[![PyPI version sec-certs](https://badge.fury.io/py/ansicolortags.svg)](https://pypi.org/project/sec-certs/) +[![PyPI pyversions](https://img.shields.io/pypi/pyversions/ansicolortags.svg)](https://pypi.python.org/pypi/sec-certs/) +![GitHub actions tests](https://github.com/crocs-muni/sec-certs/actions/workflows/GA_CI.yml/badge.svg) ## Installation (CC) - The tool requires several Python packages as well as the `pdftotext` binary somewhere on the `PATH`. The easiest way to setup the tool is to install it in a virtual environment, e.g.: Install Python virtual environment (if not yet): -- cgit v1.3.1 From 5c043f881c547654d254cf4b461bc56a69291b04 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 11:24:11 +0200 Subject: fix docker username --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d76ffc16..f9a77ee8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -20,7 +20,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_LOGIN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build and push id: docker_build -- cgit v1.3.1 From 961b603b09667eabe445c360044aaf4afe789c4b Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 11:49:48 +0200 Subject: fix docker hub repo name --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index f9a77ee8..d23e1ef1 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -27,4 +27,4 @@ jobs: uses: docker/build-push-action@v2 with: push: true - tags: user/app:latest + tags: seccerts/sec-certs:tagname -- cgit v1.3.1 From cdcdac9d6ede6db0191a1f4a88a51528dd4d1638 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 11:59:31 +0200 Subject: test action to get version name --- .github/workflows/get_version.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/get_version.yml diff --git a/.github/workflows/get_version.yml b/.github/workflows/get_version.yml new file mode 100644 index 00000000..804739db --- /dev/null +++ b/.github/workflows/get_version.yml @@ -0,0 +1,10 @@ +name: get-version +on: [push, workflow_dispatch] + +steps: + - id: get_version + uses: battila7/get-version-action@v2 + + - run: echo ${{ steps.get_version.outputs.version }} + + - run: echo ${{ steps.get_version.outputs.version-without-v }} -- cgit v1.3.1 From 5764b440019a00090043908dfbc5f1e4b3b91c17 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 12:01:19 +0200 Subject: do not get version --- .github/workflows/get_version.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .github/workflows/get_version.yml diff --git a/.github/workflows/get_version.yml b/.github/workflows/get_version.yml deleted file mode 100644 index 804739db..00000000 --- a/.github/workflows/get_version.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: get-version -on: [push, workflow_dispatch] - -steps: - - id: get_version - uses: battila7/get-version-action@v2 - - - run: echo ${{ steps.get_version.outputs.version }} - - - run: echo ${{ steps.get_version.outputs.version-without-v }} -- cgit v1.3.1 From d816fab0289796540e5b4c9621b58424f51a01d2 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 12:08:58 +0200 Subject: add badges --- .github/workflows/docker-image.yml | 2 +- README.md | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d23e1ef1..fc0ca7f0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -27,4 +27,4 @@ jobs: uses: docker/build-push-action@v2 with: push: true - tags: seccerts/sec-certs:tagname + tags: seccerts/sec-certs:latest diff --git a/README.md b/README.md index 35078d09..144d7bb6 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,14 @@ Tool for analysis of security certificates and their security targets (Common Cr This project is developed by the [Centre for Research On Cryptography and Security](https://crocs.fi.muni.cz) at Faculty of Informatics, Masaryk University. -[![Website seccerts.org](https://img.shields.io/website-up-down-green-red/http/monip.org.svg)](http://seccerts.org/) -[![PyPI version sec-certs](https://badge.fury.io/py/ansicolortags.svg)](https://pypi.org/project/sec-certs/) -[![PyPI pyversions](https://img.shields.io/pypi/pyversions/ansicolortags.svg)](https://pypi.python.org/pypi/sec-certs/) -![GitHub actions tests](https://github.com/crocs-muni/sec-certs/actions/workflows/GA_CI.yml/badge.svg) +![Website](https://img.shields.io/website?down_color=red&down_message=offline&style=flat-square&up_color=SpringGreen&up_message=online&url=https%3A%2F%2Fseccerts.org) + +![PyPI](https://img.shields.io/pypi/v/sec-certs?style=flat-square) + +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sec-certs?label=Python%20versions&style=flat-square) + +![GitHub Workflow Status](https://img.shields.io/github/workflow/status/crocs-muni/sec-certs/tests?style=flat-square) + ## Installation (CC) -- cgit v1.3.1 From 56ac88fbe875f1518cab5d1d607eb1b5e2f38013 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 12:24:50 +0200 Subject: improve setup version classifiers --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5aae20b2..494dbd23 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,8 @@ setup( "License :: OSI Approved :: MIT License", "Topic :: Security", "Topic :: Security :: Cryptography", - "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Intended Audience :: Developers", "Intended Audience :: Science/Research" ], -- cgit v1.3.1 From cb02c5a6b48cd9b71a0f5a23925bcb8f6b036d66 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 24 Apr 2021 12:44:51 +0200 Subject: add badges, workflow dispatch on publish --- .github/workflows/python-publish.yml | 40 ++++++++++++++++++------------------ README.md | 24 +++++++++++++++------- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 22f85fe2..2fab292d 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -3,31 +3,31 @@ name: Upload Python Package -on: +on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] + workflow_dispatch: jobs: deploy: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} #Same as Docker, need to add an account in the secrets - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} #Same as Docker, need to add an account in the secrets + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/README.md b/README.md index 144d7bb6..340f1329 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,29 @@ Tool for analysis of security certificates and their security targets (Common Cr This project is developed by the [Centre for Research On Cryptography and Security](https://crocs.fi.muni.cz) at Faculty of Informatics, Masaryk University. -![Website](https://img.shields.io/website?down_color=red&down_message=offline&style=flat-square&up_color=SpringGreen&up_message=online&url=https%3A%2F%2Fseccerts.org) +[![Website](https://img.shields.io/website?down_color=red&down_message=offline&style=flat-square&up_color=SpringGreen&up_message=online&url=https%3A%2F%2Fseccerts.org)](https://seccerts.org) +[![PyPI](https://img.shields.io/pypi/v/sec-certs?style=flat-square)](https://pypi.org/project/sec-certs/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sec-certs?label=Python%20versions&style=flat-square)](https://pypi.org/project/sec-certs/) +[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/crocs-muni/sec-certs/tests?style=flat-square)](https://github.com/crocs-muni/sec-certs/actions/workflows/GA_CI.yml) +[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/crocs-muni/sec-certs/Docker%20Image%20CI?label=Docker%20build&style=flat-square)](https://hub.docker.com/repository/docker/seccerts/sec-certs) -![PyPI](https://img.shields.io/pypi/v/sec-certs?style=flat-square) +## Installation (CC) -![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sec-certs?label=Python%20versions&style=flat-square) +The tool requires several Python packages as well as the `pdftotext` binary somewhere on the `PATH`. +[ +The stable release is published on [PyPi](https://pypi.org/project/sec-certs/) as well as on [DockerHub](https://hub.docker.com/repository/docker/seccerts/sec-certs), you can install it with: -![GitHub Workflow Status](https://img.shields.io/github/workflow/status/crocs-muni/sec-certs/tests?style=flat-square) +``` +pip install -U sec-certs +``` +or -## Installation (CC) +``` +docker pull seccerts/sec-certs +``` -The tool requires several Python packages as well as the `pdftotext` binary somewhere on the `PATH`. -The easiest way to setup the tool is to install it in a virtual environment, e.g.: +Alternatively, you can setup the tool for development in a virtual environment, e.g.: Install Python virtual environment (if not yet): ``` python3 -m pip install --upgrade pip -- cgit v1.3.1 From fecb5ca29dfaaf65ef7e8f9fa199d83a22b504cf Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Mon, 26 Apr 2021 16:32:08 +0200 Subject: Update README.md Edit the Readme to add the instructions about how to use the image in order to get the generated files.--- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 340f1329..8935ec42 100644 --- a/README.md +++ b/README.md @@ -70,3 +70,8 @@ The analysis can be extended in several ways: 1. Additional keywords can be extracted from PDF files (modify `cert_rules.py`) 2. Data from `certificate_data_complete.json` can be analyzed in a novel way - this is why this project was concieved at the first place. 3. Help to fix problems in data extraction - some PDF files are corrupted, there are many typos even in certificate IDs... + +## How to run the application with a Docker container + 1. pull the image from the DockerHub repository : `docker pull seccerts/sec-certs` + 2. run `sudo docker run sec-certs:latest --volume ~/processed_data:/opt/sec-certs/examples/debug_dataset` + 3. All processed data will be in the `~/processed_data` directory -- cgit v1.3.1 From b94ea44b4015613abc6699c4bba18dee678ad611 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Mon, 26 Apr 2021 18:45:27 +0200 Subject: Update README.md Fixes according to the review--- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8935ec42..3dbfacc5 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,6 @@ The analysis can be extended in several ways: 3. Help to fix problems in data extraction - some PDF files are corrupted, there are many typos even in certificate IDs... ## How to run the application with a Docker container - 1. pull the image from the DockerHub repository : `docker pull seccerts/sec-certs` - 2. run `sudo docker run sec-certs:latest --volume ~/processed_data:/opt/sec-certs/examples/debug_dataset` + 1. pull the image from the DockerHub repository : `sudo docker pull seccerts/sec-certs` + 2. run `sudo docker run --volume ./processed_data:/opt/sec-certs/examples/debug_dataset -it seccerts/sec-certs` 3. All processed data will be in the `~/processed_data` directory -- cgit v1.3.1 From 6875a586f357c85738b3db5feefa6d886842d91b Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 26 Apr 2021 19:26:31 +0200 Subject: remove sudo from docker in readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3dbfacc5..320dc65c 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ The analysis can be extended in several ways: 3. Help to fix problems in data extraction - some PDF files are corrupted, there are many typos even in certificate IDs... ## How to run the application with a Docker container - 1. pull the image from the DockerHub repository : `sudo docker pull seccerts/sec-certs` - 2. run `sudo docker run --volume ./processed_data:/opt/sec-certs/examples/debug_dataset -it seccerts/sec-certs` + + 1. pull the image from the DockerHub repository : `docker pull seccerts/sec-certs` + 2. run `docker run --volume ./processed_data:/opt/sec-certs/examples/debug_dataset -it seccerts/sec-certs` 3. All processed data will be in the `~/processed_data` directory -- cgit v1.3.1 From bc9da78591854ec51d91f02a0b9e2ad11bf23cd5 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Tue, 27 Apr 2021 10:08:29 +0200 Subject: moving the process_script in a directory in order to have it inside the PyPI package --- sec_certs/entrypoints/process_certificates.py | 333 ++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 sec_certs/entrypoints/process_certificates.py diff --git a/sec_certs/entrypoints/process_certificates.py b/sec_certs/entrypoints/process_certificates.py new file mode 100644 index 00000000..5272fc67 --- /dev/null +++ b/sec_certs/entrypoints/process_certificates.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 + +import click +from sec_certs.files import load_json_files +from sec_certs.extract_certificates import * +from sec_certs.analyze_certificates import * +from sec_certs.download import download_cc_web, download_cc +from sec_certs.cert_rules import rules as cc_search_rules + + +@click.command() +@click.argument("directory", required=True, type=str) +@click.option("--fresh", "do_complete_extraction", is_flag=True, help="Whether to extract from a fresh state.") +@click.option("--do-download-meta", "do_download_meta", is_flag=True, help="Whether to download meta pages.") +@click.option("--do-extraction-meta", "do_extraction_meta", is_flag=True, help="Whether to extract information from the meta pages.") +@click.option("--do-download-certs", "do_download_certs", is_flag=True, help="Whether to download certs.") +@click.option("--do-pdftotext", "do_pdftotext", is_flag=True, help="Whether to perform pdftotext conversion of the certs.") +@click.option("--do-extraction", "do_extraction_certs", is_flag=True, help="Whether to extract information from the certs.") +@click.option("--do-pairing", "do_pairing", is_flag=True, help="Whether to pair PP stuff.") +@click.option("--do-processing", "do_processing", is_flag=True, help="Whether to process certificates.") +@click.option("--do-analysis", "do_analysis", is_flag=True, help="Whether to analyse certificates.") +@click.option("--do-analysis-fips", "do_analysis_fips", is_flag=True, help="Whether to analyse fips certificates.") +@click.option("--do-find-affected", "do_find_affected", help="Find affected certs.", multiple=True, type=str, metavar="certificate id") +@click.option("--do-find-affecting", "do_find_affecting", help="Find certificates affecting the provided one", multiple=True, type=str, metavar="certificate id") +@click.option("--do-find-affected-keyword", "do_find_affected_keywords", help="Find certs referencing all certs with specific keyword.", multiple=True, type=str, metavar="keyword") +@click.option("--analysis-label", "analysis_label", help="Optional custom label for analysis results", multiple=False, type=str, metavar="cutsom label") +@click.option("-t", "--threads", "threads", type=int, default=4, help="Amount of threads to use.") +def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_extraction_meta: bool, + do_download_certs: bool, do_pdftotext: bool, do_extraction_certs: bool, + do_pairing: bool, do_processing: bool, do_analysis: bool, do_analysis_fips: bool, do_find_affected: list, + do_find_affected_keywords: list, do_find_affecting: list, analysis_label: str, threads: int): + + directory = Path(directory) + web_dir = directory / "web" + walk_dir = directory / "certs" + certs_dir = walk_dir / "certs" + targets_dir = walk_dir / "targets" + pp_dir = directory / "pp" + fragments_dir = directory / "cert_fragments" + pp_fragments_dir = directory / "pp_fragments" + results_dir = directory / "results" + + web_dir.mkdir(parents=True, exist_ok=True) + walk_dir.mkdir(parents=True, exist_ok=True) + certs_dir.mkdir(parents=True, exist_ok=True) + targets_dir.mkdir(parents=True, exist_ok=True) + pp_dir.mkdir(parents=True, exist_ok=True) + fragments_dir.mkdir(parents=True, exist_ok=True) + pp_fragments_dir.mkdir(parents=True, exist_ok=True) + results_dir.mkdir(parents=True, exist_ok=True) + + # + # Start processing + # + do_analysis_filtered = True + + if do_complete_extraction: + # analyze all files from scratch, set 'previous' state to empty dict + prev_csv = {} + prev_html = {} + prev_download = [] + prev_front = {} + prev_keywords = {} + prev_pdf_meta = {} + else: + # load previously analyzed results + prev_csv, prev_html, prev_download, prev_front, prev_keywords, prev_pdf_meta = load_json_files( + map(lambda x: results_dir / x, ['certificate_data_csv_all.json', + 'certificate_data_html_all.json', + 'certificate_data_download_all.json', + 'certificate_data_frontpage_all.json', + 'certificate_data_keywords_all.json', + 'certificate_data_pdfmeta_all.json'])) + + if do_download_meta: + download_cc_web(web_dir, threads) + + # NOTE: Code below is preparation for differetian download of only new certificates + # - unfinished now + # print('*** Items: {} vs. {}'.format(len(current_html.keys()), len(prev_html.keys()))) + # current_html_keys = sorted(current_html.keys()) + # prev_html_keys = sorted(prev_html.keys()) + # new_items = list(set(current_html.keys()) - set(prev_html.keys())) + # print('*** New items detected: {}'.format(len(new_items))) + # + # # find new items which are not yet processed based on the value of raw csv line + # new_items = [] + # for current_item_key in current_csv.keys(): + # current_item = current_csv[current_item_key] + # current_raw_csv = current_item['csv_scan']['raw_csv_line'] + # match_found = False + # for prev_item_key in prev_csv.keys(): + # prev_item = prev_csv[prev_item_key] + # prev_raw_csv = prev_item['csv_scan']['raw_csv_line'] + # if current_raw_csv == prev_raw_csv: + # match_found = True + # break + # if not match_found: + # # we found new item + # new_items.append(current_item_key) + # + # print('*** New items detected: {}'.format(len(new_items))) + + if do_extraction_meta: + all_csv = extract_certificates_csv(web_dir) + all_html, certs, updates = extract_certificates_html(web_dir) + + with open(results_dir / "certificate_data_csv_all.json", "w") as write_file: + json.dump(all_csv, write_file, indent=4, sort_keys=True) + with open(results_dir / "certificate_data_html_all.json", "w") as write_file: + json.dump(all_html, write_file, indent=4, sort_keys=True) + with open(results_dir / "certificate_data_download_all.json", "w") as write_file: + json.dump(certs + updates, write_file, indent=4, sort_keys=True) + + if do_download_certs: + all_download = load_json_files([results_dir / "certificate_data_download_all.json"]) + download_cc(walk_dir, all_download[0], threads) + + if do_pdftotext: + convert_pdf_files(walk_dir, threads, ["-raw"]) + + if do_extraction_certs: + all_keywords = extract_certificates_keywords_parallel(walk_dir, fragments_dir, 'certificate', cc_search_rules, threads) + with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file: + json.dump(all_keywords, write_file, indent=4, sort_keys=True) + + all_front = extract_certificates_frontpage(walk_dir) + with open(results_dir / "certificate_data_frontpage_all.json", "w") as write_file: + json.dump(all_front, write_file, indent=4, sort_keys=True) + + all_pdf_meta = extract_certificates_pdfmeta_parallel(walk_dir, 'certificate', threads) + with open(results_dir / "certificate_data_pdfmeta_all.json", "w") as write_file: + json.dump(all_pdf_meta, write_file, indent=4, sort_keys=True) + + + # if do_extraction_pp: + # all_pp_csv = extract_protectionprofiles_csv(web_dir) + # all_pp_front = extract_protectionprofiles_frontpage(pp_dir) + # all_pp_keywords = extract_certificates_keywords(pp_dir, pp_fragments_dir, 'pp') + # all_pp_pdf_meta = extract_certificates_pdfmeta(pp_dir, 'pp') + # + # # save joined results + # with open("pp_data_csv_all.json", "w") as write_file: + # write_file.write(json.dumps(all_pp_csv, indent=4, sort_keys=True)) + # with open("pp_data_frontpage_all.json", "w") as write_file: + # write_file.write(json.dumps(all_pp_front, indent=4, sort_keys=True)) + # with open("pp_data_keywords_all.json", "w") as write_file: + # write_file.write(json.dumps(all_pp_keywords, indent=4, sort_keys=True)) + # with open("pp_data_pdfmeta_all.json", "w") as write_file: + # write_file.write(json.dumps(all_pp_pdf_meta, indent=4, sort_keys=True)) + + if do_pairing: + # # PROTECTION PROFILES + # # load results from previous step + # all_pp_csv, all_pp_front, all_pp_keywords, all_pp_pdf_meta = load_json_files( + # ['pp_data_csv_all.json', 'pp_data_frontpage_all.json', + # 'pp_data_keywords_all.json', 'pp_data_pdfmeta_all.json']) + # # check for unexpected results + # check_expected_pp_results({}, all_pp_csv, {}, all_pp_keywords) + # # collate all results into single file + # all_pp_items = collate_certificates_data({}, all_pp_csv, all_pp_front, all_pp_keywords, all_pp_pdf_meta, 'link_pp_document') + # # write collated result + # with open("pp_data_complete.json", "w") as write_file: + # write_file.write(json.dumps(all_pp_items, indent=4, sort_keys=True)) + + # CERTIFICATES + # load results from previous step + all_csv, all_html, all_front, all_keywords, all_pdf_meta = load_json_files( + map(lambda x: results_dir / x, ['certificate_data_csv_all.json', + 'certificate_data_html_all.json', + 'certificate_data_frontpage_all.json', + 'certificate_data_keywords_all.json', + 'certificate_data_pdfmeta_all.json'])) + # check for unexpected results + check_expected_cert_results(all_html, all_csv, all_front, all_keywords, all_pdf_meta) + # collate all results into single file + all_cert_items = collate_certificates_data(all_html, all_csv, all_front, all_keywords, all_pdf_meta, 'link_security_target') + + # write collated result + with open(results_dir / "certificate_data_complete.json", "w") as write_file: + json.dump(all_cert_items, write_file, indent=4, sort_keys=True) + + if do_processing: + # load information about protection profiles as extracted by sec-certs-pp tool + all_pp_items = {} + with open(results_dir / 'pp_data_complete_processed.json') as json_file: + all_pp_items = json.load(json_file) + + with open(results_dir / 'certificate_data_complete.json') as json_file: + all_cert_items = json.load(json_file) + + all_cert_items = process_certificates_data(all_cert_items, all_pp_items) + + with open(results_dir / "certificate_data_complete_processed.json", "w") as write_file: + json.dump(all_cert_items, write_file, indent=4, sort_keys=True) + + if do_analysis: + with open(results_dir / 'certificate_data_complete_processed.json') as json_file: + all_cert_items = json.load(json_file) + + if do_analysis_filtered: + # plot only selected analysis up to date 2020 + do_analysis_force_end_date(all_cert_items, results_dir, 2020) + + # analyze only smartcards + do_analysis_only_filtered(all_cert_items, results_dir, + ['csv_scan', 'cc_category'], 'ICs, Smart Cards and Smart Card-Related Devices and Systems') + # analyze only operating systems + do_analysis_only_filtered(all_cert_items, results_dir, + ['csv_scan', 'cc_category'], 'Operating Systems') + + # analyze separate manufacturers + do_analysis_manufacturers(all_cert_items, results_dir) + + # archived on 09/01/2019 + do_analysis_09_01_2019_archival(all_cert_items, results_dir) + + # analyze all certificates together + do_analysis_everything(all_cert_items, results_dir) + + with open(results_dir / "certificate_data_complete_processed_analyzed.json", "w") as write_file: + json.dump(all_cert_items, write_file, indent=4, sort_keys=True) + + # example: --do-find-affected-keyword v1\.02\.013 --analysis-label roca # (roca library) + # example: --do-find-affected-keyword AT90SC --do-find-affected-keyword 00\.03\.11\.05 --analysis-label minerva # (minerva library and chip) + # note: keyword search is as by regexes, so mind . etc. + if len(do_find_affected_keywords) > 0: + results_dir = results_dir \ + + search_rules = {'keyword': do_find_affected_keywords} + all_keywords = extract_certificates_keywords_parallel(walk_dir, None, 'certificate', search_rules, threads) + + # extract file names with keyword(s) match, extract cert id(s), fill do_find_affected list for further analysis + with open(results_dir / 'certificate_data_complete_processed.json') as json_file: + all_cert_items = json.load(json_file) + + # match search results to cert ids + certs_with_keywords = process_matched_keywords(all_cert_items, all_keywords, + list(do_find_affected_keywords), results_dir) + + # save list of found cert ids to separate json + name_results = get_name_for_keyword_search_results(do_find_affected_keywords) + file_name_results = analysis_label + '_' + name_results + '.json' + with open(results_dir / file_name_results, "w") as write_file: + json.dump(certs_with_keywords, write_file, indent=4, sort_keys=True) + + # populate list with certs is to analyse (same as would be --do-find-affected with explicitly specified ids) + for i in certs_with_keywords['certs'].keys(): + do_find_affected = do_find_affected + (i,) + + # set output folder according to analysis label + out_folder = analysis_label + '_' + name_results + results_out_dir = results_dir / out_folder + + do_analysis_affected(all_cert_items, results_out_dir, list(do_find_affected), analysis_label) + + # analysis of all certs referencing (directly/indirectly) the specified cert id(s) + # example: --do-find-affected BSI-DSZ-CC-0782-2012 + # example: --do-find-affected BSI-DSZ-CC-0833-2013 --do-find-affected BSI-DSZ-CC-0921-2014 --analysis-label roca_ATeHealth_Atos # (from eIDAS ID163484) + # example: --do-find-affected BSI-DSZ-CC-0758-2012 --do-find-affected BSI-DSZ-CC-0782-2012 --analysis-label roca_ATeHealth_Inf + if len(do_find_affected) > 0: + with open(results_dir / 'certificate_data_complete_processed_analyzed.json') as json_file: + all_cert_items = json.load(json_file) + # set output folder according to analysis label + results_out_dir = results_dir / analysis_label + do_analysis_affected(all_cert_items, results_out_dir, list(do_find_affected), analysis_label) + + # find all certificates which are potentially affecting security of the provided one () + # example: --do-find-affecting ANSSI-CC-2013/55 # Estonia estID + # example: --do-find-affecting ANSSI-CC-2020/44 # eTravel v2.2 EAC/BAC on MultiApp v4.0.1 platform with Filter Set 1.0 version 1.0 + if len(do_find_affecting) > 0: + with open(results_dir / 'certificate_data_complete_processed_analyzed.json') as json_file: + all_cert_items = json.load(json_file) + # set output folder according to analysis label + results_out_dir = results_dir / analysis_label + do_analysis_affecting(all_cert_items, results_out_dir, list(do_find_affecting), analysis_label) + + # analysis of fips extracted data + if do_analysis_fips: + with open(results_dir / 'fips_full_dataset.json') as json_file: + all_cert_items = json.load(json_file) + + # idea: transform into cc-like json then use same analysis functions + do_analysis_fips_certs(all_cert_items, results_dir) + + +if __name__ == "__main__": + main() + + + # TODO + # add saving of logs into file + # include parsing from protection profiles repo + # add differential partial download of new files only + processing + combine + # generate download script only for new files (need to have previous version of files stored) + # option for extraction of info just for single file? + # allow for late extraction of keywords (only newly added regexes) + # extraction of keywords done with the provided cert_rules_dict => cert_rules.py and cert_rules_new.py + # detect archival of certificates + # add tests - few selected files + # add detection of overly long regex matches + # add analysis of target CC version + # extract even more pdf file metadata https://github.com/pdfminer/pdfminer.six + # protection profiles dependency graph similarly as certid dependency graph is done + # If None == protection profile => Match PP with its assurance level and recompute + # extract info about protection profiles, download and parse pdf, map to referencing files + # analysis of PP only: which PP is the most popular?, what schemes/countries are doing most... + # analysis of certificates in time (per year) (different schemes) + # how many certificates are extended? How many times + # analysis of use of protection profiles + # analysis of security targets documents + # analysis of big cert clusters + # improve logging (info, warnings, errors, final summary) + # save as json, named segments (start_segment('name'), end_segment('name'), print('log_line', level) + # other schemes: FIPS140-2 certs, EMVCo, Visa Certification, American Express Certification, MasterCard Certification + # download and analyse CC documentation + # solve treatment of unicode characters + # analyze bibliography + # Statistics about number of characters (length), words, pages - histogram of pdf length & extracted text length + # add keywords extraction for trademarks (e.g, from 0963V2b_pdf.pdf) + # FRONTPAGE + # extract frontpage also from other than anssi and bsi certificates (US, BE...) + # add extraction of frontpage for protection profiles + # PORTABILITY + # check functionality on Linux (script %%20 expansions..., \\ vs. /) + # Add processing of docx files + # search for ATR, Response APDU, and custom commands specifying the IC type (CPLC + others) + # e.g., KECS-CR-15-105 XSmart e-Passport V1.4 EAC with SAC on M7892(eng).txt + # use pdf2text -raw switch to preserve better tables (needs to be checked wrt existing regexes) + # add tool for language detection, and if required, use automatic translation into english (https://pypi.org/project/googletrans/) + # analyze technical decisions: https://www.niap-ccevs.org/Documents_and_Guidance/view_tds.cfm + # extract names of IC (or other devices) from certificates => results in the list of certified chips in smartcards etc. + # analyze SARs and SFRs in correlation with specific company (what level of SAR/SFR can company achieve?) -- cgit v1.3.1 From 3d6995576ce49ec00f318c20d61b29d3f7a3604b Mon Sep 17 00:00:00 2001 From: KeleranV Date: Thu, 29 Apr 2021 01:03:25 -0700 Subject: moving the scripts in a folder and rewriting the setup.py to find them once they're moved --- fips_certificates.py | 212 ------------------ process_certificates.py | 333 ----------------------------- sec_certs/entrypoints/fips_certificates.py | 212 ++++++++++++++++++ setup.py | 4 +- 4 files changed, 214 insertions(+), 547 deletions(-) delete mode 100755 fips_certificates.py delete mode 100755 process_certificates.py create mode 100755 sec_certs/entrypoints/fips_certificates.py diff --git a/fips_certificates.py b/fips_certificates.py deleted file mode 100755 index eb147d48..00000000 --- a/fips_certificates.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import re -import time -from pathlib import Path -from typing import Set, Optional, List, Dict -from bs4 import BeautifulSoup - -from graphviz import Digraph -import click -import pikepdf -from tabula import read_pdf - -from sec_certs.download import download_fips_web, download_fips -from sec_certs import extract_certificates -from sec_certs.files import load_json_files, FILE_ERRORS_STRATEGY, search_files - -FIPS_BASE_URL = 'https://csrc.nist.gov' -FIPS_MODULE_URL = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/' - - -def extract_filename(file: str) -> str: - """ - Extracts filename from path - @param file: UN*X path - :return: filename without last extension - """ - return os.path.splitext(os.path.basename(file))[0] - - -def initialize_entry(current_items_found): - pass - - -def remove_algorithms_from_extracted_data(items, html): - pass - - -def validate_results(items: Dict, html: Dict): - """ - Function that validates results and finds the final connection output - :param items: All keyword items found in pdf files - :param html: All items extracted from html files - this is where we store connections - """ - broken_files = set() - for file_name in items: - for rule in items[file_name]['rules_cert_id']: - for cert in items[file_name]['rules_cert_id'][rule]: - cert_id = ''.join(filter(str.isdigit, cert)) - - if cert_id == '' or cert_id not in html: - # TEST - # if cert_id == '' or int(cert_id) > 3730: - broken_files.add(file_name) - items[file_name]['file_status'] = False - html[file_name]['file_status'] = False - break - if broken_files: - print("WARNING: CERTIFICATE FILES WITH WRONG CERTIFICATES PARSED") - print(*sorted(list(broken_files)), sep='\n') - print("... skipping these...") - print("Total non-analyzable files:", len(broken_files)) - - for file_name in items: - html[file_name]['Connections'] = [] - if not items[file_name]['file_status']: - continue - if items[file_name]['rules_cert_id'] == {}: - continue - for rule in items[file_name]['rules_cert_id']: - for cert in items[file_name]['rules_cert_id'][rule]: - cert_id = ''.join(filter(str.isdigit, cert)) - if cert_id not in html[file_name]['Connections']: - html[file_name]['Connections'].append(cert_id) - - -def parse_list_of_tables(txt: str) -> Set[str]: - pass - - -def extract_page_number(txt: str) -> Optional[str]: - """ - Parses chunks of text that are supposed to be mentioning table and having a footer - :param txt: input chunk - :return: page number - """ - # Page # of # - m = re.findall(r"(?P(?:[Pp]age) (?P\d+)(?: of \d+))", txt) - if m: - return m[-1][-1] - # Page # - m = re.findall(r"(?P(?:[Pp]age) (?P\d+)(?: of \d+)?)", txt) - if m: - return m[-1][-1] - # # of # - m = re.findall(r"(?P(?:[Pp]age)? ?(?P\d+)(?: of \d+))", txt) - if m: - return m[-1][-1] - # number alone - m = re.findall(r"(?P(?:[Pp]age)? ?(?P\d+)(?: of \d+)?)", txt) - return m[-1][-1] if m else None - - -def find_tables_iterative(file_text: str) -> List[int]: - pass - - -def find_footers(txt: str, num_pages: int) -> Optional[List]: - footer_regex = re.compile( - r"(?:Table[^\f]*)(?P^[\S\t ]*$)\n(?P(\f[ \t\S]+)$)(?P\n^[ \t\S]+?$)?", - re.MULTILINE) - - # We have 2 groups, one is optional - trying to parse 2 lines (just in case) - footer1 = [m.group('first') for m in footer_regex.finditer(txt)] - footer2 = [m.group('second') for m in footer_regex.finditer(txt)] - footer3 = [m.group('third') for m in footer_regex.finditer(txt)] - - # if len(footer2) < len(footer1): - # footer2 += [''] * (len(footer1) - len(footer2)) - - # zipping them together - footer_complete = [m[0] + m[1] + m[2] for m in zip(footer1, footer2, footer3) if - m[0] is not None and m[1] is not None and m[2] is not None] - - # removing None and duplicates - footers = [extract_page_number(x) for x in footer_complete] - footers = list(dict.fromkeys([x for x in footers if x is not None and 0 < int(x) < num_pages])) - print(footers) - if footers: - return footers - - -def find_tables(txt: str, file_name: Path) -> Optional[List]: - pass - - -def parse_algorithms(a, b=False): - pass - - -def extract_certs_from_tables(list_of_files: List, html_items: Dict) -> List[Path]: - pass - - -@click.command() -@click.argument("directory", required=True, type=str) -@click.option("--do-download-meta", "do_download_meta", is_flag=True) -@click.option("--do-download-certs", "do_download_certs", is_flag=True) -@click.option("-t", "--threads", "threads", type=int, default=4) -def main(directory, do_download_meta: bool, do_download_certs: bool, threads: int): - start = time.time() - directory = Path(directory) - web_dir = directory / "web" - fragments_dir = directory / "fragments" - results_dir = directory / "results" - policies_dir = directory / "security_policies" - - directory.mkdir(parents=True, exist_ok=True) - web_dir.mkdir(parents=True, exist_ok=True) - fragments_dir.mkdir(parents=True, exist_ok=True) - results_dir.mkdir(parents=True, exist_ok=True) - policies_dir.mkdir(parents=True, exist_ok=True) - - if do_download_meta: - download_fips_web(web_dir) - - if do_download_certs: - download_fips(web_dir, policies_dir, threads) - - print(f"Missing security policies: Total {len([])}") - print(f"Not available security policies: Total {len([])}") - files_to_load = [ - results_dir / 'fips_data_keywords_all.json', - results_dir / 'fips_html_all.json' - ] - - for file in files_to_load: - if not os.path.isfile(file): - items = extract_certificates.extract_certificates_keywords( - policies_dir, - fragments_dir, 'fips', fips_items=None, - should_censure_right_away=True) - with open(results_dir / 'fips_data_keywords_all.json', 'w') as f: - json.dump(items, f, indent=4, sort_keys=True) - break - - print("EXTRACTION DONE") - items, html = load_json_files(files_to_load) - - print("FINDING TABLES") - not_decoded = extract_certs_from_tables(search_files(policies_dir), html) - - print("NOT DECODED:", not_decoded) - with open(results_dir / 'broken_files.json', 'w') as f: - json.dump(not_decoded, f) - - print("REMOVING ALGORITHMS") - remove_algorithms_from_extracted_data(items, html) - - print("VALIDATING RESULTS") - validate_results(items, html) - with open(results_dir / 'fips_html_all.json', 'w') as f: - json.dump(html, f, indent=4, sort_keys=True) - print("PLOTTING GRAPH") - # get_dot_graph(html, results_dir / 'output') - end = time.time() - print("TIME:", end - start) - - -if __name__ == '__main__': - main() diff --git a/process_certificates.py b/process_certificates.py deleted file mode 100755 index 5272fc67..00000000 --- a/process_certificates.py +++ /dev/null @@ -1,333 +0,0 @@ -#!/usr/bin/env python3 - -import click -from sec_certs.files import load_json_files -from sec_certs.extract_certificates import * -from sec_certs.analyze_certificates import * -from sec_certs.download import download_cc_web, download_cc -from sec_certs.cert_rules import rules as cc_search_rules - - -@click.command() -@click.argument("directory", required=True, type=str) -@click.option("--fresh", "do_complete_extraction", is_flag=True, help="Whether to extract from a fresh state.") -@click.option("--do-download-meta", "do_download_meta", is_flag=True, help="Whether to download meta pages.") -@click.option("--do-extraction-meta", "do_extraction_meta", is_flag=True, help="Whether to extract information from the meta pages.") -@click.option("--do-download-certs", "do_download_certs", is_flag=True, help="Whether to download certs.") -@click.option("--do-pdftotext", "do_pdftotext", is_flag=True, help="Whether to perform pdftotext conversion of the certs.") -@click.option("--do-extraction", "do_extraction_certs", is_flag=True, help="Whether to extract information from the certs.") -@click.option("--do-pairing", "do_pairing", is_flag=True, help="Whether to pair PP stuff.") -@click.option("--do-processing", "do_processing", is_flag=True, help="Whether to process certificates.") -@click.option("--do-analysis", "do_analysis", is_flag=True, help="Whether to analyse certificates.") -@click.option("--do-analysis-fips", "do_analysis_fips", is_flag=True, help="Whether to analyse fips certificates.") -@click.option("--do-find-affected", "do_find_affected", help="Find affected certs.", multiple=True, type=str, metavar="certificate id") -@click.option("--do-find-affecting", "do_find_affecting", help="Find certificates affecting the provided one", multiple=True, type=str, metavar="certificate id") -@click.option("--do-find-affected-keyword", "do_find_affected_keywords", help="Find certs referencing all certs with specific keyword.", multiple=True, type=str, metavar="keyword") -@click.option("--analysis-label", "analysis_label", help="Optional custom label for analysis results", multiple=False, type=str, metavar="cutsom label") -@click.option("-t", "--threads", "threads", type=int, default=4, help="Amount of threads to use.") -def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_extraction_meta: bool, - do_download_certs: bool, do_pdftotext: bool, do_extraction_certs: bool, - do_pairing: bool, do_processing: bool, do_analysis: bool, do_analysis_fips: bool, do_find_affected: list, - do_find_affected_keywords: list, do_find_affecting: list, analysis_label: str, threads: int): - - directory = Path(directory) - web_dir = directory / "web" - walk_dir = directory / "certs" - certs_dir = walk_dir / "certs" - targets_dir = walk_dir / "targets" - pp_dir = directory / "pp" - fragments_dir = directory / "cert_fragments" - pp_fragments_dir = directory / "pp_fragments" - results_dir = directory / "results" - - web_dir.mkdir(parents=True, exist_ok=True) - walk_dir.mkdir(parents=True, exist_ok=True) - certs_dir.mkdir(parents=True, exist_ok=True) - targets_dir.mkdir(parents=True, exist_ok=True) - pp_dir.mkdir(parents=True, exist_ok=True) - fragments_dir.mkdir(parents=True, exist_ok=True) - pp_fragments_dir.mkdir(parents=True, exist_ok=True) - results_dir.mkdir(parents=True, exist_ok=True) - - # - # Start processing - # - do_analysis_filtered = True - - if do_complete_extraction: - # analyze all files from scratch, set 'previous' state to empty dict - prev_csv = {} - prev_html = {} - prev_download = [] - prev_front = {} - prev_keywords = {} - prev_pdf_meta = {} - else: - # load previously analyzed results - prev_csv, prev_html, prev_download, prev_front, prev_keywords, prev_pdf_meta = load_json_files( - map(lambda x: results_dir / x, ['certificate_data_csv_all.json', - 'certificate_data_html_all.json', - 'certificate_data_download_all.json', - 'certificate_data_frontpage_all.json', - 'certificate_data_keywords_all.json', - 'certificate_data_pdfmeta_all.json'])) - - if do_download_meta: - download_cc_web(web_dir, threads) - - # NOTE: Code below is preparation for differetian download of only new certificates - # - unfinished now - # print('*** Items: {} vs. {}'.format(len(current_html.keys()), len(prev_html.keys()))) - # current_html_keys = sorted(current_html.keys()) - # prev_html_keys = sorted(prev_html.keys()) - # new_items = list(set(current_html.keys()) - set(prev_html.keys())) - # print('*** New items detected: {}'.format(len(new_items))) - # - # # find new items which are not yet processed based on the value of raw csv line - # new_items = [] - # for current_item_key in current_csv.keys(): - # current_item = current_csv[current_item_key] - # current_raw_csv = current_item['csv_scan']['raw_csv_line'] - # match_found = False - # for prev_item_key in prev_csv.keys(): - # prev_item = prev_csv[prev_item_key] - # prev_raw_csv = prev_item['csv_scan']['raw_csv_line'] - # if current_raw_csv == prev_raw_csv: - # match_found = True - # break - # if not match_found: - # # we found new item - # new_items.append(current_item_key) - # - # print('*** New items detected: {}'.format(len(new_items))) - - if do_extraction_meta: - all_csv = extract_certificates_csv(web_dir) - all_html, certs, updates = extract_certificates_html(web_dir) - - with open(results_dir / "certificate_data_csv_all.json", "w") as write_file: - json.dump(all_csv, write_file, indent=4, sort_keys=True) - with open(results_dir / "certificate_data_html_all.json", "w") as write_file: - json.dump(all_html, write_file, indent=4, sort_keys=True) - with open(results_dir / "certificate_data_download_all.json", "w") as write_file: - json.dump(certs + updates, write_file, indent=4, sort_keys=True) - - if do_download_certs: - all_download = load_json_files([results_dir / "certificate_data_download_all.json"]) - download_cc(walk_dir, all_download[0], threads) - - if do_pdftotext: - convert_pdf_files(walk_dir, threads, ["-raw"]) - - if do_extraction_certs: - all_keywords = extract_certificates_keywords_parallel(walk_dir, fragments_dir, 'certificate', cc_search_rules, threads) - with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file: - json.dump(all_keywords, write_file, indent=4, sort_keys=True) - - all_front = extract_certificates_frontpage(walk_dir) - with open(results_dir / "certificate_data_frontpage_all.json", "w") as write_file: - json.dump(all_front, write_file, indent=4, sort_keys=True) - - all_pdf_meta = extract_certificates_pdfmeta_parallel(walk_dir, 'certificate', threads) - with open(results_dir / "certificate_data_pdfmeta_all.json", "w") as write_file: - json.dump(all_pdf_meta, write_file, indent=4, sort_keys=True) - - - # if do_extraction_pp: - # all_pp_csv = extract_protectionprofiles_csv(web_dir) - # all_pp_front = extract_protectionprofiles_frontpage(pp_dir) - # all_pp_keywords = extract_certificates_keywords(pp_dir, pp_fragments_dir, 'pp') - # all_pp_pdf_meta = extract_certificates_pdfmeta(pp_dir, 'pp') - # - # # save joined results - # with open("pp_data_csv_all.json", "w") as write_file: - # write_file.write(json.dumps(all_pp_csv, indent=4, sort_keys=True)) - # with open("pp_data_frontpage_all.json", "w") as write_file: - # write_file.write(json.dumps(all_pp_front, indent=4, sort_keys=True)) - # with open("pp_data_keywords_all.json", "w") as write_file: - # write_file.write(json.dumps(all_pp_keywords, indent=4, sort_keys=True)) - # with open("pp_data_pdfmeta_all.json", "w") as write_file: - # write_file.write(json.dumps(all_pp_pdf_meta, indent=4, sort_keys=True)) - - if do_pairing: - # # PROTECTION PROFILES - # # load results from previous step - # all_pp_csv, all_pp_front, all_pp_keywords, all_pp_pdf_meta = load_json_files( - # ['pp_data_csv_all.json', 'pp_data_frontpage_all.json', - # 'pp_data_keywords_all.json', 'pp_data_pdfmeta_all.json']) - # # check for unexpected results - # check_expected_pp_results({}, all_pp_csv, {}, all_pp_keywords) - # # collate all results into single file - # all_pp_items = collate_certificates_data({}, all_pp_csv, all_pp_front, all_pp_keywords, all_pp_pdf_meta, 'link_pp_document') - # # write collated result - # with open("pp_data_complete.json", "w") as write_file: - # write_file.write(json.dumps(all_pp_items, indent=4, sort_keys=True)) - - # CERTIFICATES - # load results from previous step - all_csv, all_html, all_front, all_keywords, all_pdf_meta = load_json_files( - map(lambda x: results_dir / x, ['certificate_data_csv_all.json', - 'certificate_data_html_all.json', - 'certificate_data_frontpage_all.json', - 'certificate_data_keywords_all.json', - 'certificate_data_pdfmeta_all.json'])) - # check for unexpected results - check_expected_cert_results(all_html, all_csv, all_front, all_keywords, all_pdf_meta) - # collate all results into single file - all_cert_items = collate_certificates_data(all_html, all_csv, all_front, all_keywords, all_pdf_meta, 'link_security_target') - - # write collated result - with open(results_dir / "certificate_data_complete.json", "w") as write_file: - json.dump(all_cert_items, write_file, indent=4, sort_keys=True) - - if do_processing: - # load information about protection profiles as extracted by sec-certs-pp tool - all_pp_items = {} - with open(results_dir / 'pp_data_complete_processed.json') as json_file: - all_pp_items = json.load(json_file) - - with open(results_dir / 'certificate_data_complete.json') as json_file: - all_cert_items = json.load(json_file) - - all_cert_items = process_certificates_data(all_cert_items, all_pp_items) - - with open(results_dir / "certificate_data_complete_processed.json", "w") as write_file: - json.dump(all_cert_items, write_file, indent=4, sort_keys=True) - - if do_analysis: - with open(results_dir / 'certificate_data_complete_processed.json') as json_file: - all_cert_items = json.load(json_file) - - if do_analysis_filtered: - # plot only selected analysis up to date 2020 - do_analysis_force_end_date(all_cert_items, results_dir, 2020) - - # analyze only smartcards - do_analysis_only_filtered(all_cert_items, results_dir, - ['csv_scan', 'cc_category'], 'ICs, Smart Cards and Smart Card-Related Devices and Systems') - # analyze only operating systems - do_analysis_only_filtered(all_cert_items, results_dir, - ['csv_scan', 'cc_category'], 'Operating Systems') - - # analyze separate manufacturers - do_analysis_manufacturers(all_cert_items, results_dir) - - # archived on 09/01/2019 - do_analysis_09_01_2019_archival(all_cert_items, results_dir) - - # analyze all certificates together - do_analysis_everything(all_cert_items, results_dir) - - with open(results_dir / "certificate_data_complete_processed_analyzed.json", "w") as write_file: - json.dump(all_cert_items, write_file, indent=4, sort_keys=True) - - # example: --do-find-affected-keyword v1\.02\.013 --analysis-label roca # (roca library) - # example: --do-find-affected-keyword AT90SC --do-find-affected-keyword 00\.03\.11\.05 --analysis-label minerva # (minerva library and chip) - # note: keyword search is as by regexes, so mind . etc. - if len(do_find_affected_keywords) > 0: - results_dir = results_dir \ - - search_rules = {'keyword': do_find_affected_keywords} - all_keywords = extract_certificates_keywords_parallel(walk_dir, None, 'certificate', search_rules, threads) - - # extract file names with keyword(s) match, extract cert id(s), fill do_find_affected list for further analysis - with open(results_dir / 'certificate_data_complete_processed.json') as json_file: - all_cert_items = json.load(json_file) - - # match search results to cert ids - certs_with_keywords = process_matched_keywords(all_cert_items, all_keywords, - list(do_find_affected_keywords), results_dir) - - # save list of found cert ids to separate json - name_results = get_name_for_keyword_search_results(do_find_affected_keywords) - file_name_results = analysis_label + '_' + name_results + '.json' - with open(results_dir / file_name_results, "w") as write_file: - json.dump(certs_with_keywords, write_file, indent=4, sort_keys=True) - - # populate list with certs is to analyse (same as would be --do-find-affected with explicitly specified ids) - for i in certs_with_keywords['certs'].keys(): - do_find_affected = do_find_affected + (i,) - - # set output folder according to analysis label - out_folder = analysis_label + '_' + name_results - results_out_dir = results_dir / out_folder - - do_analysis_affected(all_cert_items, results_out_dir, list(do_find_affected), analysis_label) - - # analysis of all certs referencing (directly/indirectly) the specified cert id(s) - # example: --do-find-affected BSI-DSZ-CC-0782-2012 - # example: --do-find-affected BSI-DSZ-CC-0833-2013 --do-find-affected BSI-DSZ-CC-0921-2014 --analysis-label roca_ATeHealth_Atos # (from eIDAS ID163484) - # example: --do-find-affected BSI-DSZ-CC-0758-2012 --do-find-affected BSI-DSZ-CC-0782-2012 --analysis-label roca_ATeHealth_Inf - if len(do_find_affected) > 0: - with open(results_dir / 'certificate_data_complete_processed_analyzed.json') as json_file: - all_cert_items = json.load(json_file) - # set output folder according to analysis label - results_out_dir = results_dir / analysis_label - do_analysis_affected(all_cert_items, results_out_dir, list(do_find_affected), analysis_label) - - # find all certificates which are potentially affecting security of the provided one () - # example: --do-find-affecting ANSSI-CC-2013/55 # Estonia estID - # example: --do-find-affecting ANSSI-CC-2020/44 # eTravel v2.2 EAC/BAC on MultiApp v4.0.1 platform with Filter Set 1.0 version 1.0 - if len(do_find_affecting) > 0: - with open(results_dir / 'certificate_data_complete_processed_analyzed.json') as json_file: - all_cert_items = json.load(json_file) - # set output folder according to analysis label - results_out_dir = results_dir / analysis_label - do_analysis_affecting(all_cert_items, results_out_dir, list(do_find_affecting), analysis_label) - - # analysis of fips extracted data - if do_analysis_fips: - with open(results_dir / 'fips_full_dataset.json') as json_file: - all_cert_items = json.load(json_file) - - # idea: transform into cc-like json then use same analysis functions - do_analysis_fips_certs(all_cert_items, results_dir) - - -if __name__ == "__main__": - main() - - - # TODO - # add saving of logs into file - # include parsing from protection profiles repo - # add differential partial download of new files only + processing + combine - # generate download script only for new files (need to have previous version of files stored) - # option for extraction of info just for single file? - # allow for late extraction of keywords (only newly added regexes) - # extraction of keywords done with the provided cert_rules_dict => cert_rules.py and cert_rules_new.py - # detect archival of certificates - # add tests - few selected files - # add detection of overly long regex matches - # add analysis of target CC version - # extract even more pdf file metadata https://github.com/pdfminer/pdfminer.six - # protection profiles dependency graph similarly as certid dependency graph is done - # If None == protection profile => Match PP with its assurance level and recompute - # extract info about protection profiles, download and parse pdf, map to referencing files - # analysis of PP only: which PP is the most popular?, what schemes/countries are doing most... - # analysis of certificates in time (per year) (different schemes) - # how many certificates are extended? How many times - # analysis of use of protection profiles - # analysis of security targets documents - # analysis of big cert clusters - # improve logging (info, warnings, errors, final summary) - # save as json, named segments (start_segment('name'), end_segment('name'), print('log_line', level) - # other schemes: FIPS140-2 certs, EMVCo, Visa Certification, American Express Certification, MasterCard Certification - # download and analyse CC documentation - # solve treatment of unicode characters - # analyze bibliography - # Statistics about number of characters (length), words, pages - histogram of pdf length & extracted text length - # add keywords extraction for trademarks (e.g, from 0963V2b_pdf.pdf) - # FRONTPAGE - # extract frontpage also from other than anssi and bsi certificates (US, BE...) - # add extraction of frontpage for protection profiles - # PORTABILITY - # check functionality on Linux (script %%20 expansions..., \\ vs. /) - # Add processing of docx files - # search for ATR, Response APDU, and custom commands specifying the IC type (CPLC + others) - # e.g., KECS-CR-15-105 XSmart e-Passport V1.4 EAC with SAC on M7892(eng).txt - # use pdf2text -raw switch to preserve better tables (needs to be checked wrt existing regexes) - # add tool for language detection, and if required, use automatic translation into english (https://pypi.org/project/googletrans/) - # analyze technical decisions: https://www.niap-ccevs.org/Documents_and_Guidance/view_tds.cfm - # extract names of IC (or other devices) from certificates => results in the list of certified chips in smartcards etc. - # analyze SARs and SFRs in correlation with specific company (what level of SAR/SFR can company achieve?) diff --git a/sec_certs/entrypoints/fips_certificates.py b/sec_certs/entrypoints/fips_certificates.py new file mode 100755 index 00000000..eb147d48 --- /dev/null +++ b/sec_certs/entrypoints/fips_certificates.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +import json +import os +import re +import time +from pathlib import Path +from typing import Set, Optional, List, Dict +from bs4 import BeautifulSoup + +from graphviz import Digraph +import click +import pikepdf +from tabula import read_pdf + +from sec_certs.download import download_fips_web, download_fips +from sec_certs import extract_certificates +from sec_certs.files import load_json_files, FILE_ERRORS_STRATEGY, search_files + +FIPS_BASE_URL = 'https://csrc.nist.gov' +FIPS_MODULE_URL = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/' + + +def extract_filename(file: str) -> str: + """ + Extracts filename from path + @param file: UN*X path + :return: filename without last extension + """ + return os.path.splitext(os.path.basename(file))[0] + + +def initialize_entry(current_items_found): + pass + + +def remove_algorithms_from_extracted_data(items, html): + pass + + +def validate_results(items: Dict, html: Dict): + """ + Function that validates results and finds the final connection output + :param items: All keyword items found in pdf files + :param html: All items extracted from html files - this is where we store connections + """ + broken_files = set() + for file_name in items: + for rule in items[file_name]['rules_cert_id']: + for cert in items[file_name]['rules_cert_id'][rule]: + cert_id = ''.join(filter(str.isdigit, cert)) + + if cert_id == '' or cert_id not in html: + # TEST + # if cert_id == '' or int(cert_id) > 3730: + broken_files.add(file_name) + items[file_name]['file_status'] = False + html[file_name]['file_status'] = False + break + if broken_files: + print("WARNING: CERTIFICATE FILES WITH WRONG CERTIFICATES PARSED") + print(*sorted(list(broken_files)), sep='\n') + print("... skipping these...") + print("Total non-analyzable files:", len(broken_files)) + + for file_name in items: + html[file_name]['Connections'] = [] + if not items[file_name]['file_status']: + continue + if items[file_name]['rules_cert_id'] == {}: + continue + for rule in items[file_name]['rules_cert_id']: + for cert in items[file_name]['rules_cert_id'][rule]: + cert_id = ''.join(filter(str.isdigit, cert)) + if cert_id not in html[file_name]['Connections']: + html[file_name]['Connections'].append(cert_id) + + +def parse_list_of_tables(txt: str) -> Set[str]: + pass + + +def extract_page_number(txt: str) -> Optional[str]: + """ + Parses chunks of text that are supposed to be mentioning table and having a footer + :param txt: input chunk + :return: page number + """ + # Page # of # + m = re.findall(r"(?P(?:[Pp]age) (?P\d+)(?: of \d+))", txt) + if m: + return m[-1][-1] + # Page # + m = re.findall(r"(?P(?:[Pp]age) (?P\d+)(?: of \d+)?)", txt) + if m: + return m[-1][-1] + # # of # + m = re.findall(r"(?P(?:[Pp]age)? ?(?P\d+)(?: of \d+))", txt) + if m: + return m[-1][-1] + # number alone + m = re.findall(r"(?P(?:[Pp]age)? ?(?P\d+)(?: of \d+)?)", txt) + return m[-1][-1] if m else None + + +def find_tables_iterative(file_text: str) -> List[int]: + pass + + +def find_footers(txt: str, num_pages: int) -> Optional[List]: + footer_regex = re.compile( + r"(?:Table[^\f]*)(?P^[\S\t ]*$)\n(?P(\f[ \t\S]+)$)(?P\n^[ \t\S]+?$)?", + re.MULTILINE) + + # We have 2 groups, one is optional - trying to parse 2 lines (just in case) + footer1 = [m.group('first') for m in footer_regex.finditer(txt)] + footer2 = [m.group('second') for m in footer_regex.finditer(txt)] + footer3 = [m.group('third') for m in footer_regex.finditer(txt)] + + # if len(footer2) < len(footer1): + # footer2 += [''] * (len(footer1) - len(footer2)) + + # zipping them together + footer_complete = [m[0] + m[1] + m[2] for m in zip(footer1, footer2, footer3) if + m[0] is not None and m[1] is not None and m[2] is not None] + + # removing None and duplicates + footers = [extract_page_number(x) for x in footer_complete] + footers = list(dict.fromkeys([x for x in footers if x is not None and 0 < int(x) < num_pages])) + print(footers) + if footers: + return footers + + +def find_tables(txt: str, file_name: Path) -> Optional[List]: + pass + + +def parse_algorithms(a, b=False): + pass + + +def extract_certs_from_tables(list_of_files: List, html_items: Dict) -> List[Path]: + pass + + +@click.command() +@click.argument("directory", required=True, type=str) +@click.option("--do-download-meta", "do_download_meta", is_flag=True) +@click.option("--do-download-certs", "do_download_certs", is_flag=True) +@click.option("-t", "--threads", "threads", type=int, default=4) +def main(directory, do_download_meta: bool, do_download_certs: bool, threads: int): + start = time.time() + directory = Path(directory) + web_dir = directory / "web" + fragments_dir = directory / "fragments" + results_dir = directory / "results" + policies_dir = directory / "security_policies" + + directory.mkdir(parents=True, exist_ok=True) + web_dir.mkdir(parents=True, exist_ok=True) + fragments_dir.mkdir(parents=True, exist_ok=True) + results_dir.mkdir(parents=True, exist_ok=True) + policies_dir.mkdir(parents=True, exist_ok=True) + + if do_download_meta: + download_fips_web(web_dir) + + if do_download_certs: + download_fips(web_dir, policies_dir, threads) + + print(f"Missing security policies: Total {len([])}") + print(f"Not available security policies: Total {len([])}") + files_to_load = [ + results_dir / 'fips_data_keywords_all.json', + results_dir / 'fips_html_all.json' + ] + + for file in files_to_load: + if not os.path.isfile(file): + items = extract_certificates.extract_certificates_keywords( + policies_dir, + fragments_dir, 'fips', fips_items=None, + should_censure_right_away=True) + with open(results_dir / 'fips_data_keywords_all.json', 'w') as f: + json.dump(items, f, indent=4, sort_keys=True) + break + + print("EXTRACTION DONE") + items, html = load_json_files(files_to_load) + + print("FINDING TABLES") + not_decoded = extract_certs_from_tables(search_files(policies_dir), html) + + print("NOT DECODED:", not_decoded) + with open(results_dir / 'broken_files.json', 'w') as f: + json.dump(not_decoded, f) + + print("REMOVING ALGORITHMS") + remove_algorithms_from_extracted_data(items, html) + + print("VALIDATING RESULTS") + validate_results(items, html) + with open(results_dir / 'fips_html_all.json', 'w') as f: + json.dump(html, f, indent=4, sort_keys=True) + print("PLOTTING GRAPH") + # get_dot_graph(html, results_dir / 'output') + end = time.time() + print("TIME:", end - start) + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 494dbd23..d3a80ce0 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( }, entry_points=""" [console_scripts] - process-certs=process_certificates:main - fips-certs=fips_certificates:main + process-certs=sec_certs.entrypoints.process_certificates:main + fips-certs=sec_certs.entrypoints.fips_certificates:main """ ) -- cgit v1.3.1 From 0605835a24e49840a7b475116ecd6f0390212695 Mon Sep 17 00:00:00 2001 From: Léo Vansimay Date: Thu, 29 Apr 2021 11:10:49 +0200 Subject: Update python-publish.yml The method currently used only works when we tag a version number, I'm trying to use testpypi to avoid a failed build on each try. Moreover the build and push to PyPI will only trigger on a push with a tag--- .github/workflows/python-publish.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 2fab292d..782ec7aa 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -24,10 +24,20 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine - - name: Build and publish + + - name: Build and publish to PyPI + if: startsWith(github.ref, 'refs/tags') env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} #Same as Docker, need to add an account in the secrets TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python setup.py sdist bdist_wheel twine upload dist/* + + - name: Build and publish to PyPI + env: + TWINE_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} #Same as Docker, need to add an account in the secrets + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload --repository testpypi dist/* -- cgit v1.3.1