From 4863f27fab81fa0301554a5df623feb8c72fd404 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 24 Apr 2022 20:59:47 +0200 Subject: sphinx quickstart --- docs/Makefile | 20 ++++++++++++++++++++ docs/conf.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 20 ++++++++++++++++++++ docs/make.bat | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat (limited to 'docs') diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..3617f9fc --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,55 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +from typing import List + +# -- Project information ----------------------------------------------------- + +project = "sec-certs" +copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" +author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" + +# The full version, including alpha/beta/rc tags +release = "0.0.4" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions: List[str] = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "alabaster" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..9e16d741 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +.. sec-certs documentation master file, created by + sphinx-quickstart on Sun Apr 24 20:56:46 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to sec-certs's documentation! +===================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd -- cgit v1.3.1 From 2ac699107479e29ca4b3f1848dfa8a2bae7e4191 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 25 Apr 2022 16:48:04 +0200 Subject: link external docs sources, introduce myst --- CONTRIBUTING.md | 18 +++++++++--------- LICENSE | 2 +- README.md | 4 +++- docs/code_of_conduct.md | 4 ++++ docs/conf.py | 4 ++-- docs/contributing.md | 4 ++++ docs/index.md | 10 ++++++++++ docs/index.rst | 20 -------------------- docs/license.md | 6 ++++++ docs/readme.md | 4 ++++ requirements/dev_requirements.in | 4 +++- requirements/dev_requirements.txt | 38 +++++++++++++++++++++++++++++++++++--- 12 files changed, 81 insertions(+), 37 deletions(-) create mode 100644 docs/code_of_conduct.md create mode 100644 docs/contributing.md create mode 100644 docs/index.md delete mode 100644 docs/index.rst create mode 100644 docs/license.md create mode 100644 docs/readme.md (limited to 'docs') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d539f656..afefc20b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -## Contributing +# Contributing You contribution is warmly welcomed. You can help by: @@ -20,9 +20,9 @@ Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/docker/Docker ### Requirements Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-tools). The main ideas are: -- List actual dependencies in `.in` files inside [requirements](requirements) folder without pinning them. -- Those dependencies are loaded into [setup.py](setup.py) file. -- Additionally, [compile.sh](requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. +- List actual dependencies in `.in` files inside [requirements](https://github.com/crocs-muni/sec-certs/blob/main/requirements) folder without pinning them. +- Those dependencies are loaded into [setup.py](https://github.com/crocs-muni/sec-certs/blob/main/setup.py) file. +- Additionally, [compile.sh](https://github.com/crocs-muni/sec-certs/blob/main/requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. - Tests, linting and Docker all run against this reproducible environment of pinned requirements. ## Branches and releases @@ -35,12 +35,12 @@ Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-too All commits shall pass the lint pipeline of the following tools: -- Mypy (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) -- Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) -- isort (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) -- Flake8 (see [.flake8](https://github.com/crocs-muni/sec-certs/blob/dev/.flake8) for settings) +- Mypy (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) +- Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) +- isort (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) +- Flake8 (see [.flake8](https://github.com/crocs-muni/sec-certs/blob/main/.flake8) for settings) -These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/dev/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you. +These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/main/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you. It should thus suffice to: diff --git a/LICENSE b/LICENSE index 5359145a..55e4ec93 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 Petr Svenda +Copyright (c) 2019-2022 Petr Svenda, Adam Janovsky, Jan Jancar, Jiri Michalik, Stanislav Bobon Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 29da0021..69a036d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# ![](docs/_static/logo.svg) +# Sec-certs + +![](docs/_static/logo.svg) Tool for analysis of security certificates and their security targets (Common Criteria, NIST FIPS140-2...). diff --git a/docs/code_of_conduct.md b/docs/code_of_conduct.md new file mode 100644 index 00000000..74e5474b --- /dev/null +++ b/docs/code_of_conduct.md @@ -0,0 +1,4 @@ +```{include} ../CODE_OF_CONDUCT.md +:relative-docs: docs/ +:relative-images: +``` diff --git a/docs/conf.py b/docs/conf.py index 3617f9fc..23392e9c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,7 @@ release = "0.0.4" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions: List[str] = [] +extensions: List[str] = ["myst_parser"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -47,7 +47,7 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "alabaster" +html_theme = "sphinx_book_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 00000000..fc1b2138 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,4 @@ +```{include} ../CONTRIBUTING.md +:relative-docs: docs/ +:relative-images: +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..4554a3b2 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,10 @@ +# Sec-certs documentation + +```{toctree} +:maxdepth: 2 +readme.md +contributing.md +code_of_conduct.md +license.md +``` + diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 9e16d741..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. sec-certs documentation master file, created by - sphinx-quickstart on Sun Apr 24 20:56:46 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to sec-certs's documentation! -===================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 00000000..4f6e8060 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,6 @@ +# License + +```{include} ../LICENSE +:relative-docs: docs/ +:relative-images: +``` diff --git a/docs/readme.md b/docs/readme.md new file mode 100644 index 00000000..2cb706b3 --- /dev/null +++ b/docs/readme.md @@ -0,0 +1,4 @@ +```{include} ../README.md +:relative-docs: docs/ +:relative-images: +``` diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in index 78f2c088..824a102f 100644 --- a/requirements/dev_requirements.in +++ b/requirements/dev_requirements.in @@ -11,4 +11,6 @@ isort flake8 pre-commit pip-tools -sphinx \ No newline at end of file +sphinx +myst-parser +sphinx-book-theme \ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 2b91f06b..97debe7f 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -4,6 +4,8 @@ attrs==21.4.0 # via pytest babel==2.10.1 # via sphinx +beautifulsoup4==4.11.1 + # via pydata-sphinx-theme black==22.3.0 # via -r dev_requirements.in certifi==2021.10.8 @@ -21,7 +23,10 @@ coverage[toml]==6.3.2 distlib==0.3.4 # via virtualenv docutils==0.17.1 - # via sphinx + # via + # myst-parser + # pydata-sphinx-theme + # sphinx filelock==3.6.0 # via virtualenv flake8==4.0.1 @@ -41,11 +46,21 @@ iniconfig==1.1.1 isort==5.10.1 # via -r dev_requirements.in jinja2==3.1.1 - # via sphinx + # via + # myst-parser + # sphinx +markdown-it-py==2.1.0 + # via + # mdit-py-plugins + # myst-parser markupsafe==2.1.1 # via jinja2 mccabe==0.6.1 # via flake8 +mdit-py-plugins==0.3.0 + # via myst-parser +mdurl==0.1.1 + # via markdown-it-py memory-profiler==0.60.0 # via pytest-monitor mypy==0.942 @@ -54,10 +69,13 @@ mypy-extensions==0.4.3 # via # black # mypy +myst-parser==0.17.2 + # via -r dev_requirements.in nodeenv==1.6.0 # via pre-commit packaging==21.3 # via + # pydata-sphinx-theme # pytest # sphinx pathspec==0.9.0 @@ -82,6 +100,8 @@ py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 +pydata-sphinx-theme==0.8.1 + # via sphinx-book-theme pyflakes==2.4.0 # via flake8 pygments==2.12.0 @@ -103,7 +123,10 @@ pytest-profiling==1.7.0 pytz==2022.1 # via babel pyyaml==6.0 - # via pre-commit + # via + # myst-parser + # pre-commit + # sphinx-book-theme requests==2.27.1 # via # pytest-monitor @@ -114,7 +137,15 @@ six==1.16.0 # virtualenv snowballstemmer==2.2.0 # via sphinx +soupsieve==2.3.2.post1 + # via beautifulsoup4 sphinx==4.5.0 + # via + # -r dev_requirements.in + # myst-parser + # pydata-sphinx-theme + # sphinx-book-theme +sphinx-book-theme==0.3.2 # via -r dev_requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx @@ -149,6 +180,7 @@ typing-extensions==4.1.1 # via # black # mypy + # myst-parser urllib3==1.26.9 # via requests virtualenv==20.14.0 -- cgit v1.3.1 From 5ede8f4dc9b3322b04480b3f9caa99c382fa1042 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 25 Apr 2022 18:11:07 +0200 Subject: add notebooks to docs --- docs/conf.py | 3 +- docs/index.md | 13 +- docs/notebooks | 1 + notebooks/examples/dataset.ipynb | 65 +++++++++ requirements/dev_requirements.in | 2 +- requirements/dev_requirements.txt | 274 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 340 insertions(+), 18 deletions(-) create mode 120000 docs/notebooks create mode 100644 notebooks/examples/dataset.ipynb (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index 23392e9c..092fe3c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,6 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) -from typing import List # -- Project information ----------------------------------------------------- @@ -31,7 +30,7 @@ release = "0.0.4" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions: List[str] = ["myst_parser"] +extensions = ["myst_nb"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/index.md b/docs/index.md index 4554a3b2..d8f04edf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,10 +1,19 @@ # Sec-certs documentation +## Notebooks + +```{toctree} +:caption: Notebook examples +notebooks/dataset.ipynb +``` + +## GitHub artifacts + ```{toctree} :maxdepth: 2 +:caption: GitHub artifacts readme.md contributing.md code_of_conduct.md license.md -``` - +``` \ No newline at end of file diff --git a/docs/notebooks b/docs/notebooks new file mode 120000 index 00000000..b50be0f4 --- /dev/null +++ b/docs/notebooks @@ -0,0 +1 @@ +./../notebooks/examples \ No newline at end of file diff --git a/notebooks/examples/dataset.ipynb b/notebooks/examples/dataset.ipynb new file mode 100644 index 00000000..7f9287aa --- /dev/null +++ b/notebooks/examples/dataset.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Dataset class examples\n", + "\n", + "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sec_certs.dataset.common_criteria import CCDataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4985\n" + ] + } + ], + "source": [ + "dset = CCDataset.from_web_latest()\n", + "print(len(dset))" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" + }, + "kernelspec": { + "display_name": "Python 3.8.1 ('certsvenv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in index 824a102f..baed56a5 100644 --- a/requirements/dev_requirements.in +++ b/requirements/dev_requirements.in @@ -12,5 +12,5 @@ flake8 pre-commit pip-tools sphinx -myst-parser +myst-nb sphinx-book-theme \ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 97debe7f..9c652d29 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -1,15 +1,41 @@ alabaster==0.7.12 # via sphinx +anyio==3.5.0 + # via jupyter-server +appnope==0.1.3 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via + # jupyter-server + # notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asttokens==2.0.5 + # via stack-data attrs==21.4.0 - # via pytest + # via + # jsonschema + # jupyter-cache + # markdown-it-py + # pytest babel==2.10.1 # via sphinx +backcall==0.2.0 + # via ipython beautifulsoup4==4.11.1 - # via pydata-sphinx-theme + # via + # nbconvert + # pydata-sphinx-theme black==22.3.0 # via -r dev_requirements.in +bleach==5.0.0 + # via nbconvert certifi==2021.10.8 # via requests +cffi==1.15.0 + # via argon2-cffi-bindings cfgv==3.3.1 # via pre-commit charset-normalizer==2.0.12 @@ -18,70 +44,203 @@ click==8.1.2 # via # black # pip-tools +colorama==0.4.4 + # via nbdime coverage[toml]==6.3.2 # via pytest-cov +debugpy==1.6.0 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert distlib==0.3.4 # via virtualenv docutils==0.17.1 # via + # myst-nb # myst-parser # pydata-sphinx-theme # sphinx + # sphinx-togglebutton +entrypoints==0.4 + # via + # jupyter-client + # nbconvert +executing==0.8.3 + # via stack-data +fastjsonschema==2.15.3 + # via nbformat filelock==3.6.0 # via virtualenv flake8==4.0.1 # via -r dev_requirements.in +gitdb==4.0.9 + # via gitpython +gitpython==3.1.27 + # via nbdime gprof2dot==2021.2.21 # via pytest-profiling +greenlet==1.1.2 + # via sqlalchemy identify==2.4.12 # via pre-commit idna==3.3 - # via requests + # via + # anyio + # requests imagesize==1.3.0 # via sphinx importlib-metadata==4.11.3 - # via sphinx + # via + # myst-nb + # sphinx +importlib-resources==5.7.1 + # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.13.0 + # via + # ipywidgets + # notebook +ipython==8.2.0 + # via + # ipykernel + # ipywidgets + # jupyter-sphinx + # myst-nb +ipython-genutils==0.2.0 + # via + # ipywidgets + # notebook +ipywidgets==7.7.0 + # via + # jupyter-sphinx + # myst-nb isort==5.10.1 # via -r dev_requirements.in +jedi==0.18.1 + # via ipython jinja2==3.1.1 # via + # jupyter-server # myst-parser + # nbconvert + # nbdime + # notebook # sphinx -markdown-it-py==2.1.0 +jsonschema==4.4.0 + # via nbformat +jupyter-cache==0.4.3 + # via myst-nb +jupyter-client==7.2.2 + # via + # ipykernel + # jupyter-server + # nbclient + # notebook +jupyter-core==4.10.0 + # via + # jupyter-client + # jupyter-server + # nbconvert + # nbformat + # notebook +jupyter-server==1.16.0 + # via + # jupyter-server-mathjax + # nbdime +jupyter-server-mathjax==0.2.5 + # via nbdime +jupyter-sphinx==0.3.2 + # via myst-nb +jupyterlab-pygments==0.2.2 + # via nbconvert +jupyterlab-widgets==1.1.0 + # via ipywidgets +markdown-it-py==1.1.0 # via # mdit-py-plugins # myst-parser markupsafe==2.1.1 - # via jinja2 + # via + # jinja2 + # nbconvert +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 -mdit-py-plugins==0.3.0 +mdit-py-plugins==0.2.8 # via myst-parser -mdurl==0.1.1 - # via markdown-it-py memory-profiler==0.60.0 # via pytest-monitor +mistune==0.8.4 + # via nbconvert mypy==0.942 # via -r dev_requirements.in mypy-extensions==0.4.3 # via # black # mypy -myst-parser==0.17.2 +myst-nb==0.13.2 # via -r dev_requirements.in +myst-parser==0.15.2 + # via myst-nb +nbclient==0.5.13 + # via + # jupyter-cache + # nbconvert +nbconvert==6.5.0 + # via + # jupyter-server + # jupyter-sphinx + # myst-nb + # notebook +nbdime==3.1.1 + # via jupyter-cache +nbformat==5.3.0 + # via + # ipywidgets + # jupyter-cache + # jupyter-server + # jupyter-sphinx + # myst-nb + # nbclient + # nbconvert + # nbdime + # notebook +nest-asyncio==1.5.5 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit +notebook==6.4.11 + # via widgetsnbextension packaging==21.3 # via + # ipykernel + # jupyter-server + # nbconvert # pydata-sphinx-theme # pytest # sphinx +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.5.1 # via -r dev_requirements.in platformdirs==2.5.1 @@ -92,22 +251,43 @@ pluggy==1.0.0 # via pytest pre-commit==2.18.1 # via -r dev_requirements.in +prometheus-client==0.14.1 + # via + # jupyter-server + # notebook +prompt-toolkit==3.0.29 + # via ipython psutil==5.9.0 # via + # ipykernel # memory-profiler # pytest-monitor +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 +pycparser==2.21 + # via cffi pydata-sphinx-theme==0.8.1 # via sphinx-book-theme pyflakes==2.4.0 # via flake8 pygments==2.12.0 - # via sphinx + # via + # ipython + # nbconvert + # nbdime + # sphinx pyparsing==3.0.7 # via packaging +pyrsistent==0.18.1 + # via jsonschema pytest==7.1.1 # via # -r dev_requirements.in @@ -120,21 +300,41 @@ pytest-monitor==1.6.3 # via -r dev_requirements.in pytest-profiling==1.7.0 # via -r dev_requirements.in +python-dateutil==2.8.2 + # via jupyter-client pytz==2022.1 # via babel pyyaml==6.0 # via + # myst-nb # myst-parser # pre-commit # sphinx-book-theme +pyzmq==22.3.0 + # via + # jupyter-client + # jupyter-server + # notebook requests==2.27.1 # via + # nbdime # pytest-monitor # sphinx +send2trash==1.8.0 + # via + # jupyter-server + # notebook six==1.16.0 # via + # asttokens + # bleach # pytest-profiling + # python-dateutil # virtualenv +smmap==5.0.0 + # via gitdb +sniffio==1.2.0 + # via anyio snowballstemmer==2.2.0 # via sphinx soupsieve==2.3.2.post1 @@ -142,11 +342,16 @@ soupsieve==2.3.2.post1 sphinx==4.5.0 # via # -r dev_requirements.in + # jupyter-sphinx + # myst-nb # myst-parser # pydata-sphinx-theme # sphinx-book-theme + # sphinx-togglebutton sphinx-book-theme==0.3.2 # via -r dev_requirements.in +sphinx-togglebutton==0.3.1 + # via myst-nb sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -159,6 +364,16 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx +sqlalchemy==1.4.35 + # via jupyter-cache +stack-data==0.2.0 + # via ipython +terminado==0.13.3 + # via + # jupyter-server + # notebook +tinycss2==1.1.1 + # via nbconvert toml==0.10.2 # via pre-commit tomli==2.0.1 @@ -168,6 +383,27 @@ tomli==2.0.1 # mypy # pep517 # pytest +tornado==6.1 + # via + # ipykernel + # jupyter-client + # jupyter-server + # nbdime + # notebook + # terminado +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-server + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook types-python-dateutil==2.8.10 # via -r dev_requirements.in types-pyyaml==6.0.5 @@ -180,17 +416,29 @@ typing-extensions==4.1.1 # via # black # mypy - # myst-parser urllib3==1.26.9 # via requests virtualenv==20.14.0 # via pre-commit +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.3.2 + # via jupyter-server wheel==0.37.1 # via # pip-tools # pytest-monitor + # sphinx-togglebutton +widgetsnbextension==3.6.0 + # via ipywidgets zipp==3.8.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip -- cgit v1.3.1 From 2ada5810445ff7122c585752c8d01e712a395394 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 08:21:14 +0200 Subject: . --- .github/workflows/docs.yml | 16 ++++++++++++---- .github/workflows/draft_release.yml | 21 ++++----------------- .gitignore | 2 ++ CONTRIBUTING.md | 23 +++++++++++++++++++---- VERSION | 1 - docs/conf.py | 8 ++++---- pyproject.toml | 26 +++++++++++++++----------- requirements/requirements.in | 1 + requirements/requirements.txt | 8 ++++++++ setup.py | 2 -- 10 files changed, 65 insertions(+), 43 deletions(-) delete mode 100644 VERSION (limited to 'docs') diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4971a49d..056153c8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,8 +6,16 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - uses: ammaraskar/sphinx-action@master + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 with: - pre-build-command: python -m pip install -r requirements/dev_requirements.txt - docs-folder: "docs/" + python-version: 3.8 + - name: Install sec-certs and deps + run: | + pip install -r requirements/requirements.txt + pip install -r requirements/dev_requirements.txt + pip install -e . + - name: Build docs + run: | + cd docs + make html diff --git a/.github/workflows/draft_release.yml b/.github/workflows/draft_release.yml index 4490c0d0..95e2cadc 100644 --- a/.github/workflows/draft_release.yml +++ b/.github/workflows/draft_release.yml @@ -2,25 +2,12 @@ name: Create release draft on: push: - # branches: - # - "main" - # tags: - # - "*.*.*" + branches: + - "main" + tags: + - "*.*.*" jobs: - check-version: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Read VERSION file - run: echo "ACTUAL_VERSION=$(cat VERSION)" >> $GITHUB_ENV - - name: Check versions - run: | - if ((${{env.PACKAGE_NUMBER}} != ${{github.ref_name}})) ; then - echo "Tag does not match version number in VERSION file" - exit 1 - fi draft-release: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 3176de71..1d2da538 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ # results folder results*/ +# version file by setuptools-scm +sec_certs/_version.py # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afefc20b..80585c2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,11 +25,26 @@ Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-too - Additionally, [compile.sh](https://github.com/crocs-muni/sec-certs/blob/main/requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. - Tests, linting and Docker all run against this reproducible environment of pinned requirements. -## Branches and releases +## Branches + +`main` is the default branch against which all pull requests are to be made. This branch is not neccessarily stable, only the releases are. + +## Releases and version strings + +- On each revision pushed onto `main` that has `*.*.*` tag, a draft release is created with prepared changelog (this step can be skipped and the Release created right from the GitHub GUI). +- This draft release is to be published manually by the maintainer. +- Version string is not indexed in `git` but can be retreived maintained by `setuptools-scm` from git tags instead. +- `setuptools-scm` will automatically, upon editable/real install of a package, infer its version and write it to `sec_certs/_version.py`. This file is not indexed as well. See more at [setuptools-scm GitHub](https://github.com/pypa/setuptools_scm) +- On publishing a release, the tool is automatically published to [PyPi](https://pypi.org/project/sec-certs/) and [DockerHub](https://hub.docker.com/repository/docker/seccerts/sec-certs). + +Note on single-sourcing the package version: More can be read [here](https://packaging.python.org/en/latest/guides/single-sourcing-package-version/). The downside of our approach is that `.git` folder and editable/real install is needed to infer the version of the package. Releases can be infered without installing the project. + +### Currently, the release process is as follows + +1. (skip this optionally) Tag a revision with `*.*.*` tag -- this will create a draft release in GitHub. +2. Modify changelog and publish the release (or create it from scratch with new tag). +3. This will automatically update PyPi and DockerHub packages. -- `main` is the default branch against which all pull requests are to be made. This branch is not neccessarily stable, only the releases are. -- [Releases](https://github.com/crocs-muni/sec-certs/releases) are prepared manually by tagging revisions of `main` branch with `x.y.z` according to [semantic versioning](https://semver.org). -- Upon each release, the tool is automatically published to [PyPi](https://pypi.org/project/sec-certs/) and [DockerHub](https://hub.docker.com/repository/docker/seccerts/sec-certs). ## Quality assurance diff --git a/VERSION b/VERSION deleted file mode 100644 index 1750564f..00000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.6 diff --git a/docs/conf.py b/docs/conf.py index 092fe3c4..6eb14738 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,8 +12,8 @@ # # import os # import sys -# sys.path.insert(0, os.path.abspath('.')) - +# sys.path.insert(0, os.path.abspath(".")) +from importlib.metadata import version # -- Project information ----------------------------------------------------- @@ -22,8 +22,8 @@ copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanis author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" # The full version, including alpha/beta/rc tags -release = "0.0.4" - +# Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker +release = version("sec-certs") # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 7ac509d6..6bae9df8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ +[build-system] +requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] + +[tool.setuptools_scm] +write_to = "sec_certs/_version.py" + [tool.black] line-length = 120 exclude = ''' @@ -16,21 +22,19 @@ exclude = ''' ''' [tool.isort] -multi_line_output=3 -include_trailing_comma=true -force_grid_wrap=0 -use_parentheses=true -ensure_newline_before_comments=true -line_length=120 -skip=["certsvenv", "build"] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 120 +skip = ["certsvenv", "build"] [tool.mypy] plugins = ["numpy.typing.mypy_plugin"] ignore_missing_imports = true -exclude="build/" +exclude = "build/" [tool.pytest.ini_options] -markers = [ - "slow: marks tests as slow (deselect with '-m \"not slow\"')" -] +markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"] addopts = "--cov sec_certs" diff --git a/requirements/requirements.in b/requirements/requirements.in index f7fc03a3..4b67a179 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -19,3 +19,4 @@ requests scikit-learn tabula-py tqdm +setuptools-scm diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f72cd3f3..a66833ac 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -50,6 +50,7 @@ packaging==21.3 # via # matplotlib # pikepdf + # setuptools-scm pandas==1.4.2 # via # -r requirements.in @@ -88,6 +89,8 @@ scikit-learn==1.0.2 # via -r requirements.in scipy==1.8.0 # via scikit-learn +setuptools-scm==6.4.2 + # via -r requirements.in six==1.16.0 # via # html5lib @@ -98,6 +101,8 @@ tabula-py==2.3.0 # via -r requirements.in threadpoolctl==3.1.0 # via scikit-learn +tomli==2.0.1 + # via setuptools-scm tqdm==4.64.0 # via -r requirements.in urllib3==1.26.9 @@ -106,3 +111,6 @@ webencodings==0.5.1 # via html5lib zipp==3.8.0 # via importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.py b/setup.py index 802dc9c9..79c76d9b 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,6 @@ setup( name="sec-certs", author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky, Jiri Michalik", author_email="svenda@fi.muni.cz", - version_config=True, - setup_requires=["setuptools-git-versioning"], packages=find_packages(), license="MIT", description="Tool for analysis of security certificates", -- cgit v1.3.1 From d04b507ab033bc0d53dc118bc1ac7772967d0b83 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 08:30:49 +0200 Subject: non-conflicting version variable name --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index 6eb14738..b821cd0a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ # import os # import sys # sys.path.insert(0, os.path.abspath(".")) -from importlib.metadata import version +from importlib.metadata import version as get_version # -- Project information ----------------------------------------------------- @@ -23,7 +23,7 @@ author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon # The full version, including alpha/beta/rc tags # Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker -release = version("sec-certs") +release = get_version("sec-certs") # -- General configuration --------------------------------------------------- -- cgit v1.3.1 From ac4dc0e7b5fa977103e63f955a92c99010e2a8b5 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 08:53:54 +0200 Subject: . --- docs/index.md | 2 ++ 1 file changed, 2 insertions(+) (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index d8f04edf..b88dd91c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,7 @@ # Sec-certs documentation +The documentation currently does not have all modules documented with `autodoc`. Most of the API showcases are documented in the *notebooks* examples below. If you want, you can run the notebooks. They are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). + ## Notebooks ```{toctree} -- cgit v1.3.1 From 8e7a3d8a172f62e13833f4d033b52e0cd38d66f6 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 11:41:58 +0200 Subject: populate docs structure --- docs/index.md | 14 ++++++++++---- docs/installation.md | 3 +++ docs/quickstart.md | 3 +++ docs/tutorial.md | 3 +++ docs/web_dashboard.md | 3 +++ 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 docs/installation.md create mode 100644 docs/quickstart.md create mode 100644 docs/tutorial.md create mode 100644 docs/web_dashboard.md (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index b88dd91c..8c43766c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,16 +1,22 @@ # Sec-certs documentation -The documentation currently does not have all modules documented with `autodoc`. Most of the API showcases are documented in the *notebooks* examples below. If you want, you can run the notebooks. They are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). +![](_static/logo.svg) -## Notebooks +There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* + +```{toctree} +:caption: User's guide +installation.md +quickstart.md +tutorial.md +web_dashboard.md +``` ```{toctree} :caption: Notebook examples notebooks/dataset.ipynb ``` -## GitHub artifacts - ```{toctree} :maxdepth: 2 :caption: GitHub artifacts diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..0e6c025a --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,3 @@ +# Installation + +TBA \ No newline at end of file diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 00000000..b40bab01 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,3 @@ +# Quickstart + +TBA \ No newline at end of file diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 00000000..47e78870 --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,3 @@ +# Tutorial + +TBA \ No newline at end of file diff --git a/docs/web_dashboard.md b/docs/web_dashboard.md new file mode 100644 index 00000000..a5909db3 --- /dev/null +++ b/docs/web_dashboard.md @@ -0,0 +1,3 @@ +# Web dashboard + +TBA \ No newline at end of file -- cgit v1.3.1 From e6a433e86be235cb4bfc0b80bcef719df42812d7 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 16:10:36 +0200 Subject: add simple installation instr. and quickstart --- CONTRIBUTING.md | 7 +------ README.md | 22 +--------------------- docs/installation.md | 31 ++++++++++++++++++++++++++++++- docs/quickstart.md | 3 ++- 4 files changed, 34 insertions(+), 29 deletions(-) (limited to 'docs') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80585c2d..542a39da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,12 +10,7 @@ You contribution is warmly welcomed. You can help by: ## Dependencies -Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/docker/Dockerfile) presents all the required dependencies, elaborated below. - -- [Java](https://www.java.com/en) is needed to parse tables in FIPS pdf documents, must be available from `PATH`. -- Some imported libraries have non-trivial dependencies to resolve: - - [pdftotext](https://github.com/jalan/pdftotext) requires [Poppler](https://poppler.freedesktop.org/) to be installed. We've experienced issues with older versions of Poppler (`0.x`), make sure to install `20.x` version of these libraries. - - [graphviz](https://pypi.org/project/graphviz/) requires `graphviz` to be on the path +For complete list of system dependencies, see [docs/installation](seccerts.org/docs/installation). ### Requirements diff --git a/README.md b/README.md index 69a036d3..6464900a 100644 --- a/README.md +++ b/README.md @@ -15,27 +15,7 @@ This project is developed by the [Centre for Research On Cryptography and Securi ## Installation -The tool can be pulled as a docker image with - -```bash -docker pull seccerts/sec-certs -``` - -Alternatively, it can be installed from PyPi with - -```bash -pip install -U sec-certs -``` - -Note, however, that `Python>=3.8` is required and there are some [additional dependencies](https://github.com/crocs-muni/sec-certs/blob/main/CONTRIBUTING.md#dependencies). - -The stable release is also published on [GitHub](https://github.com/crocs-muni/sec-certs/releases) from where it can be setup for development with - -```bash -python3 -m venv venv -source venv/bin/activate -pip install -e . -``` +Use Docker with `docker pull seccerts/sec-certs` or just `pip install -U sec-certs`. For more elaborate description, see [docs](seccerts.org/docs/installation) ## Usage (CC) diff --git a/docs/installation.md b/docs/installation.md index 0e6c025a..236a9dc9 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,3 +1,32 @@ # Installation -TBA \ No newline at end of file +The tool can be pulled as a docker image with + +```bash +docker pull seccerts/sec-certs +``` + +Alternatively, it can be installed from PyPi with + +```bash +pip install -U sec-certs +``` + +Note, however, that `Python>=3.8` is required and there are some additional dependencies (see below) that are not shipped with the binary distribution. + +The stable release is also published on [GitHub](https://github.com/crocs-muni/sec-certs/releases) from where it can be setup for development with + +```bash +python3 -m venv venv +source venv/bin/activate +pip install -e . +``` + +Alternatively, our Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/docker/Dockerfile) represents a reproducible way of setting up the environment. + +## Dependencies + +- [Java](https://www.java.com/en) is needed to parse tables in FIPS pdf documents, must be available from `PATH`. +- Some imported libraries have non-trivial dependencies to resolve: + - [pdftotext](https://github.com/jalan/pdftotext) requires [Poppler](https://poppler.freedesktop.org/) to be installed. We've experienced issues with older versions of Poppler (`0.x`), make sure to install `20.x` version of these libraries. + - [graphviz](https://pypi.org/project/graphviz/) requires `graphviz` to be on the path \ No newline at end of file diff --git a/docs/quickstart.md b/docs/quickstart.md index b40bab01..74285025 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,3 +1,4 @@ # Quickstart -TBA \ No newline at end of file +1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). +2. Use either `cc_cli.py` or `fips_cli.py` for full-fledged processing of the certificate pipeline, or explore the [dataset notebook](notebooks/dataset.ipynb). -- cgit v1.3.1 From 3049a55dd37a0fabf4f3116b512fdeb1e7218e62 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 16:45:17 +0200 Subject: drop web_dashboard docs --- docs/index.md | 1 - 1 file changed, 1 deletion(-) (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index 8c43766c..5b390b63 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,6 @@ There are three main parts of this documentation. *User's guide* describes high- installation.md quickstart.md tutorial.md -web_dashboard.md ``` ```{toctree} -- cgit v1.3.1 From 8031a46c38f20f624a384e2a69da2ff775c86470 Mon Sep 17 00:00:00 2001 From: J08nY Date: Wed, 27 Apr 2022 13:42:09 +0200 Subject: Fix docs build. --- .github/workflows/docs.yml | 15 ++++++++------- docs/web_dashboard.md | 3 --- 2 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 docs/web_dashboard.md (limited to 'docs') diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 45f80e68..f5df9712 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,16 +23,12 @@ jobs: run: | cd docs make html - - name: Archive docs - run: | - cd _build/html - zip -r docs.zip * - cd ../.. - name: Save docs artifact uses: actions/upload-artifact@v3 with: name: docs - path: _build/html/docs.zip + path: docs/_build/html/ + retention-days: 7 upload: runs-on: ubuntu-latest needs: build @@ -42,6 +38,11 @@ jobs: uses: actions/download-artifact@v3 with: name: docs + path: docs + - name: Archive docs + run: | + cd docs + zip -r docs.zip * - name: Push docs to website run: | - curl -F data=@docs.zip https://seccerts.org/docs/upload?token=${{ secrets.DOCS_AUTH_TOKEN }} + curl -F data=@docs/docs.zip https://seccerts.org/docs/upload?token=${{ secrets.DOCS_AUTH_TOKEN }} diff --git a/docs/web_dashboard.md b/docs/web_dashboard.md deleted file mode 100644 index a5909db3..00000000 --- a/docs/web_dashboard.md +++ /dev/null @@ -1,3 +0,0 @@ -# Web dashboard - -TBA \ No newline at end of file -- cgit v1.3.1 From e05e66bc8c9230e704e8aa4332a92727a18c68dc Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 17:06:37 +0200 Subject: docs: add favicon and logo --- docs/conf.py | 3 +++ docs/index.md | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index b821cd0a..701f4924 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,3 +52,6 @@ html_theme = "sphinx_book_theme" # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] + +html_logo = "_static/logo.svg" +html_favicon = "_static/logo_badge.svg" diff --git a/docs/index.md b/docs/index.md index 5b390b63..dd7c9e53 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,7 +1,5 @@ # Sec-certs documentation -![](_static/logo.svg) - There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* ```{toctree} -- cgit v1.3.1 From d4760ca46ace74c48e27a383058cc8219c4ab2a5 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 17:30:21 +0200 Subject: docs: don't show whole version, add navigation --- docs/conf.py | 3 +-- docs/index.md | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index 701f4924..c7f4c6ff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,9 +21,8 @@ project = "sec-certs" copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" -# The full version, including alpha/beta/rc tags # Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker -release = get_version("sec-certs") +release = ".".join(get_version("sec-certs").split(".")[:3]) # -- General configuration --------------------------------------------------- diff --git a/docs/index.md b/docs/index.md index dd7c9e53..084f7d08 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,6 +2,14 @@ There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* +```{toctree} +:hidden: +:caption: Navigation +Seccerts homepage +Seccerts docs +GitHub repo +``` + ```{toctree} :caption: User's guide installation.md -- cgit v1.3.1 From 84c87543cd63bf4e04be7645c08ee8d6a80509a1 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 20:11:03 +0200 Subject: fix consq. of moving dockerfile --- .github/workflows/release.yml | 6 ++--- docker/Dockerfile | 57 ------------------------------------------- docs/installation.md | 2 +- 3 files changed, 4 insertions(+), 61 deletions(-) delete mode 100644 docker/Dockerfile (limited to 'docs') diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9063beaa..3a061417 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ name: Release (PyPi, DockerHub) -on: +on: release: types: [published] @@ -27,7 +27,7 @@ jobs: docker_release: name: Release on DockerHub runs-on: ubuntu-latest - if: github.repository == 'crocs-muni/sec-certs' + if: github.repository == 'crocs-muni/sec-certs' steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -47,6 +47,6 @@ jobs: id: docker_build uses: docker/build-push-action@v2 with: - file: docker/Dockerfile + file: Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index eaf2d326..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -FROM ubuntu:jammy-20220428 - -ENV USER="user" -ENV NB_UID=1000 -ENV HOME /home/${USER} - -#installing dependencies -RUN apt-get update -RUN apt-get install python3 -y -RUN apt-get install python3-pip -y -RUN apt-get install python3-venv -y -RUN apt-get install git -y -RUN apt-get install curl -y - -# Install dependencies fo PyPDF2 and pdftotext -RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata -RUN apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y -RUN apt-get install libqpdf-dev -y -RUN apt-get install default-jdk -y -RUN apt-get install graphviz -y - -RUN adduser --disabled-password \ - --gecos "Default user" \ - --uid ${NB_UID} \ - ${USER} - -RUN chown -R ${NB_UID} ${HOME} -USER ${USER} -WORKDIR ${HOME} - -# Download only snapshot of repository -RUN \ - curl -L https://api.github.com/repos/crocs-muni/sec-certs/tarball/main > sec-certs.tar.gz && \ - mkdir sec-certs && \ - tar zxf sec-certs.tar.gz --strip-components=1 --directory sec-certs && \ - rm sec-certs.tar.gz - -WORKDIR ${HOME}/sec-certs - -# Create virtual environment -ENV VENV_PATH=${HOME}/venv -RUN python3 -m venv ${VENV_PATH} -ENV PATH="${VENV_PATH}/bin:$PATH" - -# Install dependencies, notebook is because of mybinder.org -RUN \ - pip3 install -U pip && \ - pip3 install wheel && \ - pip3 install -r requirements/requirements.txt && \ - pip3 install --no-cache notebook jupyterlab && \ - pip3 install -e . - -# #just to be sure that pdftotext is in $PATH -ENV PATH /usr/bin/pdftotext:${PATH} - -# # Run the application: -# CMD ["python3", "./cc_cli.py"] diff --git a/docs/installation.md b/docs/installation.md index 236a9dc9..8d71a170 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -22,7 +22,7 @@ source venv/bin/activate pip install -e . ``` -Alternatively, our Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/docker/Dockerfile) represents a reproducible way of setting up the environment. +Alternatively, our Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/Dockerfile) represents a reproducible way of setting up the environment. ## Dependencies -- cgit v1.3.1 From b4e575208791327b4eaf61040c39e8c31a01effb Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 20:15:32 +0200 Subject: fix symbolic link path --- docs/index.md | 2 +- docs/notebooks | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index 084f7d08..a362a10e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ tutorial.md ```{toctree} :caption: Notebook examples -notebooks/dataset.ipynb +notebooks/examples/dataset.ipynb ``` ```{toctree} diff --git a/docs/notebooks b/docs/notebooks index b50be0f4..7e2fa2a0 120000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -./../notebooks/examples \ No newline at end of file +./../notebooks \ No newline at end of file -- cgit v1.3.1 From 7ff47777d6865b826cf0613e531f6ebece635bd5 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 20:25:53 +0200 Subject: add mybinder launch buttons for docs --- docs/conf.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index c7f4c6ff..e6b76999 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,3 +54,12 @@ html_static_path = ["_static"] html_logo = "_static/logo.svg" html_favicon = "_static/logo_badge.svg" + +html_theme_options = { + "repository_url": "https://github.com/crocs-muni/sec-certs", + "repository_branch": "issue/195-Docs", + "launch_buttons": {"binderhub_url": "https://mybinder.org"}, +} + +# Don't exeute notbooks +nb_execution_mode = "off" -- cgit v1.3.1 From 586269eae0779a83bf269e881fbda1e1aadf7d97 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 21:17:55 +0200 Subject: update myst-nb --- docs/index.md | 4 + requirements/dev_requirements.in | 2 +- requirements/dev_requirements.txt | 161 +++++--------------------------------- 3 files changed, 23 insertions(+), 144 deletions(-) (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index a362a10e..371b917c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,6 +22,10 @@ tutorial.md notebooks/examples/dataset.ipynb ``` +```{admonition} Launch notebooks in MyBinder +Each of the notebooks can be launched interactively in MyBinder by clicking on 🚀 icon (top-right corner). +``` + ```{toctree} :maxdepth: 2 :caption: GitHub artifacts diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in index baed56a5..635c62d1 100644 --- a/requirements/dev_requirements.in +++ b/requirements/dev_requirements.in @@ -12,5 +12,5 @@ flake8 pre-commit pip-tools sphinx -myst-nb +myst-nb>=0.14 sphinx-book-theme \ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 9c652d29..125c77e5 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -1,17 +1,9 @@ alabaster==0.7.12 # via sphinx -anyio==3.5.0 - # via jupyter-server appnope==0.1.3 # via # ipykernel # ipython -argon2-cffi==21.3.0 - # via - # jupyter-server - # notebook -argon2-cffi-bindings==21.2.0 - # via argon2-cffi asttokens==2.0.5 # via stack-data attrs==21.4.0 @@ -25,17 +17,11 @@ babel==2.10.1 backcall==0.2.0 # via ipython beautifulsoup4==4.11.1 - # via - # nbconvert - # pydata-sphinx-theme + # via pydata-sphinx-theme black==22.3.0 # via -r dev_requirements.in -bleach==5.0.0 - # via nbconvert certifi==2021.10.8 # via requests -cffi==1.15.0 - # via argon2-cffi-bindings cfgv==3.3.1 # via pre-commit charset-normalizer==2.0.12 @@ -43,17 +29,14 @@ charset-normalizer==2.0.12 click==8.1.2 # via # black + # jupyter-cache # pip-tools -colorama==0.4.4 - # via nbdime coverage[toml]==6.3.2 # via pytest-cov debugpy==1.6.0 # via ipykernel decorator==5.1.1 # via ipython -defusedxml==0.7.1 - # via nbconvert distlib==0.3.4 # via virtualenv docutils==0.17.1 @@ -64,9 +47,7 @@ docutils==0.17.1 # sphinx # sphinx-togglebutton entrypoints==0.4 - # via - # jupyter-client - # nbconvert + # via jupyter-client executing==0.8.3 # via stack-data fastjsonschema==2.15.3 @@ -75,10 +56,6 @@ filelock==3.6.0 # via virtualenv flake8==4.0.1 # via -r dev_requirements.in -gitdb==4.0.9 - # via gitpython -gitpython==3.1.27 - # via nbdime gprof2dot==2021.2.21 # via pytest-profiling greenlet==1.1.2 @@ -86,13 +63,12 @@ greenlet==1.1.2 identify==2.4.12 # via pre-commit idna==3.3 - # via - # anyio - # requests + # via requests imagesize==1.3.0 # via sphinx importlib-metadata==4.11.3 # via + # jupyter-cache # myst-nb # sphinx importlib-resources==5.7.1 @@ -100,22 +76,10 @@ importlib-resources==5.7.1 iniconfig==1.1.1 # via pytest ipykernel==6.13.0 - # via - # ipywidgets - # notebook + # via myst-nb ipython==8.2.0 # via # ipykernel - # ipywidgets - # jupyter-sphinx - # myst-nb -ipython-genutils==0.2.0 - # via - # ipywidgets - # notebook -ipywidgets==7.7.0 - # via - # jupyter-sphinx # myst-nb isort==5.10.1 # via -r dev_requirements.in @@ -123,114 +87,66 @@ jedi==0.18.1 # via ipython jinja2==3.1.1 # via - # jupyter-server # myst-parser - # nbconvert - # nbdime - # notebook # sphinx jsonschema==4.4.0 # via nbformat -jupyter-cache==0.4.3 +jupyter-cache==0.5.0 # via myst-nb jupyter-client==7.2.2 # via # ipykernel - # jupyter-server # nbclient - # notebook jupyter-core==4.10.0 # via # jupyter-client - # jupyter-server - # nbconvert # nbformat - # notebook -jupyter-server==1.16.0 - # via - # jupyter-server-mathjax - # nbdime -jupyter-server-mathjax==0.2.5 - # via nbdime -jupyter-sphinx==0.3.2 - # via myst-nb -jupyterlab-pygments==0.2.2 - # via nbconvert -jupyterlab-widgets==1.1.0 - # via ipywidgets markdown-it-py==1.1.0 # via # mdit-py-plugins # myst-parser markupsafe==2.1.1 - # via - # jinja2 - # nbconvert + # via jinja2 matplotlib-inline==0.1.3 # via # ipykernel # ipython mccabe==0.6.1 # via flake8 -mdit-py-plugins==0.2.8 +mdit-py-plugins==0.3.0 # via myst-parser memory-profiler==0.60.0 # via pytest-monitor -mistune==0.8.4 - # via nbconvert mypy==0.942 # via -r dev_requirements.in mypy-extensions==0.4.3 # via # black # mypy -myst-nb==0.13.2 +myst-nb==0.14.0 # via -r dev_requirements.in -myst-parser==0.15.2 +myst-parser==0.17.2 # via myst-nb nbclient==0.5.13 - # via - # jupyter-cache - # nbconvert -nbconvert==6.5.0 - # via - # jupyter-server - # jupyter-sphinx - # myst-nb - # notebook -nbdime==3.1.1 # via jupyter-cache nbformat==5.3.0 # via - # ipywidgets # jupyter-cache - # jupyter-server - # jupyter-sphinx # myst-nb # nbclient - # nbconvert - # nbdime - # notebook nest-asyncio==1.5.5 # via # ipykernel # jupyter-client # nbclient - # notebook nodeenv==1.6.0 # via pre-commit -notebook==6.4.11 - # via widgetsnbextension packaging==21.3 # via # ipykernel - # jupyter-server - # nbconvert # pydata-sphinx-theme # pytest # sphinx -pandocfilters==1.5.0 - # via nbconvert parso==0.8.3 # via jedi pathspec==0.9.0 @@ -251,10 +167,6 @@ pluggy==1.0.0 # via pytest pre-commit==2.18.1 # via -r dev_requirements.in -prometheus-client==0.14.1 - # via - # jupyter-server - # notebook prompt-toolkit==3.0.29 # via ipython psutil==5.9.0 @@ -263,17 +175,13 @@ psutil==5.9.0 # memory-profiler # pytest-monitor ptyprocess==0.7.0 - # via - # pexpect - # terminado + # via pexpect pure-eval==0.2.2 # via stack-data py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 -pycparser==2.21 - # via cffi pydata-sphinx-theme==0.8.1 # via sphinx-book-theme pyflakes==2.4.0 @@ -281,8 +189,6 @@ pyflakes==2.4.0 pygments==2.12.0 # via # ipython - # nbconvert - # nbdime # sphinx pyparsing==3.0.7 # via packaging @@ -306,35 +212,23 @@ pytz==2022.1 # via babel pyyaml==6.0 # via + # jupyter-cache # myst-nb # myst-parser # pre-commit # sphinx-book-theme pyzmq==22.3.0 - # via - # jupyter-client - # jupyter-server - # notebook + # via jupyter-client requests==2.27.1 # via - # nbdime # pytest-monitor # sphinx -send2trash==1.8.0 - # via - # jupyter-server - # notebook six==1.16.0 # via # asttokens - # bleach # pytest-profiling # python-dateutil # virtualenv -smmap==5.0.0 - # via gitdb -sniffio==1.2.0 - # via anyio snowballstemmer==2.2.0 # via sphinx soupsieve==2.3.2.post1 @@ -342,7 +236,6 @@ soupsieve==2.3.2.post1 sphinx==4.5.0 # via # -r dev_requirements.in - # jupyter-sphinx # myst-nb # myst-parser # pydata-sphinx-theme @@ -368,12 +261,8 @@ sqlalchemy==1.4.35 # via jupyter-cache stack-data==0.2.0 # via ipython -terminado==0.13.3 - # via - # jupyter-server - # notebook -tinycss2==1.1.1 - # via nbconvert +tabulate==0.8.9 + # via jupyter-cache toml==0.10.2 # via pre-commit tomli==2.0.1 @@ -387,23 +276,15 @@ tornado==6.1 # via # ipykernel # jupyter-client - # jupyter-server - # nbdime - # notebook - # terminado traitlets==5.1.1 # via # ipykernel # ipython - # ipywidgets # jupyter-client # jupyter-core - # jupyter-server # matplotlib-inline # nbclient - # nbconvert # nbformat - # notebook types-python-dateutil==2.8.10 # via -r dev_requirements.in types-pyyaml==6.0.5 @@ -416,25 +297,19 @@ typing-extensions==4.1.1 # via # black # mypy + # myst-nb + # myst-parser urllib3==1.26.9 # via requests virtualenv==20.14.0 # via pre-commit wcwidth==0.2.5 # via prompt-toolkit -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.3.2 - # via jupyter-server wheel==0.37.1 # via # pip-tools # pytest-monitor # sphinx-togglebutton -widgetsnbextension==3.6.0 - # via ipywidgets zipp==3.8.0 # via # importlib-metadata -- cgit v1.3.1 From 766164f7534e4d41a1668cc553c51c0ad3c0630f Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 21:57:52 +0200 Subject: update notes --- CONTRIBUTING.md | 2 +- README.md | 2 +- cc_cli.py | 4 +- docs/conf.py | 12 +++--- docs/index.md | 3 +- docs/quickstart.md | 31 ++++++++++++++- fips_cli.py | 4 +- notebooks/examples/common_criteria.ipynb | 65 ++++++++++++++++++++++++++++++++ notebooks/examples/dataset.ipynb | 65 -------------------------------- notebooks/examples/fips.ipynb | 65 ++++++++++++++++++++++++++++++++ 10 files changed, 177 insertions(+), 76 deletions(-) create mode 100644 notebooks/examples/common_criteria.ipynb delete mode 100644 notebooks/examples/dataset.ipynb create mode 100644 notebooks/examples/fips.ipynb (limited to 'docs') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 542a39da..803cb340 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,7 @@ You contribution is warmly welcomed. You can help by: ## Dependencies -For complete list of system dependencies, see [docs/installation](seccerts.org/docs/installation). +For complete list of system dependencies, see [docs/installation](https://seccerts.org/docs/installation.html). ### Requirements diff --git a/README.md b/README.md index 6464900a..c4b19802 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This project is developed by the [Centre for Research On Cryptography and Securi ## Installation -Use Docker with `docker pull seccerts/sec-certs` or just `pip install -U sec-certs`. For more elaborate description, see [docs](seccerts.org/docs/installation) +Use Docker with `docker pull seccerts/sec-certs` or just `pip install -U sec-certs`. For more elaborate description, see [docs](https://seccerts.org/docs/installation.html) ## Usage (CC) diff --git a/cc_cli.py b/cc_cli.py index 11e1a40e..a9c20fc1 100755 --- a/cc_cli.py +++ b/cc_cli.py @@ -23,8 +23,10 @@ logger = logging.getLogger(__name__) @click.option( "-o", "--output", - type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), help="Path where the output of the experiment will be stored. May overwrite existing content.", + default=Path("./cc_dset/"), + show_default=True, ) @click.option( "-c", diff --git a/docs/conf.py b/docs/conf.py index e6b76999..384e6569 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,8 +18,8 @@ from importlib.metadata import version as get_version # -- Project information ----------------------------------------------------- project = "sec-certs" -copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" -author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" +copyright = "2020-2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon." +# author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" # Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker release = ".".join(get_version("sec-certs").split(".")[:3]) @@ -31,6 +31,9 @@ release = ".".join(get_version("sec-certs").split(".")[:3]) # ones. extensions = ["myst_nb"] +# Don't exeute notbooks +nb_execution_mode = "off" + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -39,7 +42,6 @@ templates_path = ["_templates"] # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -59,7 +61,7 @@ html_theme_options = { "repository_url": "https://github.com/crocs-muni/sec-certs", "repository_branch": "issue/195-Docs", "launch_buttons": {"binderhub_url": "https://mybinder.org"}, + "use_fullscreen_button": False, } -# Don't exeute notbooks -nb_execution_mode = "off" +myst_heading_anchors = 3 diff --git a/docs/index.md b/docs/index.md index 371b917c..8cd9c8ef 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,8 @@ tutorial.md ```{toctree} :caption: Notebook examples -notebooks/examples/dataset.ipynb +notebooks/examples/common_criteria.ipynb +notebooks/examples/fips.ipynb ``` ```{admonition} Launch notebooks in MyBinder diff --git a/docs/quickstart.md b/docs/quickstart.md index 74285025..48f7f1ce 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,4 +1,33 @@ # Quickstart 1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). -2. Use either `cc_cli.py` or `fips_cli.py` for full-fledged processing of the certificate pipeline, or explore the [dataset notebook](notebooks/dataset.ipynb). +2. Use +```python +dset = CCDataset.from_web_latest() +``` + +(Common Criteria) or + +```python +dset = FIPSDataset.from_web_latest() +``` + +(FIPS 140) to obtain freshly processed datasets from [seccerts.org](https://seccerts.org). + +```{hint} +You can work with those with the help of the [common criteria notebook](notebooks/examples/common_criteria.ipynb) or [fips notebook](notebooks/examples/fips.ipynb) and even launch them in MyBinder without installing anything. Just use the 🚀 icon (top-right corner). +``` + +If you insist on processing the whole certificates pipeline, make sure that you installed all [dependencies](installation.md#dependencies). Then, run + +```bash +cc-certs all +``` + +for Common Criteria processing, or + +```bash +fips-certs all +``` + +for FIPS 140 processing. This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`. \ No newline at end of file diff --git a/fips_cli.py b/fips_cli.py index 149d2f81..9ebbf6f1 100755 --- a/fips_cli.py +++ b/fips_cli.py @@ -26,8 +26,10 @@ logger = logging.getLogger(__name__) @click.option( "-o", "--output", - type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), help="Path to the directory where the output of the 'build' or 'new-run' actions will be stored.", + default=Path("./fips_dset/"), + show_default=True, ) @click.option( "-c", diff --git a/notebooks/examples/common_criteria.ipynb b/notebooks/examples/common_criteria.ipynb new file mode 100644 index 00000000..4aab6c61 --- /dev/null +++ b/notebooks/examples/common_criteria.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Common Criteria dataset class\n", + "\n", + "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sec_certs.dataset.common_criteria import CCDataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4985\n" + ] + } + ], + "source": [ + "dset = CCDataset.from_web_latest()\n", + "print(len(dset))" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" + }, + "kernelspec": { + "display_name": "Python 3.8.1 ('certsvenv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/examples/dataset.ipynb b/notebooks/examples/dataset.ipynb deleted file mode 100644 index 7f9287aa..00000000 --- a/notebooks/examples/dataset.ipynb +++ /dev/null @@ -1,65 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Dataset class examples\n", - "\n", - "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from sec_certs.dataset.common_criteria import CCDataset" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4985\n" - ] - } - ], - "source": [ - "dset = CCDataset.from_web_latest()\n", - "print(len(dset))" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" - }, - "kernelspec": { - "display_name": "Python 3.8.1 ('certsvenv': venv)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.1" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/examples/fips.ipynb b/notebooks/examples/fips.ipynb new file mode 100644 index 00000000..ab20ee93 --- /dev/null +++ b/notebooks/examples/fips.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FIPS Dataset class\n", + "\n", + "This notebook illustrates basic functionality with the `FIPSDataset` class that holds FIPS 140 dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sec_certs.dataset.fips import FIPSDataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4985\n" + ] + } + ], + "source": [ + "dset = FIPSDataset.from_web_latest()\n", + "print(len(dset))" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" + }, + "kernelspec": { + "display_name": "Python 3.8.1 ('certsvenv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} -- cgit v1.3.1 From 8ec5e35af3d504fa93195cbabff1240e867fb1e6 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 2 May 2022 07:59:08 +0200 Subject: declare scope of docs --- docs/index.md | 2 ++ 1 file changed, 2 insertions(+) (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index 8cd9c8ef..cb819301 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,7 @@ # Sec-certs documentation +Welcome to the technical documentation of *sec-certs* tool for the data analysis of products certified with Common Criteria or FIPS 140 frameworks. If you're looking for general description of the tool, its use cases and capabilites, we refer you to [sec-certs homepage](https://seccerts.org/). If you are looking for more advanced knowledge, e.g. how to mine your own data, how to extend the tool, and so forth, this is the right place. + There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* ```{toctree} -- cgit v1.3.1 From 43b9202d40b8aaa24bc697c583ec5135feffcf92 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 3 May 2022 17:14:44 +0200 Subject: add link to pypi docs --- docs/index.md | 1 + 1 file changed, 1 insertion(+) (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index cb819301..d7ee1808 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,6 +10,7 @@ There are three main parts of this documentation. *User's guide* describes high- Seccerts homepage Seccerts docs GitHub repo +Seccerts PyPi ``` ```{toctree} -- cgit v1.3.1 From 67c268b727a138efdcc51b1c1ecbaf30ab11b182 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 May 2022 12:18:52 +0200 Subject: Docs: autodoc, model package docs --- CONTRIBUTING.md | 9 ++++++++- docs/api/model.md | 20 +++++++++++++++++++ docs/conf.py | 2 +- docs/index.md | 5 +++++ sec_certs/model/sar_transformer.py | 39 +++++++++++++++++++++++++++++--------- 5 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 docs/api/model.md (limited to 'docs') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 803cb340..11cd09f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,4 +64,11 @@ To ivoke the tools manually, you can, in the repository root, use: - Mypy: `mypy .` - Black: `black --check .` (without the flag to reformat) - isort: `isort --check-only .` (without the flag to actually fix the issue) -- Flake8: `flake8 .` \ No newline at end of file +- Flake8: `flake8 .` + +## Documentation + +Every public method of a module that can be leveraged as an API by user should be documented. The docstrng style should +be `sphinx-oneline`. + +The documentation is built using `sphinx` with `mnyst` extension that allows for markdown files. Folder `notebooks/examples` is symbolically linked to `/docs` and its contents will be automatically parsed. These notebooks are supposed to be runnable from Binder. \ No newline at end of file diff --git a/docs/api/model.md b/docs/api/model.md new file mode 100644 index 00000000..372815c7 --- /dev/null +++ b/docs/api/model.md @@ -0,0 +1,20 @@ +# Model package + +## CPEClassifier + +```{eval-rst} +.. currentmodule:: sec_certs.model +.. autoclass:: CPEClassifier + :members: +``` + +## SARTranformer + +```{eval-rst} +.. currentmodule:: sec_certs.model +.. autoclass:: SARTransformer + :members: +``` + +## DependencyFinder + diff --git a/docs/conf.py b/docs/conf.py index 384e6569..001c209a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ release = ".".join(get_version("sec-certs").split(".")[:3]) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["myst_nb"] +extensions = ["myst_nb", "sphinx.ext.autodoc"] # Don't exeute notbooks nb_execution_mode = "off" diff --git a/docs/index.md b/docs/index.md index d7ee1808..8e0883dc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,6 +26,11 @@ notebooks/examples/common_criteria.ipynb notebooks/examples/fips.ipynb ``` +```{toctree} +:caption: API +api/model.md +``` + ```{admonition} Launch notebooks in MyBinder Each of the notebooks can be launched interactively in MyBinder by clicking on 🚀 icon (top-right corner). ``` diff --git a/sec_certs/model/sar_transformer.py b/sec_certs/model/sar_transformer.py index ee99b279..58f96c0b 100644 --- a/sec_certs/model/sar_transformer.py +++ b/sec_certs/model/sar_transformer.py @@ -19,17 +19,38 @@ class SARTransformer(BaseEstimator, TransformerMixin): """ def fit(self, certificates: Iterable[CommonCriteriaCert]) -> SARTransformer: + """ + Just returns self, no fitting needed + + :param Iterable[CommonCriteriaCert] certificates: Unused parameter + :return SARTransformer: return self + """ return self def transform(self, certificates: Iterable[CommonCriteriaCert]) -> List[Optional[Set[SAR]]]: + """ + Just a wrapper around transform_single_cert() called on an iterable of CommonCriteriaCert. + + :param Iterable[CommonCriteriaCert] certificates: Iterable of CommonCriteriaCert objects to perform the extraction on. + :return List[Optional[Set[SAR]]]: Returns List of results from transform_single_cert(). + """ return [self.transform_single_cert(cert) for cert in certificates] def transform_single_cert(self, cert: CommonCriteriaCert) -> Optional[Set[SAR]]: - sec_level_candidates, st_candidates, report_candidates = self.collect_sar_candidates_from_all_sources(cert) - return self.resolve_candidate_conflicts(sec_level_candidates, st_candidates, report_candidates, cert.dgst) + """ + Given CommonCriteriaCert, will transform SAR keywords extracted from txt files + into a set of SAR objects. Also handles extractin of correct SAR levels, duplicities and filtering. + Uses three sources: CSV scan, security target, and certification report. + The caller should assure that the certificates have the keywords extracted. + + :param CommonCriteriaCert cert: Certificate to extract SARs from + :return Optional[Set[SAR]]: Set of SARs, None if none were identified. + """ + sec_level_candidates, st_candidates, report_candidates = self._collect_sar_candidates_from_all_sources(cert) + return self._resolve_candidate_conflicts(sec_level_candidates, st_candidates, report_candidates, cert.dgst) @staticmethod - def collect_sar_candidates_from_all_sources(cert: CommonCriteriaCert) -> Tuple[Set[SAR], Set[SAR], Set[SAR]]: + def _collect_sar_candidates_from_all_sources(cert: CommonCriteriaCert) -> Tuple[Set[SAR], Set[SAR], Set[SAR]]: """ Parses SARs from three distinct sources and returns the results as a three tuple: - Security level from CSV scan @@ -43,24 +64,24 @@ class SARTransformer(BaseEstimator, TransformerMixin): def report_keywords_may_have_sars(sample: CommonCriteriaCert): return sample.pdf_data.report_keywords and SAR_DICT_KEY in sample.pdf_data.report_keywords - sec_level_sars = SARTransformer.parse_sars_from_security_level_list(cert.security_level) + sec_level_sars = SARTransformer._parse_sars_from_security_level_list(cert.security_level) if st_keywords_may_have_sars(cert): st_dict: Dict = cast(Dict, cert.pdf_data.st_keywords) - st_sars = SARTransformer.parse_sar_dict(st_dict[SAR_DICT_KEY], cert.dgst) + st_sars = SARTransformer._parse_sar_dict(st_dict[SAR_DICT_KEY], cert.dgst) else: st_sars = set() if report_keywords_may_have_sars(cert): report_dict: Dict = cast(Dict, cert.pdf_data.report_keywords) - report_sars = SARTransformer.parse_sar_dict(report_dict[SAR_DICT_KEY], cert.dgst) + report_sars = SARTransformer._parse_sar_dict(report_dict[SAR_DICT_KEY], cert.dgst) else: report_sars = set() return sec_level_sars, st_sars, report_sars @staticmethod - def resolve_candidate_conflicts( + def _resolve_candidate_conflicts( sec_level_candidates: Set[SAR], st_candidates: Set[SAR], report_candidates: Set[SAR], cert_dgst: str ) -> Optional[Set[SAR]]: final_candidates: Dict[str, SAR] = {x.family: x for x in sec_level_candidates} @@ -94,7 +115,7 @@ class SARTransformer(BaseEstimator, TransformerMixin): return set(final_candidates.values()) if final_candidates else None @staticmethod - def parse_sar_dict(dct: Dict[str, int], dgst: str) -> Set[SAR]: + def _parse_sar_dict(dct: Dict[str, int], dgst: str) -> Set[SAR]: """ Accepts st_keywords or report_keywords dictionary. Will reconstruct SAR objects from it. Each SAR family can appear multiple times in the dictionary (due to conflicts) with different levels. Iterated item will replace @@ -126,7 +147,7 @@ class SARTransformer(BaseEstimator, TransformerMixin): return {x[0] for x in sars.values()} if sars else set() @staticmethod - def parse_sars_from_security_level_list(lst: Iterable[str]) -> Set[SAR]: + def _parse_sars_from_security_level_list(lst: Iterable[str]) -> Set[SAR]: sars = set() for element in lst: try: -- cgit v1.3.1 From 99768447af7031d4ceb982ba58f49fcf8d4c4591 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 May 2022 15:25:28 +0200 Subject: update docs --- docs/api/model.md | 22 ++++++++++++++++++++++ docs/index.md | 9 +++++---- docs/installation.md | 25 ++++++++++++++++++++----- docs/quickstart.md | 29 ++++++++++++++++++++++------- 4 files changed, 69 insertions(+), 16 deletions(-) (limited to 'docs') diff --git a/docs/api/model.md b/docs/api/model.md index 372815c7..406d24be 100644 --- a/docs/api/model.md +++ b/docs/api/model.md @@ -1,5 +1,14 @@ # Model package +```{eval-rst} +.. automodule:: sec_certs.model + :no-members: +``` + +```{tip} +The examples related to this package can be found at [model notebook](./../notebooks/examples/model.ipynb). +``` + ## CPEClassifier ```{eval-rst} @@ -18,3 +27,16 @@ ## DependencyFinder +```{eval-rst} +.. currentmodule:: sec_certs.model +.. autoclass:: DependencyFinder + :members: +``` + +## DependencyVulnerabilityFinder + +```{eval-rst} +.. currentmodule:: sec_certs.model +.. autoclass:: DependencyVulnerabilityFinder + :members: +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 8e0883dc..3f44633b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,6 +4,10 @@ Welcome to the technical documentation of *sec-certs* tool for the data analysis There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* +```{admonition} Launch notebooks in MyBinder +Each of the notebooks can be launched interactively in MyBinder by clicking on 🚀 icon (top-right corner). +``` + ```{toctree} :hidden: :caption: Navigation @@ -24,6 +28,7 @@ tutorial.md :caption: Notebook examples notebooks/examples/common_criteria.ipynb notebooks/examples/fips.ipynb +notebooks/examples/model.ipynb ``` ```{toctree} @@ -31,10 +36,6 @@ notebooks/examples/fips.ipynb api/model.md ``` -```{admonition} Launch notebooks in MyBinder -Each of the notebooks can be launched interactively in MyBinder by clicking on 🚀 icon (top-right corner). -``` - ```{toctree} :maxdepth: 2 :caption: GitHub artifacts diff --git a/docs/installation.md b/docs/installation.md index 8d71a170..71e685db 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,18 +1,28 @@ # Installation -The tool can be pulled as a docker image with +::::{tab-set} +:::{tab-item} PyPi (pip) + +The tool can be installed from PyPi with ```bash -docker pull seccerts/sec-certs +pip install -U sec-certs ``` -Alternatively, it can be installed from PyPi with +Note, that `Python>=3.8` is required. + +::: + +:::{tab-item} Docker + +The tool can be pulled as a docker image with ```bash -pip install -U sec-certs +docker pull seccerts/sec-certs ``` -Note, however, that `Python>=3.8` is required and there are some additional dependencies (see below) that are not shipped with the binary distribution. +::: +:::{tab-item} Build from sources The stable release is also published on [GitHub](https://github.com/crocs-muni/sec-certs/releases) from where it can be setup for development with @@ -24,6 +34,11 @@ pip install -e . Alternatively, our Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/Dockerfile) represents a reproducible way of setting up the environment. +::: +:::: + +If you're not using Docker, you must install the dependencies as described below. + ## Dependencies - [Java](https://www.java.com/en) is needed to parse tables in FIPS pdf documents, must be available from `PATH`. diff --git a/docs/quickstart.md b/docs/quickstart.md index 48f7f1ce..14121212 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,18 +1,29 @@ # Quickstart +::::{tab-set} + +:::{tab-item} Common Criteria 1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). 2. Use ```python dset = CCDataset.from_web_latest() ``` +to obtain to obtain freshly processed dataset from [seccerts.org](https://seccerts.org). -(Common Criteria) or +3. Play with the dataset. See [example notebook](./notebooks/examples/common_criteria.ipynb). +::: +:::{tab-item} FIPS 140 +1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). +2. Use ```python dset = FIPSDataset.from_web_latest() ``` +to obtain to obtain freshly processed dataset from [seccerts.org](https://seccerts.org). -(FIPS 140) to obtain freshly processed datasets from [seccerts.org](https://seccerts.org). +3. Play with the dataset. See [example notebook](./notebooks/examples/fips.ipynb). +::: +:::: ```{hint} You can work with those with the help of the [common criteria notebook](notebooks/examples/common_criteria.ipynb) or [fips notebook](notebooks/examples/fips.ipynb) and even launch them in MyBinder without installing anything. Just use the 🚀 icon (top-right corner). @@ -20,14 +31,18 @@ You can work with those with the help of the [common criteria notebook](notebook If you insist on processing the whole certificates pipeline, make sure that you installed all [dependencies](installation.md#dependencies). Then, run +::::{tab-set} +:::{tab-item} Common Criteria ```bash -cc-certs all +$ cc-certs all ``` +::: -for Common Criteria processing, or - +:::{tab-item} FIPS 140 ```bash -fips-certs all +$ fips-certs new-run ``` +::: +:::: -for FIPS 140 processing. This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`. \ No newline at end of file +This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`. \ No newline at end of file -- cgit v1.3.1 From 4d7078d81f38d5ab4e31525ff8094be449e3b3b8 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 May 2022 15:32:11 +0200 Subject: docs tabs and copybuttons --- docs/conf.py | 8 +++++++- requirements/dev_requirements.in | 4 +++- requirements/dev_requirements.txt | 6 ++++++ 3 files changed, 16 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index 001c209a..fb66db5e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ release = ".".join(get_version("sec-certs").split(".")[:3]) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["myst_nb", "sphinx.ext.autodoc"] +extensions = ["myst_nb", "sphinx.ext.autodoc", "sphinx_design", "sphinx_copybutton"] # Don't exeute notbooks nb_execution_mode = "off" @@ -37,6 +37,12 @@ nb_execution_mode = "off" # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] +# Don't show typehints in autodoc files +autodoc_typehints = "none" + +# This is recommended by sphinx_design extension +myst_enable_extensions = ["colon_fence"] + # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in index 635c62d1..7f6d9712 100644 --- a/requirements/dev_requirements.in +++ b/requirements/dev_requirements.in @@ -13,4 +13,6 @@ pre-commit pip-tools sphinx myst-nb>=0.14 -sphinx-book-theme \ No newline at end of file +sphinx-book-theme +sphinx-design +sphinx-copybutton \ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 125c77e5..3ea4c3af 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -240,9 +240,15 @@ sphinx==4.5.0 # myst-parser # pydata-sphinx-theme # sphinx-book-theme + # sphinx-copybutton + # sphinx-design # sphinx-togglebutton sphinx-book-theme==0.3.2 # via -r dev_requirements.in +sphinx-copybutton==0.5.0 + # via -r dev_requirements.in +sphinx-design==0.1.0 + # via -r dev_requirements.in sphinx-togglebutton==0.3.1 # via myst-nb sphinxcontrib-applehelp==1.0.2 -- cgit v1.3.1 From 9db545dbb2b644a01b8e49b4b2506ea03ec36b00 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 May 2022 17:20:48 +0200 Subject: Docs: add some examples --- docs/api/dataset.md | 18 +++ docs/api/sample.md | 18 +++ docs/index.md | 19 ++- docs/quickstart.md | 10 +- examples/cc_oop_demo.py | 44 ------- notebooks/examples/common_criteria.ipynb | 198 ++++++++++++++++++++++++++++--- notebooks/examples/fips.ipynb | 139 ++++++++++++++++++++-- notebooks/examples/model.ipynb | 76 ++++++++++++ sec_certs/dataset/fips.py | 5 +- 9 files changed, 450 insertions(+), 77 deletions(-) create mode 100644 docs/api/dataset.md create mode 100644 docs/api/sample.md delete mode 100644 examples/cc_oop_demo.py create mode 100644 notebooks/examples/model.ipynb (limited to 'docs') diff --git a/docs/api/dataset.md b/docs/api/dataset.md new file mode 100644 index 00000000..ed92b31e --- /dev/null +++ b/docs/api/dataset.md @@ -0,0 +1,18 @@ +# Dataset package + +```{eval-rst} +.. automodule:: sec_certs.dataset + :no-members: +``` + +```{tip} +The examples related to this package can be found at [common criteria notebook](./../notebooks/examples/common_criteria.ipynb) and [fips notebook](./../notebooks/examples/fips.ipynb). +``` + +## CCDataset + +```{eval-rst} +.. currentmodule:: sec_certs.dataset +.. autoclass:: CCDataset + :members: +``` \ No newline at end of file diff --git a/docs/api/sample.md b/docs/api/sample.md new file mode 100644 index 00000000..29266f90 --- /dev/null +++ b/docs/api/sample.md @@ -0,0 +1,18 @@ +# Sample package + +```{eval-rst} +.. automodule:: sec_certs.sample + :no-members: +``` + +```{tip} +The examples related to this package can be found at [common criteria notebook](./../notebooks/examples/common_criteria.ipynb) and [fips notebook](./../notebooks/examples/fips.ipynb). +``` + +## CommonCriteriaCert + +```{eval-rst} +.. currentmodule:: sec_certs.sample +.. autoclass:: CommonCriteriaCert + :members: +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 3f44633b..8949debe 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,6 +4,13 @@ Welcome to the technical documentation of *sec-certs* tool for the data analysis There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* +```{button-ref} quickstart +:align: center +:color: primary +:ref-type: myst +Show me Quickstart! +``` + ```{admonition} Launch notebooks in MyBinder Each of the notebooks can be launched interactively in MyBinder by clicking on 🚀 icon (top-right corner). ``` @@ -11,6 +18,7 @@ Each of the notebooks can be launched interactively in MyBinder by clicking on ```{toctree} :hidden: :caption: Navigation +:maxdepth: 1 Seccerts homepage Seccerts docs GitHub repo @@ -18,7 +26,9 @@ Seccerts PyPi ``` ```{toctree} +:hidden: True :caption: User's guide +:maxdepth: 1 installation.md quickstart.md tutorial.md @@ -26,6 +36,8 @@ tutorial.md ```{toctree} :caption: Notebook examples +:hidden: True +:maxdepth: 1 notebooks/examples/common_criteria.ipynb notebooks/examples/fips.ipynb notebooks/examples/model.ipynb @@ -33,11 +45,16 @@ notebooks/examples/model.ipynb ```{toctree} :caption: API +:hidden: True +:maxdepth: 1 +api/sample.md +api/dataset.md api/model.md ``` ```{toctree} -:maxdepth: 2 +:maxdepth: 1 +:hidden: True :caption: GitHub artifacts readme.md contributing.md diff --git a/docs/quickstart.md b/docs/quickstart.md index 14121212..721e1341 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -6,6 +6,8 @@ 1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). 2. Use ```python +from sec-certs.dataset import CCDataset + dset = CCDataset.from_web_latest() ``` to obtain to obtain freshly processed dataset from [seccerts.org](https://seccerts.org). @@ -17,6 +19,8 @@ to obtain to obtain freshly processed dataset from [seccerts.org](https://seccer 1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). 2. Use ```python +from sec-certs.dataset import FIPSDataset + dset = FIPSDataset.from_web_latest() ``` to obtain to obtain freshly processed dataset from [seccerts.org](https://seccerts.org). @@ -45,4 +49,8 @@ $ fips-certs new-run ::: :::: -This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`. \ No newline at end of file +This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`. + +:::{hint} +If you installed the docker image, use `docker run -it sec-certs bash` to run the container interactively. +::: diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py deleted file mode 100644 index 7876c71f..00000000 --- a/examples/cc_oop_demo.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging -from datetime import datetime - -from sec_certs.config.configuration import config -from sec_certs.dataset import CCDataset - -logger = logging.getLogger(__name__) - - -def main(): - file_handler = logging.FileHandler(config.log_filepath) - stream_handler = logging.StreamHandler() - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - file_handler.setFormatter(formatter) - stream_handler.setFormatter(formatter) - logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler]) - start = datetime.now() - - # Full processing pipeline of Common Criteria dataset - dset = CCDataset() - dset.get_certs_from_web() - dset.process_protection_profiles() - dset.download_all_pdfs() - dset.convert_all_pdfs() - dset.analyze_certificates() # calls dset._extract_data() and dset._compute_heuristics() - - # Other useful API below - - # explicitly dump to json - # dset.to_json(dset.json_path) - - # Load dataset from JSON - # new_dset = CCDataset.from_json("./debug_dataset/cc_full_dataset.json") - # assert dset == new_dset - - # transform to pandas DataFrame - # df = dset.to_pandas() - - end = datetime.now() - logger.info(f"The computation took {(end-start)} seconds.") - - -if __name__ == "__main__": - main() diff --git a/notebooks/examples/common_criteria.ipynb b/notebooks/examples/common_criteria.ipynb index 4aab6c61..83a56379 100644 --- a/notebooks/examples/common_criteria.ipynb +++ b/notebooks/examples/common_criteria.ipynb @@ -4,36 +4,202 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Common Criteria dataset class\n", + "# Common Criteria example\n", "\n", - "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset" + "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset and of its sample `CommonCriteriaCert`" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "from sec_certs.dataset.common_criteria import CCDataset" + "from sec_certs.dataset.common_criteria import CCDataset\n", + "from sec_certs.dataset.common_criteria import CommonCriteriaCert\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get fresh dataset snapshot from mirror" ] }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4985\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "dset = CCDataset.from_web_latest()\n", - "print(len(dset))" + "print(len(dset)) # Print number of certificates in the dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Do some basic dataset serialization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Dump dataset into json and load it back\n", + "dset.to_json(\"./cc_dset.json\")\n", + "new_dset: CCDataset = CCDataset.from_json(\"./cc_dset.json\")\n", + "assert dset == new_dset" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Turn dataset into Pandas DataFrame\n", + "df = dset.to_pandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple dataset manipulation" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "# Iterate over certificates in dataset\n", + "for cert in dset:\n", + " pass\n", + "\n", + "# Get certificates produced by Infineon manufacturer\n", + "infineon_certs = [x for x in dset if \"Infineon\" in x.manufacturer]\n", + "df_infineon = df.loc[df.manufacturer.str.contains(\"Infineon\", case=False)]\n", + "\n", + "# Get certificates with some CVE\n", + "vulnerable_certs = [x for x in dset if x.heuristics.related_cves]\n", + "df_vulnerable = df.loc[~df.related_cves.isna()]\n", + "\n", + "# Show CVE ids of some vulnerable certificate\n", + "print(f\"{vulnerable_certs[0].heuristics.related_cves=}\")\n", + "\n", + "# Get certificates from 2015 and newer\n", + "df_2015_and_newer = df.loc[df.year_from > 2014]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Plot distribution of years of certification\n", + "df.year_from.value_counts().sort_index().plot.line()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dissect single certificate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Select a certificate and print some attributes\n", + "cert: CommonCriteriaCert = dset[\"bad93fb821395db2\"]\n", + "print(f\"{cert.name=}\")\n", + "print(f\"{cert.heuristics.cpe_matches=}\")\n", + "print(f\"{cert.heuristics.report_references.directly_referencing=}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "vulnerable_certs = [x for x in dset if x.heuristics.related_cves]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Serialize single certificate" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "cert.to_json(\"./cert.json\")\n", + "new_cert = cert.from_json(\"./cert.json\")\n", + "assert cert == new_cert\n", + "\n", + "# Serialize as Pandas series\n", + "ser = pd.Series(cert.pandas_tuple, index=cert.pandas_columns)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Assign dataset with CPE records and compute vulnerabilities\n", + "\n", + "*Note*: The data is already computed on dataset obtained with `from_web_latest()`, this is just for illustration. \n", + "*Note*: This may likely not run in Binder, as the corresponding `CVEDataset` and `CPEDataset` instances take a lot of memory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Automatically match CPEs and CVEs\n", + "_, cpe_dset, _ = dset.compute_cpe_heuristics()\n", + "dset.compute_related_cves()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create new dataset and fully process it\n", + "\n", + "*Warning*: It's not good idea to run this from notebook. It may take several hours to finnish. We recommend using `from_web_latest()` or turning this into a Python script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dset = CCDataset()\n", + "dset.get_certs_from_web()\n", + "dset.process_protection_profiles()\n", + "dset.download_all_pdfs()\n", + "dset.convert_all_pdfs()\n", + "dset.analyze_certificates()" ] } ], diff --git a/notebooks/examples/fips.ipynb b/notebooks/examples/fips.ipynb index ab20ee93..28eb2b1a 100644 --- a/notebooks/examples/fips.ipynb +++ b/notebooks/examples/fips.ipynb @@ -11,30 +11,143 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "from sec_certs.dataset.fips import FIPSDataset" + "from sec_certs.dataset.fips import FIPSDataset\n", + "from sec_certs.sample import FIPSCertificate" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get fresh dataset snapshot from mirror" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4985\n" - ] - } - ], + "outputs": [], "source": [ - "dset = FIPSDataset.from_web_latest()\n", + "dset: FIPSDataset = FIPSDataset.from_web_latest()\n", "print(len(dset))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Do some basic dataset serialization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Dump dataset into json and load it back\n", + "dset.to_json(\"./fips_dataset.json\")\n", + "new_dset = FIPSDataset.from_json(\"./fips_dataset.json\")\n", + "assert dset == new_dset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple dataset manipulation" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Get certificates from a single manufacturer\n", + "cisco_certs = [cert for cert in dset if \"Cisco\" in cert.web_scan.vendor]\n", + "\n", + "# Get certificates with some CVE\n", + "vulnerable_certs = [cert for cert in dset if cert.heuristics.related_cves]\n", + "\n", + "# Show CVE ids of some vulnerable certificate\n", + "print(f\"{vulnerable_certs[0].heuristics.related_cves=}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dissect single certificate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Select a certificate and print some attributes\n", + "cert: FIPSCertificate = dset[\"542cacae1d41132a\"]\n", + "\n", + "print(f\"{cert.web_scan.module_name=}\")\n", + "print(f\"{cert.heuristics.cpe_matches=}\")\n", + "print(f\"{cert.web_scan.level=}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Serialize single certificate" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "cert.to_json(\"./cert.json\")\n", + "new_cert: FIPSCertificate = FIPSCertificate.from_json(\"./cert.json\")\n", + "assert new_cert == cert" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create new dataset and fully process it\n", + "\n", + "*Warning*: It's not good idea to run this from notebook. It may take several hours to finnish. We recommend using `from_web_latest()` or turning this into a Python script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dset = FIPSDataset()\n", + "dset.get_certs_from_web()\n", + "dset.convert_all_pdfs()\n", + "dset.pdf_scan()\n", + "dset.extract_certs_from_tables()\n", + "dset.finalize_results()\n", + "\n", + "# TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/210 and refactor this\n", + "# if not no_download_algs:\n", + "# aset.get_certs_from_web()\n", + "# logger.info(f\"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.\")\n", + "# dset.algorithms = aset\n", + "\n", + "# TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/211 and uncomment\n", + "# dset.plot_graphs(show=False)" + ] } ], "metadata": { diff --git a/notebooks/examples/model.ipynb b/notebooks/examples/model.ipynb new file mode 100644 index 00000000..370b13cd --- /dev/null +++ b/notebooks/examples/model.ipynb @@ -0,0 +1,76 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Model\n", + "\n", + "This notebook illustrates basic functionality with the `model` package that apply complex transformations on certificates.\n", + "\n", + "*Note*: You probably don't need to use this. Instead, you should use `CCDataset` or `FIPSDataset` classes to handle the transformations for yourself." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sec_certs.dataset.common_criteria import CCDataset\n", + "from sec_certs.model import SARTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dset: CCDataset = CCDataset.from_web_latest()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SARTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "transformer = SARTransformer().fit(dset.certs.values())\n", + "extracted_sars = {x.dgst: transformer.transform_single_cert(x) for x in dset}" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" + }, + "kernelspec": { + "display_name": "Python 3.8.1 ('certsvenv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py index 2defd9f1..8cef1520 100644 --- a/sec_certs/dataset/fips.py +++ b/sec_certs/dataset/fips.py @@ -254,8 +254,9 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): len(dset), len(dset.algorithms) if dset.algorithms is not None else 0, ) - logger.info("The dataset does not contain the results of the dependency analysis - calculating them now...") - dset.finalize_results() + # TODO: Fixme, this is really costly + # logger.info("The dataset does not contain the results of the dependency analysis - calculating them now...") + # dset.finalize_results() return dset def set_local_paths(self) -> None: -- cgit v1.3.1 From 4fe5372ec385c3662a7a05d75ec40922d40c414e Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 May 2022 18:24:01 +0200 Subject: Docs: automatically generate configuration page --- .gitignore | 3 +++ docs/Makefile | 7 +++++++ docs/generate_config_page.py | 20 ++++++++++++++++++++ docs/index.md | 3 ++- sec_certs/config/configuration.py | 8 +++++++- 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100755 docs/generate_config_page.py (limited to 'docs') diff --git a/.gitignore b/.gitignore index 1d2da538..8faf062e 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,9 @@ instance/ # Sphinx documentation docs/_build/ +# Dynamically built configuration page +docs/configuration.md + # PyBuilder target/ diff --git a/docs/Makefile b/docs/Makefile index d4bb2cbb..0405fafe 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,11 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile + @if [ "$@" = "html" ]; then \ + chmod +x ./generate_config_page.py ;\ + python3 ./generate_config_page.py ;\ + fi @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @if [ "$@" = "html" ]; then \ + rm ./configuration.md ;\ + fi \ No newline at end of file diff --git a/docs/generate_config_page.py b/docs/generate_config_page.py new file mode 100755 index 00000000..d928c178 --- /dev/null +++ b/docs/generate_config_page.py @@ -0,0 +1,20 @@ +# This will generate configuration.md page from `settings.yaml` in sec_certs.configuration file + +from sec_certs.config import configuration + + +def main(): + cfg = configuration.config + + with open("./configuration.md", "w") as handle: + handle.write("# Configuration\n\n") + handle.write( + "The configuration is stored in yaml file `settings.yaml` at `sec_certs.config`. These are the supported options, descriptions and default values.\n\n" + ) + for key in cfg.__dict__: + handle.write(f"`{key}`\n\n- Description: {cfg.get_desription(key)}\n") + handle.write(f"- Default value: `{cfg.__getattribute__(key)}`\n\n") + + +if __name__ == "__main__": + main() diff --git a/docs/index.md b/docs/index.md index 8949debe..61c39ec5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,6 +32,7 @@ Seccerts PyPi installation.md quickstart.md tutorial.md +configuration.md ``` ```{toctree} @@ -44,7 +45,7 @@ notebooks/examples/model.ipynb ``` ```{toctree} -:caption: API +:caption: API reference :hidden: True :maxdepth: 1 api/sample.md diff --git a/sec_certs/config/configuration.py b/sec_certs/config/configuration.py index 8560529c..6a9a5880 100644 --- a/sec_certs/config/configuration.py +++ b/sec_certs/config/configuration.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import Any, Union +from typing import Any, Optional, Union import jsonschema import yaml @@ -30,6 +30,12 @@ class Configuration(object): return res["value"] return object.__getattribute__(self, key) + def get_desription(self, key: str) -> Optional[str]: + res = object.__getattribute__(self, key) + if isinstance(res, dict) and "description" in res: + return res["description"] + return None + DEFAULT_CONFIG_PATH = Path(__file__).parent / "settings.yaml" config = Configuration() -- cgit v1.3.1 From b01f20c1b219199ab4e1369c8cfa47c1409ba443 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 21 May 2022 09:41:28 +0200 Subject: Docs: configuratiom.md without editing makefile --- .gitignore | 3 --- docs/Makefile | 7 ------- docs/configuration.md | 26 ++++++++++++++++++++++++++ docs/generate_config_page.py | 20 -------------------- 4 files changed, 26 insertions(+), 30 deletions(-) create mode 100644 docs/configuration.md delete mode 100755 docs/generate_config_page.py (limited to 'docs') diff --git a/.gitignore b/.gitignore index 8faf062e..1d2da538 100644 --- a/.gitignore +++ b/.gitignore @@ -78,9 +78,6 @@ instance/ # Sphinx documentation docs/_build/ -# Dynamically built configuration page -docs/configuration.md - # PyBuilder target/ diff --git a/docs/Makefile b/docs/Makefile index 0405fafe..d4bb2cbb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,11 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @if [ "$@" = "html" ]; then \ - chmod +x ./generate_config_page.py ;\ - python3 ./generate_config_page.py ;\ - fi @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - @if [ "$@" = "html" ]; then \ - rm ./configuration.md ;\ - fi \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..27d07760 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,26 @@ +--- +file_format: mystnb +mystnb: + remove_code_source: true + execution_mode: 'force' +--- +# Configuration + +The configuration is stored in yaml file `settings.yaml` at `sec_certs.config`. These are the supported options, descriptions and default values. + + +```{code-cell} python +from sec_certs.config import configuration +from myst_nb import glue +from IPython.display import Markdown + +cfg = configuration.config +text = "" +for key in cfg.__dict__: + text += f"`{key}`\n\n- Description: {cfg.get_desription(key)}\n" + text += f"- Default value: `{cfg.__getattribute__(key)}`\n\n" +glue("text", Markdown(text)) +``` +```{glue:md} text +:format: myst +``` diff --git a/docs/generate_config_page.py b/docs/generate_config_page.py deleted file mode 100755 index d928c178..00000000 --- a/docs/generate_config_page.py +++ /dev/null @@ -1,20 +0,0 @@ -# This will generate configuration.md page from `settings.yaml` in sec_certs.configuration file - -from sec_certs.config import configuration - - -def main(): - cfg = configuration.config - - with open("./configuration.md", "w") as handle: - handle.write("# Configuration\n\n") - handle.write( - "The configuration is stored in yaml file `settings.yaml` at `sec_certs.config`. These are the supported options, descriptions and default values.\n\n" - ) - for key in cfg.__dict__: - handle.write(f"`{key}`\n\n- Description: {cfg.get_desription(key)}\n") - handle.write(f"- Default value: `{cfg.__getattribute__(key)}`\n\n") - - -if __name__ == "__main__": - main() -- cgit v1.3.1 From 5fcb216c86dd9b1c3e9cc82822276233ed1024a8 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 21 May 2022 09:44:26 +0200 Subject: docs: fix typo --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docs') diff --git a/docs/configuration.md b/docs/configuration.md index 27d07760..d5f6abda 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,7 +6,7 @@ mystnb: --- # Configuration -The configuration is stored in yaml file `settings.yaml` at `sec_certs.config`. These are the supported options, descriptions and default values. +The configuration is stored in yaml file `settings.yaml` at `sec_certs.config` package. These are the supported options, descriptions and default values. ```{code-cell} python -- cgit v1.3.1 From 5d1e1e8346489911eda02892e67823738dfb9e47 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 22 May 2022 13:43:36 +0200 Subject: add fipsdataset docs --- docs/api/dataset.md | 12 ++- sec_certs/dataset/common_criteria.py | 8 +- sec_certs/dataset/cpe.py | 47 ++++++++-- sec_certs/dataset/dataset.py | 4 +- sec_certs/dataset/fips.py | 168 ++++++++++++++++++----------------- 5 files changed, 147 insertions(+), 92 deletions(-) (limited to 'docs') diff --git a/docs/api/dataset.md b/docs/api/dataset.md index ed92b31e..168fa12a 100644 --- a/docs/api/dataset.md +++ b/docs/api/dataset.md @@ -5,6 +5,8 @@ :no-members: ``` +This documentation doesn't provide full API reference for all members of `dataset` package. Instead, it concentrattes on the Dataset that are immediately exposed to the users. Namely, we focus on `CCDataset` and `FIPSDataset`. + ```{tip} The examples related to this package can be found at [common criteria notebook](./../notebooks/examples/common_criteria.ipynb) and [fips notebook](./../notebooks/examples/fips.ipynb). ``` @@ -15,4 +17,12 @@ The examples related to this package can be found at [common criteria notebook]( .. currentmodule:: sec_certs.dataset .. autoclass:: CCDataset :members: -``` \ No newline at end of file +``` + +## FIPSDataset + +```{eval-rst} +.. currentmodule:: sec_certs.dataset +.. autoclass:: FIPSDataset + :members: +``` diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py index 7bdb5ae1..76bd9f94 100644 --- a/sec_certs/dataset/common_criteria.py +++ b/sec_certs/dataset/common_criteria.py @@ -89,7 +89,7 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType): def root_dir(self, new_dir: Union[str, Path]) -> None: old_dset = copy.deepcopy(self) Dataset.root_dir.fset(self, new_dir) # type: ignore - self.set_local_paths() + self._set_local_paths() if self.state and old_dset.root_dir != Path(".."): logger.info(f"Changing root dir of partially processed dataset. All contents will get copied to {new_dir}") @@ -220,7 +220,7 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType): return all_cert_ids - def set_local_paths(self): + def _set_local_paths(self): for cert in self: cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir) @@ -299,7 +299,7 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType): if not keep_metadata: shutil.rmtree(self.web_dir) - self.set_local_paths() + self._set_local_paths() self.state.meta_sources_parsed = True def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> Dict[str, "CommonCriteriaCert"]: @@ -791,7 +791,6 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType): update_dset: CCDatasetMaintenanceUpdates = CCDatasetMaintenanceUpdates( {x.dgst: x for x in updates}, root_dir=self.mu_dataset_path, name="Maintenance updates" ) - update_dset.set_local_paths() update_dset.download_all_pdfs() update_dset.convert_all_pdfs() update_dset._extract_data() @@ -822,6 +821,7 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType): ): super().__init__(certs, root_dir, name, description, state) # type: ignore self.state.meta_sources_parsed = True + self._set_local_paths() @property def certs_dir(self) -> Path: diff --git a/sec_certs/dataset/cpe.py b/sec_certs/dataset/cpe.py index 43ae3365..0ae35081 100644 --- a/sec_certs/dataset/cpe.py +++ b/sec_certs/dataset/cpe.py @@ -20,6 +20,10 @@ logger = logging.getLogger(__name__) @dataclass class CPEDataset(ComplexSerializableType): + """ + Dataset of CPE records. Includes look-up dictionaries for fast search. + """ + was_enhanced_with_vuln_cpes: bool json_path: Path cpes: Dict[str, CPE] @@ -68,7 +72,7 @@ class CPEDataset(ComplexSerializableType): def build_lookup_dicts(self) -> None: """ - Will build look-up dictionaries that are used for fast matching + Will build look-up dictionaries that are used for fast matching. """ logger.info("CPE dataset: building lookup dictionaries.") self.vendor_to_versions = {x.vendor: set() for x in self} @@ -90,6 +94,13 @@ class CPEDataset(ComplexSerializableType): @classmethod def from_web(cls, json_path: Union[str, Path], init_lookup_dicts: bool = True) -> "CPEDataset": + """ + Creates CPEDataset from NIST resources published on-line + + :param Union[str, Path] json_path: Path to store the dataset to + :param bool init_lookup_dicts: If dictionaries for fast matching should be computed, defaults to True + :return CPEDataset: The resulting dataset + """ with tempfile.TemporaryDirectory() as tmp_dir: xml_path = Path(tmp_dir) / cls.cpe_xml_basename zip_path = Path(tmp_dir) / (cls.cpe_xml_basename + ".zip") @@ -98,10 +109,10 @@ class CPEDataset(ComplexSerializableType): with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(tmp_dir) - return cls.from_xml(xml_path, json_path, init_lookup_dicts) + return cls._from_xml(xml_path, json_path, init_lookup_dicts) @classmethod - def from_xml( + def _from_xml( cls, xml_path: Union[str, Path], json_path: Union[str, Path], init_lookup_dicts: bool = True ) -> "CPEDataset": logger.info("Loading CPE dataset from XML.") @@ -128,22 +139,48 @@ class CPEDataset(ComplexSerializableType): @classmethod def from_json(cls, input_path: Union[str, Path]) -> "CPEDataset": + """ + Loads dataset from json + + :param Union[str, Path] input_path: Path to the serialized json dataset + :return CPEDataset: the resulting dataset. + """ dset = cast("CPEDataset", ComplexSerializableType.from_json(input_path)) dset.json_path = Path(input_path) return dset @classmethod def from_dict(cls, dct: Dict[str, Any], init_lookup_dicts: bool = True) -> "CPEDataset": + """ + Loads dataset from dictionary. + + :param Dict[str, Any] dct: Dictionary that holds the dataset + :param bool init_lookup_dicts: Whether look-up dicts should be computed as a part of initialization, defaults to True + :return CPEDataset: the resulting dataset. + """ return cls(dct["was_enhanced_with_vuln_cpes"], Path("../"), dct["cpes"], init_lookup_dicts) def to_pandas(self) -> pd.DataFrame: + """ + Turns the dataset into pandas DataFrame. Each CPE record forms a row. + + :return pd.DataFrame: the resulting DataFrame + """ df = pd.DataFrame([x.pandas_tuple for x in self], columns=CPE.pandas_columns) df = df.set_index("uri") return df @serialize def enhance_with_cpes_from_cve_dataset(self, cve_dset: Union[CVEDataset, str, Path]) -> None: - def adding_condition( + """ + Some CPEs are present only in the CVEDataset and are missing from the CPE Dataset. + This method goes through the provided CVEDataset and enriches self with CPEs from + the CVEDataset. + + :param Union[CVEDataset, str, Path] cve_dset: CVEDataset of a path to it. + """ + + def _adding_condition( considered_cpe: CPE, vndr_item_lookup: Set[Tuple[str, str]], vndr_item_version_lookup: Set[Tuple[str, str, str]], @@ -174,7 +211,7 @@ class CPEDataset(ComplexSerializableType): vendor_item_lookup = {(cpe.vendor, cpe.item_name) for cpe in self} vendor_item_version_lookup = {(cpe.vendor, cpe.item_name, cpe.version) for cpe in self} for cpe in helpers.tqdm(all_cpes_in_cve_dset, desc="Enriching CPE dataset with new CPEs"): - if adding_condition(cpe, vendor_item_lookup, vendor_item_version_lookup): + if _adding_condition(cpe, vendor_item_lookup, vendor_item_version_lookup): new_cpe = copy.deepcopy(cpe) new_cpe.start_version = None new_cpe.end_version = None diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py index e63aea9f..34dc9003 100644 --- a/sec_certs/dataset/dataset.py +++ b/sec_certs/dataset/dataset.py @@ -128,10 +128,10 @@ class Dataset(Generic[CertSubType], ABC): def from_json(cls: Type[DatasetSubType], input_path: Union[str, Path]) -> DatasetSubType: dset = cast("DatasetSubType", ComplexSerializableType.from_json(input_path)) dset.root_dir = Path(input_path).parent.absolute() - dset.set_local_paths() + dset._set_local_paths() return dset - def set_local_paths(self) -> None: + def _set_local_paths(self) -> None: raise NotImplementedError("Not meant to be implemented by the base class.") @abstractmethod diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py index 8cef1520..44c6ebdf 100644 --- a/sec_certs/dataset/fips.py +++ b/sec_certs/dataset/fips.py @@ -1,14 +1,11 @@ import logging -import os import tempfile -from itertools import groupby from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set from bs4 import BeautifulSoup, NavigableString from graphviz import Digraph -from sec_certs import constants as constants from sec_certs import helpers as helpers from sec_certs import parallel_processing as cert_processing from sec_certs.config.configuration import config @@ -23,6 +20,10 @@ logger = logging.getLogger(__name__) class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): + """ + Class for processing of FIPSCertificate samples. Inherits from `ComplexSerializableType` and base abstract `Dataset` class. + """ + def __init__( self, certs: Dict[str, FIPSCertificate] = dict(), @@ -36,48 +37,36 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): self.new_files = 0 @property - def results_dir(self) -> Path: - return self.root_dir / "results" - - @property - def policies_dir(self) -> Path: + def _policies_dir(self) -> Path: return self.root_dir / "security_policies" @property - def fragments_dir(self) -> Path: + def _fragments_dir(self) -> Path: return self.root_dir / "fragments" @property - def algs_dir(self) -> Path: + def _algs_dir(self) -> Path: return self.web_dir / "algorithms" - # After web scan, there should be a FIPSCertificate object created for every entry - @property - def successful_web_scan(self) -> bool: - return all(self.certs) and all(cert.web_scan for cert in self.certs.values() if cert is not None) - - @property - def successful_pdf_scan(self) -> bool: - return all(cert.pdf_scan for cert in self.certs.values() if cert is not None) - def get_certs_from_name(self, module_name: str) -> List[FIPSCertificate]: - return [crt for crt in self if crt.web_scan.module_name == module_name] + """ + Returns list of certificates that match given name. - def find_empty_pdfs(self) -> Tuple[List, List]: - missing = [] - not_available = [] - for i in self.certs: - if not (self.policies_dir / f"{i}.pdf").exists(): - missing.append(i) - elif os.path.getsize(self.policies_dir / f"{i}.pdf") < constants.FIPS_NOT_AVAILABLE_CERT_SIZE: - not_available.append(i) - return missing, not_available + :param str module_name: name to search for + :return List[FIPSCertificate]: List of certificates with web_scan.module_name == module_name + """ + return [crt for crt in self if crt.web_scan.module_name == module_name] @serialize def pdf_scan(self, redo: bool = False) -> None: + """ + Extracts data from pdf files + + :param bool redo: Whether to try again with failed files, defaults to False + """ logger.info("Entering PDF scan.") - self.fragments_dir.mkdir(parents=True, exist_ok=True) + self._fragments_dir.mkdir(parents=True, exist_ok=True) keywords = cert_processing.process_parallel( FIPSCertificate.find_keywords, @@ -89,7 +78,7 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): for keyword, cert in keywords: self.certs[cert.dgst].pdf_scan.keywords = keyword - def match_algs(self) -> Dict[str, int]: + def _match_algs(self) -> Dict[str, int]: output = {} for cert in self.certs.values(): # if the pdf has not been processed, no matching can be done @@ -103,18 +92,24 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): return output def download_all_pdfs(self, cert_ids: Optional[Set[str]] = None) -> None: + """ + Downloads all pdf files related to the certificates specified with cert_ids. + + :param Optional[Set[str]] cert_ids: cert_ids to download the pdfs foor, defaults to None + :raises RuntimeError: If no cert_ids are specified, raises. + """ sp_paths, sp_urls = [], [] - self.policies_dir.mkdir(exist_ok=True) + self._policies_dir.mkdir(exist_ok=True) if cert_ids is None: raise RuntimeError("You need to provide cert ids to FIPS download PDFs functionality.") for cert_id in cert_ids: - if not (self.policies_dir / f"{cert_id}.pdf").exists() or ( + if not (self._policies_dir / f"{cert_id}.pdf").exists() or ( fips_dgst(cert_id) in self.certs and not self.certs[fips_dgst(cert_id)].state.txt_state ): sp_urls.append( f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf" ) - sp_paths.append(self.policies_dir / f"{cert_id}.pdf") + sp_paths.append(self._policies_dir / f"{cert_id}.pdf") logger.info(f"downloading {len(sp_urls)} module pdf files") cert_processing.process_parallel( FIPSCertificate.download_security_policy, @@ -124,7 +119,7 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): ) self.new_files += len(sp_urls) - def download_all_htmls(self, cert_ids: Set[str]) -> List[str]: + def _download_all_htmls(self, cert_ids: Set[str]) -> List[str]: html_paths, html_urls = [], [] new_files = [] self.web_dir.mkdir(exist_ok=True) @@ -158,17 +153,20 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): @serialize def convert_all_pdfs(self) -> None: + """ + Converts all pdfs to text files + """ logger.info("Converting FIPS sample reports to .txt") tuples = [ - (cert, self.policies_dir / f"{cert.cert_id}.pdf", self.policies_dir / f"{cert.cert_id}.pdf.txt") + (cert, self._policies_dir / f"{cert.cert_id}.pdf", self._policies_dir / f"{cert.cert_id}.pdf.txt") for cert in self.certs.values() - if not cert.state.txt_state and (self.policies_dir / f"{cert.cert_id}.pdf").exists() + if not cert.state.txt_state and (self._policies_dir / f"{cert.cert_id}.pdf").exists() ] cert_processing.process_parallel( FIPSCertificate.convert_pdf_file, tuples, config.n_threads, progress_bar_desc="Converting to txt" ) - def prepare_dataset(self, test: Optional[Path] = None, update: bool = False) -> Set[str]: + def _prepare_dataset(self, test: Optional[Path] = None, update: bool = False) -> Set[str]: if test: html_files = [test] else: @@ -197,8 +195,8 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): return cert_ids - def download_neccessary_files(self, cert_ids: Set[str]) -> None: - self.download_all_htmls(cert_ids) + def _download_neccessary_files(self, cert_ids: Set[str]) -> None: + self._download_all_htmls(cert_ids) self.download_all_pdfs(cert_ids) def _get_certificates_from_html(self, html_file: Path, update: bool = False) -> Set[str]: @@ -220,15 +218,21 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): @serialize def web_scan(self, cert_ids: Set[int], redo: bool = False) -> None: + """ + Creates FIPSCertificate object from the relevant html file that must be downlaoded. + + :param Set[int] cert_ids: Cert ids to create FIPSCertificate objects for. + :param bool redo: whether to re-attempt with failed certificates, defaults to False + """ logger.info("Entering web scan.") for cert_id in cert_ids: dgst = fips_dgst(cert_id) self.certs[dgst] = FIPSCertificate.html_from_file( self.web_dir / f"{cert_id}.html", FIPSCertificate.State( - (self.policies_dir / str(cert_id)).with_suffix(".pdf"), + (self._policies_dir / str(cert_id)).with_suffix(".pdf"), (self.web_dir / str(cert_id)).with_suffix(".html"), - (self.fragments_dir / str(cert_id)).with_suffix(".txt"), + (self._fragments_dir / str(cert_id)).with_suffix(".txt"), False, None, False, @@ -239,6 +243,9 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): @classmethod def from_web_latest(cls) -> "FIPSDataset": + """ + Fetches the fresh snapshot of FIPSDataset from mirror. + """ with tempfile.TemporaryDirectory() as tmp_dir: dset_path = Path(tmp_dir) / "fips_latest_dataset.json" logger.info("Downloading the latest FIPS dataset.") @@ -259,10 +266,10 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): # dset.finalize_results() return dset - def set_local_paths(self) -> None: + def _set_local_paths(self) -> None: cert: FIPSCertificate for cert in self.certs.values(): - cert.set_local_paths(self.policies_dir, self.web_dir, self.fragments_dir) + cert.set_local_paths(self._policies_dir, self.web_dir, self._fragments_dir) @serialize def get_certs_from_web( @@ -285,11 +292,11 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): logger.info("Downloading required html files") self.web_dir.mkdir(parents=True, exist_ok=True) - self.policies_dir.mkdir(exist_ok=True) - self.algs_dir.mkdir(exist_ok=True) + self._policies_dir.mkdir(exist_ok=True) + self._algs_dir.mkdir(exist_ok=True) # Download files containing all available module certs (always) - cert_ids = self.prepare_dataset(test, update) + cert_ids = self._prepare_dataset(test, update) if not no_download_algorithms: aset = FIPSAlgorithmDataset({}, Path(self.root_dir / "web" / "algorithms"), "algorithms", "sample algs") @@ -299,22 +306,10 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): self.algorithms = aset logger.info("Downloading certificate html and security policies") - self.download_neccessary_files(cert_ids) + self._download_neccessary_files(cert_ids) self.web_scan(cert_ids, redo=redo_web_scan) - @serialize - def deprocess(self) -> None: - # TODO think of a better way to make resulting json smaller and easier to share - logger.info( - "Removing 'heuristics' field. This dataset can be used to be uploaded and later downloaded using latest_snapshot() or something" - ) - cert: FIPSCertificate - for cert in self.certs.values(): - cert.heuristics = FIPSCertificate.FIPSHeuristics(dict(), [], 0) - - self.match_algs() - @serialize def extract_certs_from_tables(self, high_precision: bool) -> List[Path]: """ @@ -342,11 +337,11 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): certificate.pdf_scan.algorithms += algorithms return not_decoded - def remove_algorithms_from_extracted_data(self) -> None: + def _remove_algorithms_from_extracted_data(self) -> None: for cert in self.certs.values(): cert.remove_algorithms() - def unify_algorithms(self) -> None: + def _unify_algorithms(self) -> None: for certificate in self.certs.values(): new_algorithms: List[Dict] = [] united_algorithms = [ @@ -440,7 +435,7 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): return False return True - def validate_results(self) -> None: + def _validate_results(self) -> None: """ Function that validates results and finds the final connection output """ @@ -482,10 +477,16 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): @serialize def finalize_results(self, use_nist_cpe_matching_dict: bool = True, perform_cpe_heuristics: bool = True): + """ + Performs processing of extracted data. Computes all heuristics. + + :param bool use_nist_cpe_matching_dict: If NIST CPE matching dictionary shall be used to drive computing related CVEs, defaults to True + :param bool perform_cpe_heuristics: If CPE heuristics shall be computed, defaults to True + """ logger.info("Entering 'analysis' and building connections between certificates.") - self.unify_algorithms() - self.remove_algorithms_from_extracted_data() - self.validate_results() + self._unify_algorithms() + self._remove_algorithms_from_extracted_data() + self._validate_results() if perform_cpe_heuristics: _, _, cve_dset = self.compute_cpe_heuristics() self.compute_related_cves(use_nist_cpe_matching_dict=use_nist_cpe_matching_dict, cve_dset=cve_dset) @@ -521,7 +522,7 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): attr = {"pdf": "pdf_scan", "web": "web_scan", "heuristics": "heuristics"}[connection_list] return getattr(self.certs[dgst], attr).connections - def create_dot_graph( + def _create_dot_graph( self, output_file_name: str, connection_list: str = "heuristics", @@ -586,6 +587,11 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): single_dot.render(self.root_dir / (str(output_file_name) + "_single"), view=show) def to_dict(self) -> Dict[str, Any]: + """ + Serializes dataset into a dictionary + + :return Dict[str, Any]: Dictionary that holds the whole dataset. + """ return { "timestamp": self.timestamp, "sha256_digest": self.sha256_digest, @@ -598,6 +604,12 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): @classmethod def from_dict(cls, dct: Dict[str, Any]) -> "FIPSDataset": + """ + Reconstructs the original dataset from a dictionary + + :param Dict[str, Any] dct: Dictionary that holds the serialized dataset + :return FIPSDataset: Deserialized FIPSDataset that corresponds to `dct` contents + """ certs = dct["certs"] dset = cls(certs, Path("./"), dct["name"], dct["description"]) dset.algorithms = dct["algs"] @@ -607,16 +619,12 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): ) return dset - def group_vendors(self) -> Dict[str, List[str]]: - vendors: Dict[str, List[str]] = {} - v = {x.web_scan.vendor.lower() for x in self.certs.values() if x is not None and x.web_scan.vendor is not None} - v_sorted = sorted(v, key=FIPSCertificate.get_compare) - for prefix, a in groupby(v_sorted, key=FIPSCertificate.get_compare): - vendors[prefix] = list(a) - - return vendors - def plot_graphs(self, show: bool = False) -> None: - self.create_dot_graph("full_graph", show=show) - self.create_dot_graph("web_only_graph", "web", show=show) - self.create_dot_graph("pdf_only_graph", "pdf", show=show) + """ + Plots FIPS graphs. + # TODO: Currently broken, see https://github.com/crocs-muni/sec-certs/issues/211 + :param bool show: If plots should be showed with .show() method, defaults to False + """ + self._create_dot_graph("full_graph", show=show) + self._create_dot_graph("web_only_graph", "web", show=show) + self._create_dot_graph("pdf_only_graph", "pdf", show=show) -- cgit v1.3.1 From f951d8b2d4aa31b9758fe145a5553a56b486f64b Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 23 May 2022 14:23:38 +0200 Subject: fix typo in quickstart docs --- docs/quickstart.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/quickstart.md b/docs/quickstart.md index 721e1341..e16a570d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -6,7 +6,7 @@ 1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). 2. Use ```python -from sec-certs.dataset import CCDataset +from sec_certs.dataset import CCDataset dset = CCDataset.from_web_latest() ``` @@ -19,7 +19,7 @@ to obtain to obtain freshly processed dataset from [seccerts.org](https://seccer 1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). 2. Use ```python -from sec-certs.dataset import FIPSDataset +from sec_certs.dataset import FIPSDataset dset = FIPSDataset.from_web_latest() ``` -- cgit v1.3.1 From 67d7a2aa80fa99cbae1d98a7f984620ac13943df Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 23 May 2022 20:40:54 +0200 Subject: adds FIPSCertificate docs --- docs/api/sample.md | 8 +++ sec_certs/dataset/fips.py | 2 +- sec_certs/sample/fips.py | 152 ++++++++++++++++++++++++++++++++++++---------- 3 files changed, 128 insertions(+), 34 deletions(-) (limited to 'docs') diff --git a/docs/api/sample.md b/docs/api/sample.md index 29266f90..4ff17be1 100644 --- a/docs/api/sample.md +++ b/docs/api/sample.md @@ -15,4 +15,12 @@ The examples related to this package can be found at [common criteria notebook]( .. currentmodule:: sec_certs.sample .. autoclass:: CommonCriteriaCert :members: +``` + +## FIPSCertificate + +```{eval-rst} +.. currentmodule:: sec_certs.sample +.. autoclass:: FIPSCertificate + :members: ``` \ No newline at end of file diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py index 7a11a9cb..d70cf57f 100644 --- a/sec_certs/dataset/fips.py +++ b/sec_certs/dataset/fips.py @@ -228,7 +228,7 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType): logger.info("Entering web scan.") for cert_id in cert_ids: dgst = fips_dgst(cert_id) - self.certs[dgst] = FIPSCertificate.html_from_file( + self.certs[dgst] = FIPSCertificate.from_html_file( self.web_dir / f"{cert_id}.html", FIPSCertificate.State( (self._policies_dir / str(cert_id)).with_suffix(".pdf"), diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py index 0b769dec..e5ea543e 100644 --- a/sec_certs/sample/fips.py +++ b/sec_certs/sample/fips.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import copy import re from dataclasses import dataclass, field @@ -22,6 +24,13 @@ from sec_certs.serialization.json import ComplexSerializableType class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuristics"], ComplexSerializableType): + """ + Data structure for common FIPS 140 certificate. Contains several inner classes that layer the data logic. + Can be serialized into/from json (`ComplexSerializableType`). + Is basic element of `FIPSDataset`. The functionality is mostly related to holding data and transformations that + the certificate can handle itself. `FIPSDataset` class then instrument this functionality. + """ + FIPS_BASE_URL: ClassVar[str] = "https://csrc.nist.gov" FIPS_MODULE_URL: ClassVar[ str @@ -29,6 +38,10 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @dataclass(eq=True) class State(ComplexSerializableType): + """ + Holds state of the `FIPSCertificate` + """ + sp_path: Path html_path: Path fragment_path: Path @@ -67,6 +80,10 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @dataclass(eq=True) class Algorithm(ComplexSerializableType): + """ + Data structure for algorithm of `FIPSCertificate` + """ + cert_id: str vendor: str implementation: str @@ -87,6 +104,10 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @dataclass(eq=True) class WebScan(ComplexSerializableType): + """ + Data structure for data obtained from scanning certificate webpage at NIST.gov + """ + module_name: Optional[str] standard: Optional[str] status: Optional[str] @@ -147,6 +168,10 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @dataclass(eq=True) class PdfScan(ComplexSerializableType): + """ + Data structure that holds data obtained from scanning pdf files (or their converted txt documents). + """ + cert_id: int keywords: Dict algorithms: List @@ -165,6 +190,10 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @dataclass(eq=True) class FIPSHeuristics(Heuristics, ComplexSerializableType): + """ + Data structure that holds data obtained by processing the certificate and applying various heuristics. + """ + keywords: Dict[str, Dict] algorithms: List[Dict[str, Dict]] unmatched_algs: int @@ -192,6 +221,9 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @property def dgst(self) -> str: + """ + Returns primary key of the certificate, its id. + """ return fips_dgst(self.cert_id) # TODO: Fix type errors, they exist because FIPS uses this as property to change variable names, while CC and abstract class have variables @@ -221,6 +253,9 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @staticmethod def download_security_policy(cert: Tuple[str, Path]) -> None: + """ + Downloads security policy file from web. Staticmethod to allow for parametrization. + """ exit_code = helpers.download_file(*cert, delay=1) if exit_code != requests.codes.ok: logger.error(f"Failed to download security policy from {cert[0]}, code: {exit_code}") @@ -228,20 +263,26 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris def __init__( self, cert_id: int, - web_scan: "FIPSCertificate.WebScan", - pdf_scan: "FIPSCertificate.PdfScan", - heuristics: "FIPSCertificate.FIPSHeuristics", + web_scan: FIPSCertificate.WebScan, + pdf_scan: FIPSCertificate.PdfScan, + heuristics: FIPSCertificate.FIPSHeuristics, state: State, ): super().__init__() self.cert_id = cert_id self.web_scan = web_scan self.pdf_scan = pdf_scan - self.heuristics: "FIPSCertificate.FIPSHeuristics" = heuristics + self.heuristics: FIPSCertificate.FIPSHeuristics = heuristics self.state = state @classmethod - def from_dict(cls, dct: Dict) -> "FIPSCertificate": + def from_dict(cls, dct: Dict) -> FIPSCertificate: + """ + Deserializes dictionary into FIPSCertificate + + :param Dict dct: dictionary that holds the FIPSCertificate data + :return FIPSCertificate: object reconstructed from dct + """ new_dct = dct.copy() if new_dct["web_scan"].date_validation: @@ -253,6 +294,12 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @staticmethod def download_html_page(cert: Tuple[str, Path]) -> Optional[Tuple[str, Path]]: + """ + Wrapper for downloading a file. `delay=1` introduced to avoid problems with requests at NIST.gov + + :param Tuple[str, Path] cert: tuple url, output_path + :return Optional[Tuple[str, Path]]: None on success, `cert` on failure. + """ exit_code = helpers.download_file(*cert, delay=1) if exit_code != requests.codes.ok: logger.error(f"Failed to download html page from {cert[0]}, code: {exit_code}") @@ -260,7 +307,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris return None @staticmethod - def initialize_dictionary() -> Dict[str, Any]: + def _initialize_dictionary() -> Dict[str, Any]: return { "module_name": None, "standard": None, @@ -296,8 +343,9 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris def parse_caveat(current_text: str) -> Dict[str, Dict[str, int]]: """ Parses content of "Caveat" of FIPS CMVP .html file + :param current_text: text of "Caveat" - :return: dictionary of all found algorithm IDs + :return dictionary of all found algorithm IDs """ ids_found: Dict[str, Dict[str, int]] = {} r_key = r"(?P\w+)?\s?(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)+(?P\d+)" @@ -315,9 +363,10 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris def extract_algorithm_certificates(current_text: str, in_pdf: bool = False) -> List[Optional[Dict[str, List[str]]]]: """ Parses table of FIPS (non) allowed algorithms + :param current_text: Contents of the table :param in_pdf: Specifies whether the table was found in a PDF security policies file - :return: List containing one element - dictionary with all parsed algorithm cert ids + :return List containing one element - dictionary with all parsed algorithm cert ids """ set_items = set() if in_pdf: @@ -333,8 +382,9 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris def parse_table(element: Union[Tag, NavigableString]) -> List[Dict[str, Any]]: """ Parses content of tags in FIPS .html CMVP page + :param element: text in
tags - :return: list of all found algorithm IDs + :return list of all found algorithm IDs """ found_items = [] trs = element.find_all("tr") @@ -355,7 +405,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris return found_items @staticmethod - def parse_html_main(current_div: Tag, html_items_found: Dict, pairs: Dict[str, str]) -> None: + def _parse_html_main(current_div: Tag, html_items_found: Dict, pairs: Dict[str, str]) -> None: title = current_div.find("div", class_="col-md-3").text.strip() content = ( current_div.find("div", class_="col-md-9") @@ -391,7 +441,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris html_items_found[pairs[title]] = content @staticmethod - def parse_vendor(current_div: Tag, html_items_found: Dict, current_file: Path) -> None: + def _parse_vendor(current_div: Tag, html_items_found: Dict, current_file: Path) -> None: vendor_string = current_div.find("div", "panel-body").find("a") if not vendor_string: @@ -406,7 +456,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris logger.warning(f"NO VENDOR FOUND {current_file}") @staticmethod - def parse_lab(current_div: Tag, html_items_found: Dict, current_file: Path) -> None: + def _parse_lab(current_div: Tag, html_items_found: Dict, current_file: Path) -> None: html_items_found["lab"] = list(current_div.find("div", "panel-body").children)[0].strip() html_items_found["nvlap_code"] = ( list(current_div.find("div", "panel-body").children)[2].strip().split("\n")[1].strip() @@ -427,20 +477,29 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris html_items_found["certificate_www"] = constants.FIPS_BASE_URL + links[1].get("href") @staticmethod - def normalize(items: Dict) -> None: + def _normalize(items: Dict) -> None: items["module_type"] = items["module_type"].lower().replace("-", " ").title() items["embodiment"] = items["embodiment"].lower().replace("-", " ").replace("stand alone", "standalone").title() @staticmethod - def parse_validation_dates(current_div: Tag, html_items_found: Dict) -> None: + def _parse_validation_dates(current_div: Tag, html_items_found: Dict) -> None: table = current_div.find("table") rows = table.find("tbody").findAll("tr") html_items_found["date_validation"] = [parser.parse(td.text).date() for td in [row.find("td") for row in rows]] @classmethod - def html_from_file( - cls, file: Path, state: State, initialized: "FIPSCertificate" = None, redo: bool = False - ) -> "FIPSCertificate": + def from_html_file( + cls, file: Path, state: State, initialized: FIPSCertificate = None, redo: bool = False + ) -> FIPSCertificate: + """ + Constructs FIPSCertificate object from html file. + + :param Path file: path to the html file to use for initialization + :param State state: state of the certificate + :param FIPSCertificate initialized: possibly partially initialized FIPSCertificate, defaults to None + :param bool redo: if the method should be reattempted in case of failure, defaults to False + :return FIPSCertificate: resulting `FIPSCertificate` object. + """ pairs = { "Module Name": "module_name", "Standard": "standard", @@ -466,7 +525,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris "Product URL": "product_url", } if not initialized: - items_found = FIPSCertificate.initialize_dictionary() + items_found = FIPSCertificate._initialize_dictionary() items_found["cert_id"] = int(file.stem) else: items_found = initialized.web_scan.__dict__ @@ -479,28 +538,28 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris state.txt_state = initialized.state.txt_state if redo: - items_found = FIPSCertificate.initialize_dictionary() + items_found = FIPSCertificate._initialize_dictionary() items_found["cert_id"] = int(file.stem) text = helpers.load_cert_html_file(str(file)) soup = BeautifulSoup(text, "html.parser") for div in soup.find_all("div", class_="row padrow"): - FIPSCertificate.parse_html_main(div, items_found, pairs) + FIPSCertificate._parse_html_main(div, items_found, pairs) for div in soup.find_all("div", class_="panel panel-default")[1:]: if div.find("h4", class_="panel-title").text == "Vendor": - FIPSCertificate.parse_vendor(div, items_found, file) + FIPSCertificate._parse_vendor(div, items_found, file) if div.find("h4", class_="panel-title").text == "Lab": - FIPSCertificate.parse_lab(div, items_found, file) + FIPSCertificate._parse_lab(div, items_found, file) if div.find("h4", class_="panel-title").text == "Related Files": FIPSCertificate.parse_related_files(div, items_found) if div.find("h4", class_="panel-title").text == "Validation History": - FIPSCertificate.parse_validation_dates(div, items_found) + FIPSCertificate._parse_validation_dates(div, items_found) - FIPSCertificate.normalize(items_found) + FIPSCertificate._normalize(items_found) return FIPSCertificate( items_found["cert_id"], @@ -543,7 +602,13 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris ) @staticmethod - def convert_pdf_file(tup: Tuple["FIPSCertificate", Path, Path]) -> "FIPSCertificate": + def convert_pdf_file(tup: Tuple[FIPSCertificate, Path, Path]) -> FIPSCertificate: + """ + Converts pdf file of FIPSCertificate. Staticmethod to allow for paralelization. + + :param Tuple[FIPSCertificate, Path, Path] tup: object which file will be converted, path to pdf, path to txt. + :return FIPSCertificate: the modified FIPSCertificate. + """ cert, pdf_path, txt_path = tup if not cert.state.txt_state: exit_code = helpers.convert_pdf_file(pdf_path, txt_path) @@ -565,7 +630,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris return len(text) * 0.5 <= len("".join(filter(str.isalpha, text))) @staticmethod - def find_keywords(cert: "FIPSCertificate") -> Tuple[Optional[Dict], "FIPSCertificate"]: + def find_keywords(cert: FIPSCertificate) -> Tuple[Optional[Dict], FIPSCertificate]: if not cert.state.txt_state: return None, cert @@ -580,11 +645,11 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris if config.ignore_first_page: text_to_parse = text_to_parse[text_to_parse.index(" ") :] - items_found, fips_text = FIPSCertificate.parse_cert_file(FIPSCertificate.remove_platforms(text_to_parse)) + items_found, fips_text = FIPSCertificate._parse_cert_file(FIPSCertificate._remove_platforms(text_to_parse)) save_modified_cert_file(cert.state.fragment_path.with_suffix(".fips.txt"), fips_text, unicode_error) - common_items_found, common_text = FIPSCertificate.parse_cert_file_common( + common_items_found, common_text = FIPSCertificate._parse_cert_file_common( text_to_parse, text_with_newlines, fips_common_rules ) @@ -594,7 +659,13 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris return items_found, cert @staticmethod - def match_web_algs_to_pdf(cert: "FIPSCertificate") -> int: + def match_web_algs_to_pdf(cert: FIPSCertificate) -> int: + """ + Finds algorithms in FIPSCertificate. Staticmethod to allow for parallelization. + + :param FIPSCertificate cert: cert to search for algorithms. + :return int: number of identified algorithms. + """ algs_vals = list(cert.pdf_scan.keywords["rules_fips_algorithms"].values()) table_vals = [x["Certificate"] for x in cert.pdf_scan.algorithms] tables = [x.strip() for y in table_vals for x in y] @@ -619,7 +690,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris return len(not_found) @staticmethod - def remove_platforms(text_to_parse: str) -> str: + def _remove_platforms(text_to_parse: str) -> str: pat = re.compile(r"(?:(?:modification|revision|change) history|version control)\n[\s\S]*? ", re.IGNORECASE) for match in pat.finditer(text_to_parse): text_to_parse = text_to_parse.replace(match.group(), "x" * len(match.group())) @@ -669,7 +740,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris items_found[rule_str][match][constants.TAG_MATCH_MATCHES].append([match_span[0], match_span[1]]) @staticmethod - def parse_cert_file_common( + def _parse_cert_file_common( text_to_parse: str, whole_text_with_newlines: str, search_rules: Dict ) -> Tuple[Dict[Pattern, Dict], str]: # apply all rules @@ -699,7 +770,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris return items_found_all, whole_text_with_newlines @staticmethod - def parse_cert_file(text_to_parse: str) -> Tuple[Dict[Pattern, Dict], str]: + def _parse_cert_file(text_to_parse: str) -> Tuple[Dict[Pattern, Dict], str]: # apply all rules items_found_all: Dict = {} @@ -733,7 +804,13 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris return items_found_all, text_to_parse @staticmethod - def analyze_tables(tup: Tuple["FIPSCertificate", bool]) -> Tuple[bool, "FIPSCertificate", List]: + def analyze_tables(tup: Tuple[FIPSCertificate, bool]) -> Tuple[bool, FIPSCertificate, List]: + """ + Searches for tables in pdf documents of the instance. + + :param Tuple[FIPSCertificate, bool] tup: certificate object, whether to use high precision results or approx. results + :return Tuple[bool, FIPSCertificate, List]: True on success / False otherwise, modified cert object, List of processed tables. + """ cert, precision = tup if (not precision and cert.state.tables_done) or ( precision and cert.heuristics.unmatched_algs < config.cert_threshold @@ -801,6 +878,9 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris to_pop.add(cert) def remove_algorithms(self) -> None: + """ + Removes algorithms from the certificate. + """ self.state.file_status = True if not self.pdf_scan.keywords: return @@ -829,12 +909,18 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris @staticmethod def get_compare(vendor: str) -> str: + """ + Tokenizes vendor name of the certificate. + """ vendor_split = ( vendor.replace(",", "").replace("-", " ").replace("+", " ").replace("®", "").replace("(R)", "").split() ) return vendor_split[0][:4] if len(vendor_split) > 0 else vendor def compute_heuristics_version(self) -> None: + """ + Heuristically computes the version of the product. + """ versions_for_extraction = "" if self.web_scan.module_name: versions_for_extraction += f" {self.web_scan.module_name}" -- cgit v1.3.1 From 43239b26154dc0e9afee4552b15dcd36b72965e3 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 23 May 2022 20:54:15 +0200 Subject: random docs fixes --- docs/configuration.md | 2 +- sec_certs/sample/fips.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'docs') diff --git a/docs/configuration.md b/docs/configuration.md index d5f6abda..93c926ba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,7 +6,7 @@ mystnb: --- # Configuration -The configuration is stored in yaml file `settings.yaml` at `sec_certs.config` package. These are the supported options, descriptions and default values. +The configuration is stored in yaml file `settings.yaml` at `sec_certs.config` package. Below are the supported options, descriptions and default values. ```{code-cell} python diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py index e5ea543e..b2680f0c 100644 --- a/sec_certs/sample/fips.py +++ b/sec_certs/sample/fips.py @@ -344,8 +344,8 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris """ Parses content of "Caveat" of FIPS CMVP .html file - :param current_text: text of "Caveat" - :return dictionary of all found algorithm IDs + :param str current_text: text of "Caveat" + :return Dict[str, Dict[str, int]]: dictionary of all found algorithm IDs """ ids_found: Dict[str, Dict[str, int]] = {} r_key = r"(?P\w+)?\s?(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)+(?P\d+)" @@ -364,9 +364,9 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris """ Parses table of FIPS (non) allowed algorithms - :param current_text: Contents of the table - :param in_pdf: Specifies whether the table was found in a PDF security policies file - :return List containing one element - dictionary with all parsed algorithm cert ids + :param str current_text: Contents of the table + :param bool in_pdf: Specifies whether the table was found in a PDF security policies file, defaults to False + :return List[Optional[Dict[str, List[str]]]]: List containing one element - dictionary with all parsed algorithm cert ids """ set_items = set() if in_pdf: @@ -383,9 +383,10 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris """ Parses content of
tags in FIPS .html CMVP page - :param element: text in
tags - :return list of all found algorithm IDs + :param Union[Tag, NavigableString] element: text in
tags + :return List[Dict[str, Any]]: list of all found algorithm IDs """ + found_items = [] trs = element.find_all("tr") for tr in trs: -- cgit v1.3.1 From 4f63702ac1141c6ea289f0dddf39a6ecba10aefd Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 May 2022 07:58:27 +0200 Subject: Docs: comment on cc notebook --- docs/index.md | 2 +- docs/tutorial.md | 3 --- notebooks/examples/common_criteria.ipynb | 35 +++++++++++++++++++++++++------- 3 files changed, 29 insertions(+), 11 deletions(-) delete mode 100644 docs/tutorial.md (limited to 'docs') diff --git a/docs/index.md b/docs/index.md index 61c39ec5..4760ed1c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ Welcome to the technical documentation of *sec-certs* tool for the data analysis of products certified with Common Criteria or FIPS 140 frameworks. If you're looking for general description of the tool, its use cases and capabilites, we refer you to [sec-certs homepage](https://seccerts.org/). If you are looking for more advanced knowledge, e.g. how to mine your own data, how to extend the tool, and so forth, this is the right place. -There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* +There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase some of the API that we use in the form of Jupyter notebooks. The documentation also contains some of the modules documented with `autodoc`, see *API reference*. Still, some dark corners of our codebase are not documented. To inspect the code directly, see the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* ```{button-ref} quickstart :align: center diff --git a/docs/tutorial.md b/docs/tutorial.md deleted file mode 100644 index 47e78870..00000000 --- a/docs/tutorial.md +++ /dev/null @@ -1,3 +0,0 @@ -# Tutorial - -TBA \ No newline at end of file diff --git a/notebooks/examples/common_criteria.ipynb b/notebooks/examples/common_criteria.ipynb index 83a56379..677dcd28 100644 --- a/notebooks/examples/common_criteria.ipynb +++ b/notebooks/examples/common_criteria.ipynb @@ -6,7 +6,9 @@ "source": [ "# Common Criteria example\n", "\n", - "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset and of its sample `CommonCriteriaCert`" + "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset and of its sample `CommonCriteriaCert`.\n", + "\n", + "Note that there exists a front end to this functionality at [seccerts.org/cc](https://seccerts.org/cc/). Before reinventing the wheel, it's good idea to check our web. Maybe you don't even need to run the code, but just use our web instead. " ] }, { @@ -24,7 +26,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Get fresh dataset snapshot from mirror" + "## Get fresh dataset snapshot from mirror\n", + "\n", + "There's no need to do full processing of the dataset by yourself, unless you modified `sec-certs` code. You can simply fetch the processed version from the web. \n", + "\n", + "Note, however, that you won't be able to access the `pdf` and `txt` files of the certificates. You can only get the data that we extracted from it. \n", + "\n", + "Running the whole pipeline can get you the `pdf` and `txt` data. You can see how to do that in the last cell of this notebook." ] }, { @@ -41,7 +49,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Do some basic dataset serialization" + "## Do some basic dataset serialization\n", + "\n", + "The dataset can be saved/loaded into/from `json`. Also, the dataset can be converted into a [pandas](https://pandas.pydata.org/) DataFrame. " ] }, { @@ -58,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -70,7 +80,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Simple dataset manipulation" + "## Simple dataset manipulation\n", + "\n", + "The certificates of the dataset are stored in a dictionary that maps certificate's primary key (we call it `dgst`) to the `CommonCriteriaCert` object. The primary key of the certificate is simply a hash of the attributes that make the certificate unique.\n", + "\n", + "You can iterate over the dataset which is handy when selecting some subset of certificates." ] }, { @@ -112,7 +126,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Dissect single certificate" + "## Dissect single certificate\n", + "\n", + "The `CommonCriteriaCert` is basically a data structure that holds all the data we keep about a certificate. Other classes (`CCDataset` or `model` package members) are used to transform and process the certificates. You can see all its attributes at [API docs](https://seccerts.org/docs/api/sample.html)." ] }, { @@ -134,6 +150,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Select all certificates from a dataset for which we detect at least one vulnerability.\n", "vulnerable_certs = [x for x in dset if x.heuristics.related_cves]" ] }, @@ -141,7 +158,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Serialize single certificate" + "## Serialize single certificate\n", + "\n", + "Again, a certificate can be (de)serialized into/from json. It's also possible to construct pandas `Series` from a certificate as shown below" ] }, { @@ -185,6 +204,8 @@ "source": [ "## Create new dataset and fully process it\n", "\n", + "The following piece of code roughly corresponds to `$ cc-certs all` CLI command -- it fully processes the CC pipeline. This will create a folder in current working directory where the outputs will be stored. \n", + "\n", "*Warning*: It's not good idea to run this from notebook. It may take several hours to finnish. We recommend using `from_web_latest()` or turning this into a Python script." ] }, -- cgit v1.3.1 From 1de0b977015e625db30a98f22cac4f098fbb404c Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 May 2022 07:59:34 +0200 Subject: docs conf switch ntb branch to main --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index fb66db5e..d1a81b2f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,7 +65,7 @@ html_favicon = "_static/logo_badge.svg" html_theme_options = { "repository_url": "https://github.com/crocs-muni/sec-certs", - "repository_branch": "issue/195-Docs", + "repository_branch": "main", "launch_buttons": {"binderhub_url": "https://mybinder.org"}, "use_fullscreen_button": False, } -- cgit v1.3.1