From 4863f27fab81fa0301554a5df623feb8c72fd404 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 24 Apr 2022 20:59:47 +0200 Subject: sphinx quickstart --- docs/conf.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/conf.py (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..3617f9fc --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,55 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +from typing import List + +# -- Project information ----------------------------------------------------- + +project = "sec-certs" +copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" +author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" + +# The full version, including alpha/beta/rc tags +release = "0.0.4" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions: List[str] = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "alabaster" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] -- cgit v1.3.1 From 2ac699107479e29ca4b3f1848dfa8a2bae7e4191 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 25 Apr 2022 16:48:04 +0200 Subject: link external docs sources, introduce myst --- CONTRIBUTING.md | 18 +++++++++--------- LICENSE | 2 +- README.md | 4 +++- docs/code_of_conduct.md | 4 ++++ docs/conf.py | 4 ++-- docs/contributing.md | 4 ++++ docs/index.md | 10 ++++++++++ docs/index.rst | 20 -------------------- docs/license.md | 6 ++++++ docs/readme.md | 4 ++++ requirements/dev_requirements.in | 4 +++- requirements/dev_requirements.txt | 38 +++++++++++++++++++++++++++++++++++--- 12 files changed, 81 insertions(+), 37 deletions(-) create mode 100644 docs/code_of_conduct.md create mode 100644 docs/contributing.md create mode 100644 docs/index.md delete mode 100644 docs/index.rst create mode 100644 docs/license.md create mode 100644 docs/readme.md (limited to 'docs/conf.py') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d539f656..afefc20b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -## Contributing +# Contributing You contribution is warmly welcomed. You can help by: @@ -20,9 +20,9 @@ Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/docker/Docker ### Requirements Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-tools). The main ideas are: -- List actual dependencies in `.in` files inside [requirements](requirements) folder without pinning them. -- Those dependencies are loaded into [setup.py](setup.py) file. -- Additionally, [compile.sh](requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. +- List actual dependencies in `.in` files inside [requirements](https://github.com/crocs-muni/sec-certs/blob/main/requirements) folder without pinning them. +- Those dependencies are loaded into [setup.py](https://github.com/crocs-muni/sec-certs/blob/main/setup.py) file. +- Additionally, [compile.sh](https://github.com/crocs-muni/sec-certs/blob/main/requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. - Tests, linting and Docker all run against this reproducible environment of pinned requirements. ## Branches and releases @@ -35,12 +35,12 @@ Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-too All commits shall pass the lint pipeline of the following tools: -- Mypy (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) -- Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) -- isort (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) -- Flake8 (see [.flake8](https://github.com/crocs-muni/sec-certs/blob/dev/.flake8) for settings) +- Mypy (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) +- Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) +- isort (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) +- Flake8 (see [.flake8](https://github.com/crocs-muni/sec-certs/blob/main/.flake8) for settings) -These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/dev/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you. +These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/main/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you. It should thus suffice to: diff --git a/LICENSE b/LICENSE index 5359145a..55e4ec93 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 Petr Svenda +Copyright (c) 2019-2022 Petr Svenda, Adam Janovsky, Jan Jancar, Jiri Michalik, Stanislav Bobon Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 29da0021..69a036d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# ![](docs/_static/logo.svg) +# Sec-certs + +![](docs/_static/logo.svg) Tool for analysis of security certificates and their security targets (Common Criteria, NIST FIPS140-2...). diff --git a/docs/code_of_conduct.md b/docs/code_of_conduct.md new file mode 100644 index 00000000..74e5474b --- /dev/null +++ b/docs/code_of_conduct.md @@ -0,0 +1,4 @@ +```{include} ../CODE_OF_CONDUCT.md +:relative-docs: docs/ +:relative-images: +``` diff --git a/docs/conf.py b/docs/conf.py index 3617f9fc..23392e9c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,7 @@ release = "0.0.4" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions: List[str] = [] +extensions: List[str] = ["myst_parser"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -47,7 +47,7 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "alabaster" +html_theme = "sphinx_book_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 00000000..fc1b2138 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,4 @@ +```{include} ../CONTRIBUTING.md +:relative-docs: docs/ +:relative-images: +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..4554a3b2 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,10 @@ +# Sec-certs documentation + +```{toctree} +:maxdepth: 2 +readme.md +contributing.md +code_of_conduct.md +license.md +``` + diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 9e16d741..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. sec-certs documentation master file, created by - sphinx-quickstart on Sun Apr 24 20:56:46 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to sec-certs's documentation! -===================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 00000000..4f6e8060 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,6 @@ +# License + +```{include} ../LICENSE +:relative-docs: docs/ +:relative-images: +``` diff --git a/docs/readme.md b/docs/readme.md new file mode 100644 index 00000000..2cb706b3 --- /dev/null +++ b/docs/readme.md @@ -0,0 +1,4 @@ +```{include} ../README.md +:relative-docs: docs/ +:relative-images: +``` diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in index 78f2c088..824a102f 100644 --- a/requirements/dev_requirements.in +++ b/requirements/dev_requirements.in @@ -11,4 +11,6 @@ isort flake8 pre-commit pip-tools -sphinx \ No newline at end of file +sphinx +myst-parser +sphinx-book-theme \ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 2b91f06b..97debe7f 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -4,6 +4,8 @@ attrs==21.4.0 # via pytest babel==2.10.1 # via sphinx +beautifulsoup4==4.11.1 + # via pydata-sphinx-theme black==22.3.0 # via -r dev_requirements.in certifi==2021.10.8 @@ -21,7 +23,10 @@ coverage[toml]==6.3.2 distlib==0.3.4 # via virtualenv docutils==0.17.1 - # via sphinx + # via + # myst-parser + # pydata-sphinx-theme + # sphinx filelock==3.6.0 # via virtualenv flake8==4.0.1 @@ -41,11 +46,21 @@ iniconfig==1.1.1 isort==5.10.1 # via -r dev_requirements.in jinja2==3.1.1 - # via sphinx + # via + # myst-parser + # sphinx +markdown-it-py==2.1.0 + # via + # mdit-py-plugins + # myst-parser markupsafe==2.1.1 # via jinja2 mccabe==0.6.1 # via flake8 +mdit-py-plugins==0.3.0 + # via myst-parser +mdurl==0.1.1 + # via markdown-it-py memory-profiler==0.60.0 # via pytest-monitor mypy==0.942 @@ -54,10 +69,13 @@ mypy-extensions==0.4.3 # via # black # mypy +myst-parser==0.17.2 + # via -r dev_requirements.in nodeenv==1.6.0 # via pre-commit packaging==21.3 # via + # pydata-sphinx-theme # pytest # sphinx pathspec==0.9.0 @@ -82,6 +100,8 @@ py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 +pydata-sphinx-theme==0.8.1 + # via sphinx-book-theme pyflakes==2.4.0 # via flake8 pygments==2.12.0 @@ -103,7 +123,10 @@ pytest-profiling==1.7.0 pytz==2022.1 # via babel pyyaml==6.0 - # via pre-commit + # via + # myst-parser + # pre-commit + # sphinx-book-theme requests==2.27.1 # via # pytest-monitor @@ -114,7 +137,15 @@ six==1.16.0 # virtualenv snowballstemmer==2.2.0 # via sphinx +soupsieve==2.3.2.post1 + # via beautifulsoup4 sphinx==4.5.0 + # via + # -r dev_requirements.in + # myst-parser + # pydata-sphinx-theme + # sphinx-book-theme +sphinx-book-theme==0.3.2 # via -r dev_requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx @@ -149,6 +180,7 @@ typing-extensions==4.1.1 # via # black # mypy + # myst-parser urllib3==1.26.9 # via requests virtualenv==20.14.0 -- cgit v1.3.1 From 5ede8f4dc9b3322b04480b3f9caa99c382fa1042 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 25 Apr 2022 18:11:07 +0200 Subject: add notebooks to docs --- docs/conf.py | 3 +- docs/index.md | 13 +- docs/notebooks | 1 + notebooks/examples/dataset.ipynb | 65 +++++++++ requirements/dev_requirements.in | 2 +- requirements/dev_requirements.txt | 274 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 340 insertions(+), 18 deletions(-) create mode 120000 docs/notebooks create mode 100644 notebooks/examples/dataset.ipynb (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py index 23392e9c..092fe3c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,6 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) -from typing import List # -- Project information ----------------------------------------------------- @@ -31,7 +30,7 @@ release = "0.0.4" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions: List[str] = ["myst_parser"] +extensions = ["myst_nb"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/index.md b/docs/index.md index 4554a3b2..d8f04edf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,10 +1,19 @@ # Sec-certs documentation +## Notebooks + +```{toctree} +:caption: Notebook examples +notebooks/dataset.ipynb +``` + +## GitHub artifacts + ```{toctree} :maxdepth: 2 +:caption: GitHub artifacts readme.md contributing.md code_of_conduct.md license.md -``` - +``` \ No newline at end of file diff --git a/docs/notebooks b/docs/notebooks new file mode 120000 index 00000000..b50be0f4 --- /dev/null +++ b/docs/notebooks @@ -0,0 +1 @@ +./../notebooks/examples \ No newline at end of file diff --git a/notebooks/examples/dataset.ipynb b/notebooks/examples/dataset.ipynb new file mode 100644 index 00000000..7f9287aa --- /dev/null +++ b/notebooks/examples/dataset.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Dataset class examples\n", + "\n", + "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sec_certs.dataset.common_criteria import CCDataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4985\n" + ] + } + ], + "source": [ + "dset = CCDataset.from_web_latest()\n", + "print(len(dset))" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" + }, + "kernelspec": { + "display_name": "Python 3.8.1 ('certsvenv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in index 824a102f..baed56a5 100644 --- a/requirements/dev_requirements.in +++ b/requirements/dev_requirements.in @@ -12,5 +12,5 @@ flake8 pre-commit pip-tools sphinx -myst-parser +myst-nb sphinx-book-theme \ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 97debe7f..9c652d29 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -1,15 +1,41 @@ alabaster==0.7.12 # via sphinx +anyio==3.5.0 + # via jupyter-server +appnope==0.1.3 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via + # jupyter-server + # notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asttokens==2.0.5 + # via stack-data attrs==21.4.0 - # via pytest + # via + # jsonschema + # jupyter-cache + # markdown-it-py + # pytest babel==2.10.1 # via sphinx +backcall==0.2.0 + # via ipython beautifulsoup4==4.11.1 - # via pydata-sphinx-theme + # via + # nbconvert + # pydata-sphinx-theme black==22.3.0 # via -r dev_requirements.in +bleach==5.0.0 + # via nbconvert certifi==2021.10.8 # via requests +cffi==1.15.0 + # via argon2-cffi-bindings cfgv==3.3.1 # via pre-commit charset-normalizer==2.0.12 @@ -18,70 +44,203 @@ click==8.1.2 # via # black # pip-tools +colorama==0.4.4 + # via nbdime coverage[toml]==6.3.2 # via pytest-cov +debugpy==1.6.0 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert distlib==0.3.4 # via virtualenv docutils==0.17.1 # via + # myst-nb # myst-parser # pydata-sphinx-theme # sphinx + # sphinx-togglebutton +entrypoints==0.4 + # via + # jupyter-client + # nbconvert +executing==0.8.3 + # via stack-data +fastjsonschema==2.15.3 + # via nbformat filelock==3.6.0 # via virtualenv flake8==4.0.1 # via -r dev_requirements.in +gitdb==4.0.9 + # via gitpython +gitpython==3.1.27 + # via nbdime gprof2dot==2021.2.21 # via pytest-profiling +greenlet==1.1.2 + # via sqlalchemy identify==2.4.12 # via pre-commit idna==3.3 - # via requests + # via + # anyio + # requests imagesize==1.3.0 # via sphinx importlib-metadata==4.11.3 - # via sphinx + # via + # myst-nb + # sphinx +importlib-resources==5.7.1 + # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.13.0 + # via + # ipywidgets + # notebook +ipython==8.2.0 + # via + # ipykernel + # ipywidgets + # jupyter-sphinx + # myst-nb +ipython-genutils==0.2.0 + # via + # ipywidgets + # notebook +ipywidgets==7.7.0 + # via + # jupyter-sphinx + # myst-nb isort==5.10.1 # via -r dev_requirements.in +jedi==0.18.1 + # via ipython jinja2==3.1.1 # via + # jupyter-server # myst-parser + # nbconvert + # nbdime + # notebook # sphinx -markdown-it-py==2.1.0 +jsonschema==4.4.0 + # via nbformat +jupyter-cache==0.4.3 + # via myst-nb +jupyter-client==7.2.2 + # via + # ipykernel + # jupyter-server + # nbclient + # notebook +jupyter-core==4.10.0 + # via + # jupyter-client + # jupyter-server + # nbconvert + # nbformat + # notebook +jupyter-server==1.16.0 + # via + # jupyter-server-mathjax + # nbdime +jupyter-server-mathjax==0.2.5 + # via nbdime +jupyter-sphinx==0.3.2 + # via myst-nb +jupyterlab-pygments==0.2.2 + # via nbconvert +jupyterlab-widgets==1.1.0 + # via ipywidgets +markdown-it-py==1.1.0 # via # mdit-py-plugins # myst-parser markupsafe==2.1.1 - # via jinja2 + # via + # jinja2 + # nbconvert +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 -mdit-py-plugins==0.3.0 +mdit-py-plugins==0.2.8 # via myst-parser -mdurl==0.1.1 - # via markdown-it-py memory-profiler==0.60.0 # via pytest-monitor +mistune==0.8.4 + # via nbconvert mypy==0.942 # via -r dev_requirements.in mypy-extensions==0.4.3 # via # black # mypy -myst-parser==0.17.2 +myst-nb==0.13.2 # via -r dev_requirements.in +myst-parser==0.15.2 + # via myst-nb +nbclient==0.5.13 + # via + # jupyter-cache + # nbconvert +nbconvert==6.5.0 + # via + # jupyter-server + # jupyter-sphinx + # myst-nb + # notebook +nbdime==3.1.1 + # via jupyter-cache +nbformat==5.3.0 + # via + # ipywidgets + # jupyter-cache + # jupyter-server + # jupyter-sphinx + # myst-nb + # nbclient + # nbconvert + # nbdime + # notebook +nest-asyncio==1.5.5 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit +notebook==6.4.11 + # via widgetsnbextension packaging==21.3 # via + # ipykernel + # jupyter-server + # nbconvert # pydata-sphinx-theme # pytest # sphinx +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.5.1 # via -r dev_requirements.in platformdirs==2.5.1 @@ -92,22 +251,43 @@ pluggy==1.0.0 # via pytest pre-commit==2.18.1 # via -r dev_requirements.in +prometheus-client==0.14.1 + # via + # jupyter-server + # notebook +prompt-toolkit==3.0.29 + # via ipython psutil==5.9.0 # via + # ipykernel # memory-profiler # pytest-monitor +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 +pycparser==2.21 + # via cffi pydata-sphinx-theme==0.8.1 # via sphinx-book-theme pyflakes==2.4.0 # via flake8 pygments==2.12.0 - # via sphinx + # via + # ipython + # nbconvert + # nbdime + # sphinx pyparsing==3.0.7 # via packaging +pyrsistent==0.18.1 + # via jsonschema pytest==7.1.1 # via # -r dev_requirements.in @@ -120,21 +300,41 @@ pytest-monitor==1.6.3 # via -r dev_requirements.in pytest-profiling==1.7.0 # via -r dev_requirements.in +python-dateutil==2.8.2 + # via jupyter-client pytz==2022.1 # via babel pyyaml==6.0 # via + # myst-nb # myst-parser # pre-commit # sphinx-book-theme +pyzmq==22.3.0 + # via + # jupyter-client + # jupyter-server + # notebook requests==2.27.1 # via + # nbdime # pytest-monitor # sphinx +send2trash==1.8.0 + # via + # jupyter-server + # notebook six==1.16.0 # via + # asttokens + # bleach # pytest-profiling + # python-dateutil # virtualenv +smmap==5.0.0 + # via gitdb +sniffio==1.2.0 + # via anyio snowballstemmer==2.2.0 # via sphinx soupsieve==2.3.2.post1 @@ -142,11 +342,16 @@ soupsieve==2.3.2.post1 sphinx==4.5.0 # via # -r dev_requirements.in + # jupyter-sphinx + # myst-nb # myst-parser # pydata-sphinx-theme # sphinx-book-theme + # sphinx-togglebutton sphinx-book-theme==0.3.2 # via -r dev_requirements.in +sphinx-togglebutton==0.3.1 + # via myst-nb sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -159,6 +364,16 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx +sqlalchemy==1.4.35 + # via jupyter-cache +stack-data==0.2.0 + # via ipython +terminado==0.13.3 + # via + # jupyter-server + # notebook +tinycss2==1.1.1 + # via nbconvert toml==0.10.2 # via pre-commit tomli==2.0.1 @@ -168,6 +383,27 @@ tomli==2.0.1 # mypy # pep517 # pytest +tornado==6.1 + # via + # ipykernel + # jupyter-client + # jupyter-server + # nbdime + # notebook + # terminado +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-server + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook types-python-dateutil==2.8.10 # via -r dev_requirements.in types-pyyaml==6.0.5 @@ -180,17 +416,29 @@ typing-extensions==4.1.1 # via # black # mypy - # myst-parser urllib3==1.26.9 # via requests virtualenv==20.14.0 # via pre-commit +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.3.2 + # via jupyter-server wheel==0.37.1 # via # pip-tools # pytest-monitor + # sphinx-togglebutton +widgetsnbextension==3.6.0 + # via ipywidgets zipp==3.8.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip -- cgit v1.3.1 From 2ada5810445ff7122c585752c8d01e712a395394 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 08:21:14 +0200 Subject: . --- .github/workflows/docs.yml | 16 ++++++++++++---- .github/workflows/draft_release.yml | 21 ++++----------------- .gitignore | 2 ++ CONTRIBUTING.md | 23 +++++++++++++++++++---- VERSION | 1 - docs/conf.py | 8 ++++---- pyproject.toml | 26 +++++++++++++++----------- requirements/requirements.in | 1 + requirements/requirements.txt | 8 ++++++++ setup.py | 2 -- 10 files changed, 65 insertions(+), 43 deletions(-) delete mode 100644 VERSION (limited to 'docs/conf.py') diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4971a49d..056153c8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,8 +6,16 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - uses: ammaraskar/sphinx-action@master + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 with: - pre-build-command: python -m pip install -r requirements/dev_requirements.txt - docs-folder: "docs/" + python-version: 3.8 + - name: Install sec-certs and deps + run: | + pip install -r requirements/requirements.txt + pip install -r requirements/dev_requirements.txt + pip install -e . + - name: Build docs + run: | + cd docs + make html diff --git a/.github/workflows/draft_release.yml b/.github/workflows/draft_release.yml index 4490c0d0..95e2cadc 100644 --- a/.github/workflows/draft_release.yml +++ b/.github/workflows/draft_release.yml @@ -2,25 +2,12 @@ name: Create release draft on: push: - # branches: - # - "main" - # tags: - # - "*.*.*" + branches: + - "main" + tags: + - "*.*.*" jobs: - check-version: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Read VERSION file - run: echo "ACTUAL_VERSION=$(cat VERSION)" >> $GITHUB_ENV - - name: Check versions - run: | - if ((${{env.PACKAGE_NUMBER}} != ${{github.ref_name}})) ; then - echo "Tag does not match version number in VERSION file" - exit 1 - fi draft-release: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 3176de71..1d2da538 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ # results folder results*/ +# version file by setuptools-scm +sec_certs/_version.py # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afefc20b..80585c2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,11 +25,26 @@ Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-too - Additionally, [compile.sh](https://github.com/crocs-muni/sec-certs/blob/main/requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. - Tests, linting and Docker all run against this reproducible environment of pinned requirements. -## Branches and releases +## Branches + +`main` is the default branch against which all pull requests are to be made. This branch is not neccessarily stable, only the releases are. + +## Releases and version strings + +- On each revision pushed onto `main` that has `*.*.*` tag, a draft release is created with prepared changelog (this step can be skipped and the Release created right from the GitHub GUI). +- This draft release is to be published manually by the maintainer. +- Version string is not indexed in `git` but can be retreived maintained by `setuptools-scm` from git tags instead. +- `setuptools-scm` will automatically, upon editable/real install of a package, infer its version and write it to `sec_certs/_version.py`. This file is not indexed as well. See more at [setuptools-scm GitHub](https://github.com/pypa/setuptools_scm) +- On publishing a release, the tool is automatically published to [PyPi](https://pypi.org/project/sec-certs/) and [DockerHub](https://hub.docker.com/repository/docker/seccerts/sec-certs). + +Note on single-sourcing the package version: More can be read [here](https://packaging.python.org/en/latest/guides/single-sourcing-package-version/). The downside of our approach is that `.git` folder and editable/real install is needed to infer the version of the package. Releases can be infered without installing the project. + +### Currently, the release process is as follows + +1. (skip this optionally) Tag a revision with `*.*.*` tag -- this will create a draft release in GitHub. +2. Modify changelog and publish the release (or create it from scratch with new tag). +3. This will automatically update PyPi and DockerHub packages. -- `main` is the default branch against which all pull requests are to be made. This branch is not neccessarily stable, only the releases are. -- [Releases](https://github.com/crocs-muni/sec-certs/releases) are prepared manually by tagging revisions of `main` branch with `x.y.z` according to [semantic versioning](https://semver.org). -- Upon each release, the tool is automatically published to [PyPi](https://pypi.org/project/sec-certs/) and [DockerHub](https://hub.docker.com/repository/docker/seccerts/sec-certs). ## Quality assurance diff --git a/VERSION b/VERSION deleted file mode 100644 index 1750564f..00000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.6 diff --git a/docs/conf.py b/docs/conf.py index 092fe3c4..6eb14738 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,8 +12,8 @@ # # import os # import sys -# sys.path.insert(0, os.path.abspath('.')) - +# sys.path.insert(0, os.path.abspath(".")) +from importlib.metadata import version # -- Project information ----------------------------------------------------- @@ -22,8 +22,8 @@ copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanis author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" # The full version, including alpha/beta/rc tags -release = "0.0.4" - +# Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker +release = version("sec-certs") # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 7ac509d6..6bae9df8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ +[build-system] +requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] + +[tool.setuptools_scm] +write_to = "sec_certs/_version.py" + [tool.black] line-length = 120 exclude = ''' @@ -16,21 +22,19 @@ exclude = ''' ''' [tool.isort] -multi_line_output=3 -include_trailing_comma=true -force_grid_wrap=0 -use_parentheses=true -ensure_newline_before_comments=true -line_length=120 -skip=["certsvenv", "build"] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 120 +skip = ["certsvenv", "build"] [tool.mypy] plugins = ["numpy.typing.mypy_plugin"] ignore_missing_imports = true -exclude="build/" +exclude = "build/" [tool.pytest.ini_options] -markers = [ - "slow: marks tests as slow (deselect with '-m \"not slow\"')" -] +markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"] addopts = "--cov sec_certs" diff --git a/requirements/requirements.in b/requirements/requirements.in index f7fc03a3..4b67a179 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -19,3 +19,4 @@ requests scikit-learn tabula-py tqdm +setuptools-scm diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f72cd3f3..a66833ac 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -50,6 +50,7 @@ packaging==21.3 # via # matplotlib # pikepdf + # setuptools-scm pandas==1.4.2 # via # -r requirements.in @@ -88,6 +89,8 @@ scikit-learn==1.0.2 # via -r requirements.in scipy==1.8.0 # via scikit-learn +setuptools-scm==6.4.2 + # via -r requirements.in six==1.16.0 # via # html5lib @@ -98,6 +101,8 @@ tabula-py==2.3.0 # via -r requirements.in threadpoolctl==3.1.0 # via scikit-learn +tomli==2.0.1 + # via setuptools-scm tqdm==4.64.0 # via -r requirements.in urllib3==1.26.9 @@ -106,3 +111,6 @@ webencodings==0.5.1 # via html5lib zipp==3.8.0 # via importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.py b/setup.py index 802dc9c9..79c76d9b 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,6 @@ setup( name="sec-certs", author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky, Jiri Michalik", author_email="svenda@fi.muni.cz", - version_config=True, - setup_requires=["setuptools-git-versioning"], packages=find_packages(), license="MIT", description="Tool for analysis of security certificates", -- cgit v1.3.1 From d04b507ab033bc0d53dc118bc1ac7772967d0b83 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 26 Apr 2022 08:30:49 +0200 Subject: non-conflicting version variable name --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py index 6eb14738..b821cd0a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ # import os # import sys # sys.path.insert(0, os.path.abspath(".")) -from importlib.metadata import version +from importlib.metadata import version as get_version # -- Project information ----------------------------------------------------- @@ -23,7 +23,7 @@ author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon # The full version, including alpha/beta/rc tags # Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker -release = version("sec-certs") +release = get_version("sec-certs") # -- General configuration --------------------------------------------------- -- cgit v1.3.1 From e05e66bc8c9230e704e8aa4332a92727a18c68dc Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 17:06:37 +0200 Subject: docs: add favicon and logo --- docs/conf.py | 3 +++ docs/index.md | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py index b821cd0a..701f4924 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,3 +52,6 @@ html_theme = "sphinx_book_theme" # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] + +html_logo = "_static/logo.svg" +html_favicon = "_static/logo_badge.svg" diff --git a/docs/index.md b/docs/index.md index 5b390b63..dd7c9e53 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,7 +1,5 @@ # Sec-certs documentation -![](_static/logo.svg) - There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* ```{toctree} -- cgit v1.3.1 From d4760ca46ace74c48e27a383058cc8219c4ab2a5 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 17:30:21 +0200 Subject: docs: don't show whole version, add navigation --- docs/conf.py | 3 +-- docs/index.md | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py index 701f4924..c7f4c6ff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,9 +21,8 @@ project = "sec-certs" copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" -# The full version, including alpha/beta/rc tags # Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker -release = get_version("sec-certs") +release = ".".join(get_version("sec-certs").split(".")[:3]) # -- General configuration --------------------------------------------------- diff --git a/docs/index.md b/docs/index.md index dd7c9e53..084f7d08 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,6 +2,14 @@ There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts* +```{toctree} +:hidden: +:caption: Navigation +Seccerts homepage +Seccerts docs +GitHub repo +``` + ```{toctree} :caption: User's guide installation.md -- cgit v1.3.1 From 7ff47777d6865b826cf0613e531f6ebece635bd5 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 20:25:53 +0200 Subject: add mybinder launch buttons for docs --- docs/conf.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py index c7f4c6ff..e6b76999 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,3 +54,12 @@ html_static_path = ["_static"] html_logo = "_static/logo.svg" html_favicon = "_static/logo_badge.svg" + +html_theme_options = { + "repository_url": "https://github.com/crocs-muni/sec-certs", + "repository_branch": "issue/195-Docs", + "launch_buttons": {"binderhub_url": "https://mybinder.org"}, +} + +# Don't exeute notbooks +nb_execution_mode = "off" -- cgit v1.3.1 From 766164f7534e4d41a1668cc553c51c0ad3c0630f Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 1 May 2022 21:57:52 +0200 Subject: update notes --- CONTRIBUTING.md | 2 +- README.md | 2 +- cc_cli.py | 4 +- docs/conf.py | 12 +++--- docs/index.md | 3 +- docs/quickstart.md | 31 ++++++++++++++- fips_cli.py | 4 +- notebooks/examples/common_criteria.ipynb | 65 ++++++++++++++++++++++++++++++++ notebooks/examples/dataset.ipynb | 65 -------------------------------- notebooks/examples/fips.ipynb | 65 ++++++++++++++++++++++++++++++++ 10 files changed, 177 insertions(+), 76 deletions(-) create mode 100644 notebooks/examples/common_criteria.ipynb delete mode 100644 notebooks/examples/dataset.ipynb create mode 100644 notebooks/examples/fips.ipynb (limited to 'docs/conf.py') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 542a39da..803cb340 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,7 @@ You contribution is warmly welcomed. You can help by: ## Dependencies -For complete list of system dependencies, see [docs/installation](seccerts.org/docs/installation). +For complete list of system dependencies, see [docs/installation](https://seccerts.org/docs/installation.html). ### Requirements diff --git a/README.md b/README.md index 6464900a..c4b19802 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This project is developed by the [Centre for Research On Cryptography and Securi ## Installation -Use Docker with `docker pull seccerts/sec-certs` or just `pip install -U sec-certs`. For more elaborate description, see [docs](seccerts.org/docs/installation) +Use Docker with `docker pull seccerts/sec-certs` or just `pip install -U sec-certs`. For more elaborate description, see [docs](https://seccerts.org/docs/installation.html) ## Usage (CC) diff --git a/cc_cli.py b/cc_cli.py index 11e1a40e..a9c20fc1 100755 --- a/cc_cli.py +++ b/cc_cli.py @@ -23,8 +23,10 @@ logger = logging.getLogger(__name__) @click.option( "-o", "--output", - type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), help="Path where the output of the experiment will be stored. May overwrite existing content.", + default=Path("./cc_dset/"), + show_default=True, ) @click.option( "-c", diff --git a/docs/conf.py b/docs/conf.py index e6b76999..384e6569 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,8 +18,8 @@ from importlib.metadata import version as get_version # -- Project information ----------------------------------------------------- project = "sec-certs" -copyright = "2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" -author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" +copyright = "2020-2022, Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon." +# author = "Adam Janovsky, Petr Svenda, Jan Jancar, Jiri Michalik, Stanislav Bobon" # Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker release = ".".join(get_version("sec-certs").split(".")[:3]) @@ -31,6 +31,9 @@ release = ".".join(get_version("sec-certs").split(".")[:3]) # ones. extensions = ["myst_nb"] +# Don't exeute notbooks +nb_execution_mode = "off" + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -39,7 +42,6 @@ templates_path = ["_templates"] # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -59,7 +61,7 @@ html_theme_options = { "repository_url": "https://github.com/crocs-muni/sec-certs", "repository_branch": "issue/195-Docs", "launch_buttons": {"binderhub_url": "https://mybinder.org"}, + "use_fullscreen_button": False, } -# Don't exeute notbooks -nb_execution_mode = "off" +myst_heading_anchors = 3 diff --git a/docs/index.md b/docs/index.md index 371b917c..8cd9c8ef 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,8 @@ tutorial.md ```{toctree} :caption: Notebook examples -notebooks/examples/dataset.ipynb +notebooks/examples/common_criteria.ipynb +notebooks/examples/fips.ipynb ``` ```{admonition} Launch notebooks in MyBinder diff --git a/docs/quickstart.md b/docs/quickstart.md index 74285025..48f7f1ce 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,4 +1,33 @@ # Quickstart 1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)). -2. Use either `cc_cli.py` or `fips_cli.py` for full-fledged processing of the certificate pipeline, or explore the [dataset notebook](notebooks/dataset.ipynb). +2. Use +```python +dset = CCDataset.from_web_latest() +``` + +(Common Criteria) or + +```python +dset = FIPSDataset.from_web_latest() +``` + +(FIPS 140) to obtain freshly processed datasets from [seccerts.org](https://seccerts.org). + +```{hint} +You can work with those with the help of the [common criteria notebook](notebooks/examples/common_criteria.ipynb) or [fips notebook](notebooks/examples/fips.ipynb) and even launch them in MyBinder without installing anything. Just use the 🚀 icon (top-right corner). +``` + +If you insist on processing the whole certificates pipeline, make sure that you installed all [dependencies](installation.md#dependencies). Then, run + +```bash +cc-certs all +``` + +for Common Criteria processing, or + +```bash +fips-certs all +``` + +for FIPS 140 processing. This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`. \ No newline at end of file diff --git a/fips_cli.py b/fips_cli.py index 149d2f81..9ebbf6f1 100755 --- a/fips_cli.py +++ b/fips_cli.py @@ -26,8 +26,10 @@ logger = logging.getLogger(__name__) @click.option( "-o", "--output", - type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), help="Path to the directory where the output of the 'build' or 'new-run' actions will be stored.", + default=Path("./fips_dset/"), + show_default=True, ) @click.option( "-c", diff --git a/notebooks/examples/common_criteria.ipynb b/notebooks/examples/common_criteria.ipynb new file mode 100644 index 00000000..4aab6c61 --- /dev/null +++ b/notebooks/examples/common_criteria.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Common Criteria dataset class\n", + "\n", + "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sec_certs.dataset.common_criteria import CCDataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4985\n" + ] + } + ], + "source": [ + "dset = CCDataset.from_web_latest()\n", + "print(len(dset))" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" + }, + "kernelspec": { + "display_name": "Python 3.8.1 ('certsvenv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/examples/dataset.ipynb b/notebooks/examples/dataset.ipynb deleted file mode 100644 index 7f9287aa..00000000 --- a/notebooks/examples/dataset.ipynb +++ /dev/null @@ -1,65 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Dataset class examples\n", - "\n", - "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from sec_certs.dataset.common_criteria import CCDataset" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4985\n" - ] - } - ], - "source": [ - "dset = CCDataset.from_web_latest()\n", - "print(len(dset))" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" - }, - "kernelspec": { - "display_name": "Python 3.8.1 ('certsvenv': venv)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.1" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/examples/fips.ipynb b/notebooks/examples/fips.ipynb new file mode 100644 index 00000000..ab20ee93 --- /dev/null +++ b/notebooks/examples/fips.ipynb @@ -0,0 +1,65 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FIPS Dataset class\n", + "\n", + "This notebook illustrates basic functionality with the `FIPSDataset` class that holds FIPS 140 dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sec_certs.dataset.fips import FIPSDataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4985\n" + ] + } + ], + "source": [ + "dset = FIPSDataset.from_web_latest()\n", + "print(len(dset))" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d" + }, + "kernelspec": { + "display_name": "Python 3.8.1 ('certsvenv': venv)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} -- cgit v1.3.1 From 67c268b727a138efdcc51b1c1ecbaf30ab11b182 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 May 2022 12:18:52 +0200 Subject: Docs: autodoc, model package docs --- CONTRIBUTING.md | 9 ++++++++- docs/api/model.md | 20 +++++++++++++++++++ docs/conf.py | 2 +- docs/index.md | 5 +++++ sec_certs/model/sar_transformer.py | 39 +++++++++++++++++++++++++++++--------- 5 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 docs/api/model.md (limited to 'docs/conf.py') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 803cb340..11cd09f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,4 +64,11 @@ To ivoke the tools manually, you can, in the repository root, use: - Mypy: `mypy .` - Black: `black --check .` (without the flag to reformat) - isort: `isort --check-only .` (without the flag to actually fix the issue) -- Flake8: `flake8 .` \ No newline at end of file +- Flake8: `flake8 .` + +## Documentation + +Every public method of a module that can be leveraged as an API by user should be documented. The docstrng style should +be `sphinx-oneline`. + +The documentation is built using `sphinx` with `mnyst` extension that allows for markdown files. Folder `notebooks/examples` is symbolically linked to `/docs` and its contents will be automatically parsed. These notebooks are supposed to be runnable from Binder. \ No newline at end of file diff --git a/docs/api/model.md b/docs/api/model.md new file mode 100644 index 00000000..372815c7 --- /dev/null +++ b/docs/api/model.md @@ -0,0 +1,20 @@ +# Model package + +## CPEClassifier + +```{eval-rst} +.. currentmodule:: sec_certs.model +.. autoclass:: CPEClassifier + :members: +``` + +## SARTranformer + +```{eval-rst} +.. currentmodule:: sec_certs.model +.. autoclass:: SARTransformer + :members: +``` + +## DependencyFinder + diff --git a/docs/conf.py b/docs/conf.py index 384e6569..001c209a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ release = ".".join(get_version("sec-certs").split(".")[:3]) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["myst_nb"] +extensions = ["myst_nb", "sphinx.ext.autodoc"] # Don't exeute notbooks nb_execution_mode = "off" diff --git a/docs/index.md b/docs/index.md index d7ee1808..8e0883dc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,6 +26,11 @@ notebooks/examples/common_criteria.ipynb notebooks/examples/fips.ipynb ``` +```{toctree} +:caption: API +api/model.md +``` + ```{admonition} Launch notebooks in MyBinder Each of the notebooks can be launched interactively in MyBinder by clicking on 🚀 icon (top-right corner). ``` diff --git a/sec_certs/model/sar_transformer.py b/sec_certs/model/sar_transformer.py index ee99b279..58f96c0b 100644 --- a/sec_certs/model/sar_transformer.py +++ b/sec_certs/model/sar_transformer.py @@ -19,17 +19,38 @@ class SARTransformer(BaseEstimator, TransformerMixin): """ def fit(self, certificates: Iterable[CommonCriteriaCert]) -> SARTransformer: + """ + Just returns self, no fitting needed + + :param Iterable[CommonCriteriaCert] certificates: Unused parameter + :return SARTransformer: return self + """ return self def transform(self, certificates: Iterable[CommonCriteriaCert]) -> List[Optional[Set[SAR]]]: + """ + Just a wrapper around transform_single_cert() called on an iterable of CommonCriteriaCert. + + :param Iterable[CommonCriteriaCert] certificates: Iterable of CommonCriteriaCert objects to perform the extraction on. + :return List[Optional[Set[SAR]]]: Returns List of results from transform_single_cert(). + """ return [self.transform_single_cert(cert) for cert in certificates] def transform_single_cert(self, cert: CommonCriteriaCert) -> Optional[Set[SAR]]: - sec_level_candidates, st_candidates, report_candidates = self.collect_sar_candidates_from_all_sources(cert) - return self.resolve_candidate_conflicts(sec_level_candidates, st_candidates, report_candidates, cert.dgst) + """ + Given CommonCriteriaCert, will transform SAR keywords extracted from txt files + into a set of SAR objects. Also handles extractin of correct SAR levels, duplicities and filtering. + Uses three sources: CSV scan, security target, and certification report. + The caller should assure that the certificates have the keywords extracted. + + :param CommonCriteriaCert cert: Certificate to extract SARs from + :return Optional[Set[SAR]]: Set of SARs, None if none were identified. + """ + sec_level_candidates, st_candidates, report_candidates = self._collect_sar_candidates_from_all_sources(cert) + return self._resolve_candidate_conflicts(sec_level_candidates, st_candidates, report_candidates, cert.dgst) @staticmethod - def collect_sar_candidates_from_all_sources(cert: CommonCriteriaCert) -> Tuple[Set[SAR], Set[SAR], Set[SAR]]: + def _collect_sar_candidates_from_all_sources(cert: CommonCriteriaCert) -> Tuple[Set[SAR], Set[SAR], Set[SAR]]: """ Parses SARs from three distinct sources and returns the results as a three tuple: - Security level from CSV scan @@ -43,24 +64,24 @@ class SARTransformer(BaseEstimator, TransformerMixin): def report_keywords_may_have_sars(sample: CommonCriteriaCert): return sample.pdf_data.report_keywords and SAR_DICT_KEY in sample.pdf_data.report_keywords - sec_level_sars = SARTransformer.parse_sars_from_security_level_list(cert.security_level) + sec_level_sars = SARTransformer._parse_sars_from_security_level_list(cert.security_level) if st_keywords_may_have_sars(cert): st_dict: Dict = cast(Dict, cert.pdf_data.st_keywords) - st_sars = SARTransformer.parse_sar_dict(st_dict[SAR_DICT_KEY], cert.dgst) + st_sars = SARTransformer._parse_sar_dict(st_dict[SAR_DICT_KEY], cert.dgst) else: st_sars = set() if report_keywords_may_have_sars(cert): report_dict: Dict = cast(Dict, cert.pdf_data.report_keywords) - report_sars = SARTransformer.parse_sar_dict(report_dict[SAR_DICT_KEY], cert.dgst) + report_sars = SARTransformer._parse_sar_dict(report_dict[SAR_DICT_KEY], cert.dgst) else: report_sars = set() return sec_level_sars, st_sars, report_sars @staticmethod - def resolve_candidate_conflicts( + def _resolve_candidate_conflicts( sec_level_candidates: Set[SAR], st_candidates: Set[SAR], report_candidates: Set[SAR], cert_dgst: str ) -> Optional[Set[SAR]]: final_candidates: Dict[str, SAR] = {x.family: x for x in sec_level_candidates} @@ -94,7 +115,7 @@ class SARTransformer(BaseEstimator, TransformerMixin): return set(final_candidates.values()) if final_candidates else None @staticmethod - def parse_sar_dict(dct: Dict[str, int], dgst: str) -> Set[SAR]: + def _parse_sar_dict(dct: Dict[str, int], dgst: str) -> Set[SAR]: """ Accepts st_keywords or report_keywords dictionary. Will reconstruct SAR objects from it. Each SAR family can appear multiple times in the dictionary (due to conflicts) with different levels. Iterated item will replace @@ -126,7 +147,7 @@ class SARTransformer(BaseEstimator, TransformerMixin): return {x[0] for x in sars.values()} if sars else set() @staticmethod - def parse_sars_from_security_level_list(lst: Iterable[str]) -> Set[SAR]: + def _parse_sars_from_security_level_list(lst: Iterable[str]) -> Set[SAR]: sars = set() for element in lst: try: -- cgit v1.3.1 From 4d7078d81f38d5ab4e31525ff8094be449e3b3b8 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 May 2022 15:32:11 +0200 Subject: docs tabs and copybuttons --- docs/conf.py | 8 +++++++- requirements/dev_requirements.in | 4 +++- requirements/dev_requirements.txt | 6 ++++++ 3 files changed, 16 insertions(+), 2 deletions(-) (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py index 001c209a..fb66db5e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ release = ".".join(get_version("sec-certs").split(".")[:3]) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["myst_nb", "sphinx.ext.autodoc"] +extensions = ["myst_nb", "sphinx.ext.autodoc", "sphinx_design", "sphinx_copybutton"] # Don't exeute notbooks nb_execution_mode = "off" @@ -37,6 +37,12 @@ nb_execution_mode = "off" # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] +# Don't show typehints in autodoc files +autodoc_typehints = "none" + +# This is recommended by sphinx_design extension +myst_enable_extensions = ["colon_fence"] + # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in index 635c62d1..7f6d9712 100644 --- a/requirements/dev_requirements.in +++ b/requirements/dev_requirements.in @@ -13,4 +13,6 @@ pre-commit pip-tools sphinx myst-nb>=0.14 -sphinx-book-theme \ No newline at end of file +sphinx-book-theme +sphinx-design +sphinx-copybutton \ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 125c77e5..3ea4c3af 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -240,9 +240,15 @@ sphinx==4.5.0 # myst-parser # pydata-sphinx-theme # sphinx-book-theme + # sphinx-copybutton + # sphinx-design # sphinx-togglebutton sphinx-book-theme==0.3.2 # via -r dev_requirements.in +sphinx-copybutton==0.5.0 + # via -r dev_requirements.in +sphinx-design==0.1.0 + # via -r dev_requirements.in sphinx-togglebutton==0.3.1 # via myst-nb sphinxcontrib-applehelp==1.0.2 -- cgit v1.3.1 From 1de0b977015e625db30a98f22cac4f098fbb404c Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 May 2022 07:59:34 +0200 Subject: docs conf switch ntb branch to main --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docs/conf.py') diff --git a/docs/conf.py b/docs/conf.py index fb66db5e..d1a81b2f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,7 +65,7 @@ html_favicon = "_static/logo_badge.svg" html_theme_options = { "repository_url": "https://github.com/crocs-muni/sec-certs", - "repository_branch": "issue/195-Docs", + "repository_branch": "main", "launch_buttons": {"binderhub_url": "https://mybinder.org"}, "use_fullscreen_button": False, } -- cgit v1.3.1