MSTICPy CI build and check #2224
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a variety of Python versions | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: MSTICPy CI build and check | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main, release/*] | |
| schedule: | |
| - cron: "0 0 * * 0,2,4" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| steps: | |
| # Print out details about the run | |
| - name: Dump GitHub context | |
| env: | |
| GITHUB_CONTEXT: ${{ toJSON(github) }} | |
| run: echo "$GITHUB_CONTEXT" | |
| - name: Dump job context | |
| env: | |
| JOB_CONTEXT: ${{ toJSON(job) }} | |
| run: echo "$JOB_CONTEXT" | |
| # end print details | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements file | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }} | |
| ${{ runner.os }}-pip | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel setuptools | |
| if [ -f requirements-all.txt ]; then | |
| python -m pip install -r requirements-all.txt | |
| elif [ -f requirements.txt ]; then | |
| python -m pip install -r requirements.txt; | |
| fi | |
| python -m pip install -e . | |
| - name: Install test dependencies | |
| # ToDo - remove pip install xgboost when flaml is fixed | |
| run: | | |
| if [ -f requirements-dev.txt ]; then | |
| python -m pip install -r requirements-dev.txt | |
| else | |
| echo "Missing requirements-dev.txt. Installing minimal requirements for testing." | |
| python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions | |
| python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml | |
| fi | |
| python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0" | |
| python -m pip install "xgboost" | |
| - name: Prepare test dummy data | |
| run: | | |
| mkdir ~/.msticpy | |
| mkdir ~/.msticpy/mordor | |
| cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy | |
| touch ~/.msticpy/GeoLite2-City.mmdb | |
| cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor | |
| touch ~/.msticpy/mordor/mitre_tact_cache.pkl | |
| touch ~/.msticpy/mordor/mitre_tech_cache.pkl | |
| touch ~/.msticpy/mordor/mordor_cache.pkl | |
| - name: Pytest | |
| env: | |
| MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }} | |
| IPSTACK_AUTH: ${{ secrets.IPSTACK_AUTH }} | |
| MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml | |
| MSTICPY_BUILD_SOURCE: fork | |
| JUPYTER_PLATFORM_DIRS: 1 | |
| run: | | |
| jupyter --paths | |
| pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticpy --cov-report=xml | |
| if: ${{ always() }} | |
| - name: Upload pytest test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pytest-results-${{ matrix.python-version }} | |
| path: junit/test-${{ matrix.python-version }}-results.xml | |
| # Use always() to always run this step to publish test results when there are test failures | |
| if: ${{ always() }} | |
| docs: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements file | |
| key: ${{ runner.os }}-pip-docs-${{ hashFiles('doc/requirements-all.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-docs-${{ hashFiles('requirements-all.txt') }} | |
| ${{ runner.os }}-pip-docs | |
| ${{ runner.os }}-pip | |
| - name: Sphinx Read the Docs build | |
| working-directory: docs | |
| run: | | |
| echo "Building docs from ${{ github.workspace }}/docs" | |
| python -m pip install sphinx readthedocs-sphinx-ext>=2.1.4 sphinx-rtd-theme>=1.0.0 | |
| python -m pip install -r ${{ github.workspace }}/docs/requirements.txt | |
| make html | |
| env: | |
| SPHINX_NOGEN: "true" | |
| lint: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements file | |
| key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }} | |
| ${{ runner.os }}-pip-lint | |
| ${{ runner.os }}-pip | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel setuptools | |
| if [ -f requirements-all.txt ]; then | |
| python -m pip install -r requirements-all.txt | |
| elif [ -f requirements.txt ]; then | |
| python -m pip install -r requirements.txt; | |
| fi | |
| python -m pip install -e . | |
| - name: Install test dependencies | |
| run: | | |
| if [ -f requirements-dev.txt ]; then | |
| python -m pip install -r requirements-dev.txt | |
| else | |
| echo "Missing requirements-dev.txt. Installing minimal requirements for testing." | |
| python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma | |
| fi | |
| - name: black | |
| run: | | |
| black --diff --check --exclude venv msticpy | |
| if: ${{ always() }} | |
| - name: flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 msticpy --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto | |
| if: ${{ always() }} | |
| - name: pylint | |
| run: | | |
| pylint msticpy --disable=duplicate-code --disable=E1135,E1101,E1133,W0101 | |
| if: ${{ always() }} | |
| - name: Cache/restore MyPy data | |
| id: cache-mypy | |
| uses: actions/cache@v4 | |
| with: | |
| # MyPy cache files are stored in `~/.mypy_cache` | |
| path: .mypy_cache | |
| key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} | |
| ${{ runner.os }}-build-mypy-${{ github.ref }} | |
| ${{ runner.os }}-build-mypy | |
| - name: mypy | |
| run: | | |
| mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticpy | |
| if: ${{ always() }} | |
| - name: Upload mypy test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Mypy results ${{ matrix.python-version }} | |
| path: junit/mypy-test-${{ matrix.python-version }}-results.xml | |
| # Use always() to always run this step to publish test results when there are test failures | |
| if: ${{ always() }} | |
| - name: bandit | |
| run: | | |
| bandit -x tests -r -s B303,B404,B603,B607,B608 msticpy | |
| if: ${{ always() }} | |
| - name: flake8 | |
| run: | | |
| flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto | |
| if: ${{ always() }} | |
| - name: pydocstyle | |
| run: | | |
| pydocstyle --convention=numpy msticpy | |
| if: ${{ always() }} | |
| - name: pyroma | |
| run: | | |
| pyroma --min 10 . | |
| if: ${{ always() }} | |
| check_status: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| needs: [build, lint, docs] | |
| steps: | |
| - name: File build fail issue | |
| if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }} | |
| uses: dacbd/create-issue-action@v1 | |
| with: | |
| token: ${{ github.token }} | |
| title: "Build failed for main branch" | |
| body: The build failed on branch ${{ github.ref }}. Please investigate | |
| labels: build_break, bug, high_severity |