Merge pull request #874 from stratosphereips/develop

Slips v1.1

.github/ISSUE_TEMPLATE/bug_report.md

@@ -29,7 +29,7 @@ Master or Develop?
 **Environment (please complete the following information):**
  - OS: [e.g. iOS]
  - Version [e.g. 22]
- - Python version [e.g. 3.8]
+ - Python version [e.g. 3.10]
  - Are you running slips in docker or locally? [yes/no]
  - Docker version (if running slips in docker) [e.g. 20.10.22]
  - Slips docker image used (if running slips in docker) [e.g. macosm1-image, macosm1-P2P-image, ubuntu-image, dependency-image]

.github/workflows/CI-production-testing.yml

@@ -1,4 +1,4 @@
-name: CI-production
+name: CI-production-testing
 
 on:
   pull_request:
@@ -10,7 +10,7 @@ jobs:
 
   unit_and_integration_tests:
     # runs the tests on a GH VM
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     # 2 hours timeout
     timeout-minutes: 7200
 
@@ -27,38 +27,79 @@ jobs:
 
     - name: Install Zeek
       run: |
-        sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_20.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list
-        curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_20.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
-        sudo apt update
-        sudo apt install -y --no-install-recommends zeek
+        sudo echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' | sudo tee /etc/apt/sources.list.d/security:zeek.list
+        curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null
+        sudo apt update && sudo apt install -y --no-install-recommends --fix-missing zeek
         sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
 
-    - name: Set up Python 3.8
+    - name: confirm that zeek is installed
+      run: |
+        which bro
+        bro -h
+
+    - name: Set up Python 3.10.12
       uses: actions/setup-python@v2
       with:
-        python-version: "3.8"
+        python-version: "3.10.12"
 
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        grep -v 'black' install/requirements.txt | xargs pip3 install  --no-cache-dir
-        pip install coverage
+        python3 -m pip install --no-cache-dir -r install/requirements.txt
 
     - name: Start redis server
       run: redis-server --daemonize yes
 
-    - name: Run unit tests
-      run: python3 -m pytest tests/ --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
-
 
     - name: Run database unit tests
       run: |
         python3 -m pytest tests/test_database.py -p no:warnings -vv
 
+    - name: main unit tests
+      run: |
+        python3 -m pytest tests/test_main.py -p no:warnings -vv
+
     - name: Flowalerts Unit Tests
       run: |
         python3 -m pytest tests/test_flowalerts.py -p no:warnings -vv
 
+    - name: conn Unit Tests
+      run: |
+        python3 -m pytest tests/test_conn.py -p no:warnings -vv
+
+    - name: downloaded file Unit Tests
+      run: |
+        python3 -m pytest tests/test_downloaded_file.py -p no:warnings -vv
+
+    - name: SSL Unit Tests
+      run: |
+        python3 -m pytest tests/test_ssl.py -p no:warnings -vv
+
+    - name: Tunnel Unit Tests
+      run: |
+        python3 -m pytest tests/test_tunnel.py -p no:warnings -vv
+
+    - name: SSH Unit Tests
+      run: |
+        python3 -m pytest tests/test_ssh.py -p no:warnings -vv
+
+
+    - name: dns Unit Tests
+      run: |
+        python3 -m pytest tests/test_dns.py -p no:warnings -vv
+
+    - name: Notice Unit Tests
+      run: |
+        python3 -m pytest tests/test_notice.py -p no:warnings -vv
+
+    - name: Software Unit Tests
+      run: |
+        python3 -m pytest tests/test_software.py -p no:warnings -vv
+
+    - name: SMTP Unit Tests
+      run: |
+        python3 -m pytest tests/test_smtp.py -p no:warnings -vv
+
     - name: Whitelist Unit Tests
       run: |
         python3 -m pytest tests/test_whitelist.py -p no:warnings -vv
@@ -71,7 +112,7 @@ jobs:
       run: |
         python3 -m pytest tests/test_blocking.py -p no:warnings -vv
 
-    - name: Flowhandler Unit Test
+    - name: Flow handler Unit Tests
       run: |
         python3 -m pytest tests/test_flow_handler.py -p no:warnings -vv
 
@@ -87,6 +128,10 @@ jobs:
       run: |
         python3 -m pytest tests/test_vertical_portscans.py -p no:warnings -vv
 
+    - name: Network Discovery Unit Tests
+      run: |
+        python3 -m pytest tests/test_network_discovery.py -p no:warnings -vv
+
     - name: Virustotal Unit Tests
       run: |
         python3 -m pytest tests/test_virustotal.py -p no:warnings -vv
@@ -119,27 +164,41 @@ jobs:
       run: |
         python3 -m pytest tests/test_ip_info.py -p no:warnings -vv
 
+    - name: evidence tests
+      run: |
+        python3 -m pytest -s tests/test_evidence.py -p no:warnings -vv
+
+    - name: Urlhaus Unit Tests
+      run: |
+        python3 -m pytest tests/test_urlhaus.py -p no:warnings -vv
+
     - name: Input Unit Tests
       run: |
         python3 -m pytest tests/test_inputProc.py -p no:warnings -vv
 
+    - name: set Evidence Unit Tests
+      run: |
+        python3 -m pytest tests/test_set_evidence.py -p no:warnings -vv
+
     - name: Clear redis cache
       run: ./slips.py -cc
 
+
+
+    - name: Config file tests
+      run: |
+        python3 -m pytest -s tests/integration_tests/test_config_files.py  -p no:warnings -vv
+
     - name: Portscan tests
       run: |
         python3 -m pytest -s tests/integration_tests/test_portscans.py -p no:warnings -vv
 
-    - name: Integration tests
+    - name: Dataset Integration tests
       run: |
         python3 -m pytest -s tests/integration_tests/test_dataset.py -p no:warnings -vv
 
 
-    - name: Config file tests
-      run: |
-        python3 -m pytest -s tests/integration_tests/test_config_files.py  -p no:warnings -vv
-
-    - name: Upload Artifact
+    - name: Upload Artifacts
       # run this job whether the above jobs failed or passed
       if: success() || failure()
       uses: actions/upload-artifact@v3

.github/workflows/CI-publishing-dependency-image.yml

@@ -0,0 +1,31 @@
+name: CI-production-publishing-dependency-image
+
+on:
+  push:
+    branches:
+    - 'master'
+    - '!develop'
+
+jobs:
+  build_and_push_dependency_image:
+    runs-on: ubuntu-latest
+    steps:
+    # clone slips and checkout branch
+    - uses: actions/checkout@v3
+      with:
+        ref: 'master'
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v2
+      with:
+        username: stratosphereips
+        password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+    - name: Build and push latest dependency image
+      id: docker_build_dependency_image
+      uses: docker/build-push-action@v2
+      with:
+        context: ./
+        file: ./docker/dependency-image/Dockerfile
+        tags: stratosphereips/slips_dependencies:latest
+        push: true

.github/workflows/CI-publishing-p2p-image.yml

@@ -0,0 +1,53 @@
+name: CI-production-publishing-p2p-image
+
+on:
+  push:
+    branches:
+    - 'master'
+    - '!develop'
+
+jobs:
+  publish_P2P_docker_image:
+    # runs the tests in a docker(built by this job) on stop of a GH VM
+    runs-on: ubuntu-20.04
+    # 2 hours timeout
+    timeout-minutes: 7200
+
+    steps:
+    - name: Get slips version
+      run: |
+        VER=$(curl -s  https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/VERSION)
+        echo "SLIPS_VERSION=$VER" >> $GITHUB_ENV
+
+    # clone slips and checkout branch
+    # By default it checks out only one commit
+    - uses: actions/checkout@v3
+      with:
+        ref: 'master'
+        # Fetch all history for all tags and branches
+        fetch-depth: ''
+        submodules: true
+
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v2
+      with:
+        username: stratosphereips
+        password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+    # Set up Docker Buildx with docker-container driver is required
+    # at the moment to be able to use a subdirectory with Git context
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Build and push p2p image using dockerfile
+      id: docker_build_p2p_for_slips
+      uses: docker/build-push-action@v3
+      with:
+        allow: network.host
+        context: ./
+        file: ./docker/P2P-image/Dockerfile
+        tags: |
+          stratosphereips/slips_p2p:latest
+          stratosphereips/slips_p2p:${{ env.SLIPS_VERSION }}
+        push: true

.github/workflows/CI-publishing-ubuntu-image.yml

@@ -0,0 +1,63 @@
+name: CI-production-publishing-ubuntu-image
+
+on:
+  push:
+    branches:
+    - 'master'
+    - '!develop'
+
+jobs:
+  create_release_tag:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Get slips version
+        run: |
+          VER=$(curl -s  https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/VERSION)
+          echo "SLIPS_VERSION=$VER" >> $GITHUB_ENV
+
+      # add release tag
+      - uses: actions/checkout@v3
+      - uses: rickstaa/action-create-tag@v1
+        with:
+          tag: ${{ env.SLIPS_VERSION }}
+          message: ""
+
+  publish_ubuntu_image:
+    # runs the tests in a docker(built by this job) on top of a GH VM
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Get Slips version
+      run: |
+        VER=$(curl -s  https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/VERSION)
+        echo "SLIPS_VERSION=$VER" >> $GITHUB_ENV
+
+    # clone slips and checkout branch
+    # By default it checks out only one commit
+    - uses: actions/checkout@v3
+      with:
+        ref: 'master'
+        # Fetch all history for all tags and branches
+        fetch-depth: ''
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v2
+      with:
+        username: stratosphereips
+        password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+    - name: Build and publish ubuntu image from Dockerfile
+      id: docker_build_slips
+      timeout-minutes: 15
+      uses: docker/build-push-action@v5
+      with:
+        debug: true
+        verbose-debug: true
+        no-cache: true
+        context: ./
+        file: ./docker/ubuntu-image/Dockerfile
+        tags: |
+          stratosphereips/ubuntu_image:latest
+          stratosphereips/ubuntu_image:${{ env.SLIPS_VERSION }}
+        push: true

.github/workflows/CI-publishing-update-code-docs.yml

@@ -0,0 +1,34 @@
+name: CI-production-update-code-docs
+
+on:
+  push:
+    branches:
+    - 'master'
+    - '!develop'
+
+jobs:
+  update_code_docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: 'code-docs-branch'
+          fetch-depth: 0 # otherwise, you will fail to push refs to the dest repo
+
+      - name: install doxygen and python
+        run: |
+          sudo apt update
+          sudo apt install python3 doxygen
+
+      # deletes old docs, generates new ones, and adds all new files to git
+      - name: update docs
+        run: python3 docs/generate_docs.py
+
+      # commit and push to code docs branch
+      - name: Commit & Push changes
+        uses: actions-js/push@master
+        with:
+          github_token: ${{ secrets.GH_TOKEN_FOR_COMMITTING_AND_PUSHING_CODE_DOCS }}
+          message: '[Github actions] Update code docs'
+          branch: 'code-docs-branch'