Add CrowdSec TIProvider (#673)

* Add CrowdSec TIProvider Signed-off-by: Shivam Sandbhor <[email protected]> * Add user agent for crowdsec tiprovider Signed-off-by: Shivam Sandbhor <[email protected]> * Implement review suggestions Signed-off-by: Shivam Sandbhor <[email protected]> * Fix import error in tests Signed-off-by: Shivam Sandbhor <[email protected]> * Extraneous braces in test data in test, unneeded ioc_param item in test data for CrowdSec Added CrowdSec settings entry to test msticpyconfig.yaml and msticpyconfig-test.yaml * Adding docstring to crowdsec.py parse_results --------- Signed-off-by: Shivam Sandbhor <[email protected]> Co-authored-by: Ian Hellen <[email protected]>
microsoft · Jul 25, 2023 · f76650d · f76650d
1 parent a8a2de4
commit f76650d
Show file tree

Hide file tree

Showing 7 changed files with 234 additions and 4 deletions.
diff --git a/docs/source/data_acquisition/TIProviders.rst b/docs/source/data_acquisition/TIProviders.rst
@@ -30,6 +30,7 @@ Features
    -  **IBM XForce**
    -  **MS Sentinel TI**
    -  **GreyNoise**
+   -  **CrowdSec**
 
 -  Other pseudo-TI providers are also included:
 
@@ -268,6 +269,11 @@ fictitious - the format of the keys may differ from what is shown
           TenantID: 228d7b5f-4920-4f8e-872f-52072b92b651
         Primary: True
         Provider: "AzSTI"
+      CrowdSec:
+        Args:
+          AuthKey: [PLACEHOLDER]
+        Primary: True
+        Provider: "CrowdSec"
 
 You need to tell `TILookup` to refresh its configuration.
 
@@ -281,12 +287,13 @@ of providers loaded.
 
 .. parsed-literal::
 
+
   ['OTX - AlientVault OTX Lookup. (primary)',
   'VirusTotal - VirusTotal Lookup. (primary)',
   'XForce - IBM XForce Lookup. (primary)',
   'GreyNoise - GreyNoise Lookup. (primary)',
-  'AzSTI - Azure Sentinel TI provider class. (primary)',
-  'OPR - Open PageRank Lookup. (secondary)']
+  'AzSTI - Microsoft Sentinel TI provider class. (primary)',
+  'CrowdSec - CrowdSec CTI Smoke Lookup. (primary)']
 
 .. warning:: Depending on the type of account that you
   have with a provider, they will typically impose a limit
@@ -459,6 +466,7 @@ class method shows the current set of providers.
     VirusTotal
     XForce
     Intsights
+    CrowdSec
 
 You can view the list of supported query types for each provider
 with the ``show_query_types=True`` parameter.
@@ -650,6 +658,8 @@ TILookup syntax
 +-------------+--------------+----------+---------------+---------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------+---------+
 | GreyNoise   | 38.75.137.9  | ipv4     | None          | False   | Not found.                                                                                           | <Response [404]>                                                                                     | https://api.greynoise.io/v3/community/38.75.137.9                      | 404     |
 +-------------+--------------+----------+---------------+---------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------+---------+
+| CrowdSec    | 38.75.137.9  | ipv4     | None          | False   | {'Background Noise': 0, 'Overall Score': 0, 'First Seen': '2021-12-26T18:45:00+00:00', 'Last See...  | {'ip_range_score': 0, 'ip': '38.75.137.9', 'ip_range': '38.75.136.0/23', 'as_name': 'AS-GLOBALTE...  | https://cti.api.crowdsec.net/v2/smoke/38.75.137.9	                     | 200     |
++-------------+--------------+----------+---------------+---------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------+---------+
 
 Pivot function syntax
 

diff --git a/docs/source/getting_started/msticpyconfig.rst b/docs/source/getting_started/msticpyconfig.rst
@@ -115,6 +115,7 @@ Currently supported provider names are:
 - IntSights
 - TorExitNodes
 - OpenPageRank
+- CrowdSec
 
 .. code:: yaml
 
@@ -136,6 +137,11 @@ Currently supported provider names are:
             KeyVault:
         Primary: False
         Provider: "OPR"
+      CrowdSec:
+        Args:
+          AuthKey: [PLACEHOLDER]
+        Primary: True
+        Provider: "CrowdSec"
 
 .. note:: You store values in the ``Args`` section as simple strings,
    as names of environment variables containing the value, or

diff --git a/msticpy/context/tiproviders/__init__.py b/msticpy/context/tiproviders/__init__.py
@@ -14,6 +14,7 @@
 __version__ = VERSION
 
 TI_PROVIDERS: Dict[str, Tuple[str, str]] = {
+    "CrowdSec": ("crowdsec", "CrowdSec"),
     "OTX": ("alienvault_otx", "OTX"),
     "AzSTI": ("azure_sent_byoti", "AzSTI"),
     "GreyNoise": ("greynoise", "GreyNoise"),

diff --git a/msticpy/context/tiproviders/crowdsec.py b/msticpy/context/tiproviders/crowdsec.py
@@ -0,0 +1,75 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""
+CrowdSec Provider.
+
+Input can be a single IoC observable or a pandas DataFrame containing
+multiple observables. Processing may require an API key and
+processing performance may be limited to a specific number of
+requests per minute for the account type that you have.
+
+"""
+from typing import Any, Dict, Tuple
+
+from ..._version import VERSION
+from ..http_provider import APILookupParams
+from .ti_http_provider import HttpTIProvider
+from .ti_provider_base import ResultSeverity
+
+__version__ = VERSION
+__author__ = "Shivam Sandbhor"
+
+
+class CrowdSec(HttpTIProvider):
+    """CrowdSec CTI Smoke Lookup."""
+
+    _BASE_URL = "https://cti.api.crowdsec.net"
+
+    _QUERIES = {
+        "ipv4": APILookupParams(
+            path="/v2/smoke/{observable}",
+            headers={
+                "x-api-key": "{AuthKey}",
+                "User-Agent": "crowdsec-msticpy-tiprovider/v1.0.0",
+            },
+        ),
+    }
+    _QUERIES["ipv6"] = _QUERIES["ipv4"]
+
+    def parse_results(self, response: Dict) -> Tuple[bool, ResultSeverity, Any]:
+        """Return the details of the response."""
+        if self._failed_response(response):
+            return False, ResultSeverity.information, response["RawResult"]["message"]
+
+        if response["RawResult"]["scores"]["overall"]["total"] <= 2:
+            result_severity = ResultSeverity.information
+        elif response["RawResult"]["scores"]["overall"]["total"] <= 3:
+            result_severity = ResultSeverity.warning
+        else:
+            result_severity = ResultSeverity.high
+
+        return (
+            True,
+            result_severity,
+            {
+                "Background Noise": response["RawResult"]["background_noise_score"],
+                "Overall Score": response["RawResult"]["scores"]["overall"]["total"],
+                "First Seen": response["RawResult"]["history"]["first_seen"],
+                "Last Seen": response["RawResult"]["history"]["last_seen"],
+                "Attack Details": ",".join(
+                    [
+                        attack_detail["label"]
+                        for attack_detail in response["RawResult"]["attack_details"]
+                    ]
+                ),
+                "Behaviors": ",".join(
+                    [
+                        behavior["name"]
+                        for behavior in response["RawResult"]["behaviors"]
+                    ]
+                ),
+            },
+        )
diff --git a/tests/context/test_tiproviders.py b/tests/context/test_tiproviders.py
@@ -157,7 +157,15 @@ def _format_json_response(resp_data, **kwargs):
     "www.microsoft.com": ("hostname", "whois"),
 }
 
-_TI_PROVIDER_TESTS = ["XForce", "OTX", "VirusTotal", "GreyNoise", "RiskIQ", "IntSights"]
+_TI_PROVIDER_TESTS = [
+    "XForce",
+    "OTX",
+    "VirusTotal",
+    "GreyNoise",
+    "RiskIQ",
+    "IntSights",
+    "CrowdSec",
+]
 
 
 @pytest.mark.parametrize("provider_name", _TI_PROVIDER_TESTS)
@@ -218,7 +226,15 @@ def verify_result(result, ti_lookup):
     for lu_result in result.to_dict(orient="records"):
         check.is_in(
             lu_result["Provider"],
-            ["OTX", "XForce", "VirusTotal", "GreyNoise", "RiskIQ", "IntSights"],
+            [
+                "OTX",
+                "XForce",
+                "VirusTotal",
+                "GreyNoise",
+                "RiskIQ",
+                "IntSights",
+                "CrowdSec",
+            ],
         )
         check.is_not_none(lu_result["Ioc"])
         check.is_not_none(lu_result["IocType"])
@@ -887,6 +903,118 @@ def _get_riskiq_classification():
             "Tags": ["tag"],
         },
     },
+    "https://cti.api.crowdsec.net": {
+        "response": {
+            "ip_range_score": 1,
+            "ip": "167.248.133.133",
+            "ip_range": "167.248.133.0/24",
+            "as_name": "CENSYS-ARIN-03",
+            "as_num": 398722,
+            "location": {
+                "country": "US",
+                "city": None,
+                "latitude": 1.751,
+                "longitude": -97.822,
+            },
+            "reverse_dns": "scanner-03.ch1.censys-scanner.com",
+            "behaviors": [
+                {
+                    "name": "sip:bruteforce",
+                    "label": "SIP Bruteforce",
+                    "description": "IP has been reported for performing a SIP (VOIP) brute force attack.",
+                },
+                {
+                    "name": "tcp:scan",
+                    "label": "TCP Scan",
+                    "description": "IP has been reported for performing TCP port scanning.",
+                },
+            ],
+            "history": {
+                "first_seen": f"{dt.datetime.now().isoformat(timespec='seconds')}+00:00",
+                "last_seen": f"{dt.datetime.now().isoformat(timespec='seconds')}+00:00",
+                "full_age": 490,
+                "days_age": 489,
+            },
+            "classifications": {
+                "false_positives": [],
+                "classifications": [
+                    {
+                        "name": "scanner:legit",
+                        "label": "Legit scanner",
+                        "description": "IP belongs to a company that scans the internet",
+                    },
+                    {
+                        "name": "scanner:censys",
+                        "label": "Known Security Company",
+                        "description": "IP belongs to a company that scans the internet: Censys.",
+                    },
+                    {
+                        "name": "community-blocklist",
+                        "label": "CrowdSec Community Blocklist",
+                        "description": "IP belongs to the CrowdSec Community Blocklist",
+                    },
+                ],
+            },
+            "attack_details": [
+                {
+                    "name": "crowdsecurity/opensips-request",
+                    "label": "SIP Bruteforce",
+                    "description": "Detect brute force on VOIP/SIP services",
+                    "references": [],
+                },
+                {
+                    "name": "firewallservices/pf-scan-multi_ports",
+                    "label": "Port Scanner",
+                    "description": "Detect tcp port scan",
+                    "references": [],
+                },
+            ],
+            "target_countries": {
+                "DE": 27,
+                "FR": 19,
+                "US": 16,
+                "EE": 16,
+                "HK": 5,
+                "DK": 2,
+                "GB": 2,
+                "FI": 2,
+                "KR": 2,
+                "SG": 2,
+            },
+            "background_noise_score": 10,
+            "scores": {
+                "overall": {
+                    "aggressiveness": 1,
+                    "threat": 4,
+                    "trust": 5,
+                    "anomaly": 0,
+                    "total": 3,
+                },
+                "last_day": {
+                    "aggressiveness": 0,
+                    "threat": 0,
+                    "trust": 0,
+                    "anomaly": 0,
+                    "total": 0,
+                },
+                "last_week": {
+                    "aggressiveness": 0,
+                    "threat": 4,
+                    "trust": 5,
+                    "anomaly": 0,
+                    "total": 3,
+                },
+                "last_month": {
+                    "aggressiveness": 0,
+                    "threat": 4,
+                    "trust": 5,
+                    "anomaly": 0,
+                    "total": 3,
+                },
+            },
+            "references": [],
+        },
+    },
 }
 
 _TOR_NODES = [

diff --git a/tests/msticpyconfig-test.yaml b/tests/msticpyconfig-test.yaml
@@ -90,6 +90,11 @@ TIProviders:
       AuthKey: "[PLACEHOLDER]"
     Primary: True
     Provider: Pulsedive
+  CrowdSec:
+    Args:
+      AuthKey: "[PLACEHOLDER]"
+    Primary: True
+    Provider: CrowdSec
 ContextProviders:
   ServiceNow:
     Primary: True

diff --git a/tests/testdata/msticpyconfig.yaml b/tests/testdata/msticpyconfig.yaml
@@ -67,6 +67,11 @@ TIProviders:
       AuthKey: "INTSIGHTS_KEY"
     Primary: False
     Provider: "IntSights"
+  CrowdSec:
+    Args:
+      AuthKey: "[PLACEHOLDER]"
+    Primary: True
+    Provider: CrowdSec
 OtherProviders:
   GeoIPLite:
     Args: