diff --git a/docs/source/data_acquisition/TIProviders.rst b/docs/source/data_acquisition/TIProviders.rst index 03d266986..d3c9f957d 100644 --- a/docs/source/data_acquisition/TIProviders.rst +++ b/docs/source/data_acquisition/TIProviders.rst @@ -30,6 +30,7 @@ Features - **IBM XForce** - **MS Sentinel TI** - **GreyNoise** + - **CrowdSec** - Other pseudo-TI providers are also included: @@ -268,6 +269,11 @@ fictitious - the format of the keys may differ from what is shown TenantID: 228d7b5f-4920-4f8e-872f-52072b92b651 Primary: True Provider: "AzSTI" + CrowdSec: + Args: + AuthKey: [PLACEHOLDER] + Primary: True + Provider: "CrowdSec" You need to tell `TILookup` to refresh its configuration. @@ -281,12 +287,13 @@ of providers loaded. .. parsed-literal:: + ['OTX - AlientVault OTX Lookup. (primary)', 'VirusTotal - VirusTotal Lookup. (primary)', 'XForce - IBM XForce Lookup. (primary)', 'GreyNoise - GreyNoise Lookup. (primary)', - 'AzSTI - Azure Sentinel TI provider class. (primary)', - 'OPR - Open PageRank Lookup. (secondary)'] + 'AzSTI - Microsoft Sentinel TI provider class. (primary)', + 'CrowdSec - CrowdSec CTI Smoke Lookup. (primary)'] .. warning:: Depending on the type of account that you have with a provider, they will typically impose a limit @@ -459,6 +466,7 @@ class method shows the current set of providers. VirusTotal XForce Intsights + CrowdSec You can view the list of supported query types for each provider with the ``show_query_types=True`` parameter. @@ -650,6 +658,8 @@ TILookup syntax +-------------+--------------+----------+---------------+---------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------+---------+ | GreyNoise | 38.75.137.9 | ipv4 | None | False | Not found. | | https://api.greynoise.io/v3/community/38.75.137.9 | 404 | +-------------+--------------+----------+---------------+---------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------+---------+ +| CrowdSec | 38.75.137.9 | ipv4 | None | False | {'Background Noise': 0, 'Overall Score': 0, 'First Seen': '2021-12-26T18:45:00+00:00', 'Last See... | {'ip_range_score': 0, 'ip': '38.75.137.9', 'ip_range': '38.75.136.0/23', 'as_name': 'AS-GLOBALTE... | https://cti.api.crowdsec.net/v2/smoke/38.75.137.9 | 200 | ++-------------+--------------+----------+---------------+---------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------+---------+ Pivot function syntax diff --git a/docs/source/getting_started/msticpyconfig.rst b/docs/source/getting_started/msticpyconfig.rst index 4244628d3..0184243dd 100644 --- a/docs/source/getting_started/msticpyconfig.rst +++ b/docs/source/getting_started/msticpyconfig.rst @@ -115,6 +115,7 @@ Currently supported provider names are: - IntSights - TorExitNodes - OpenPageRank +- CrowdSec .. code:: yaml @@ -136,6 +137,11 @@ Currently supported provider names are: KeyVault: Primary: False Provider: "OPR" + CrowdSec: + Args: + AuthKey: [PLACEHOLDER] + Primary: True + Provider: "CrowdSec" .. note:: You store values in the ``Args`` section as simple strings, as names of environment variables containing the value, or diff --git a/msticpy/context/tiproviders/__init__.py b/msticpy/context/tiproviders/__init__.py index 84c2675ba..42c977c34 100644 --- a/msticpy/context/tiproviders/__init__.py +++ b/msticpy/context/tiproviders/__init__.py @@ -14,6 +14,7 @@ __version__ = VERSION TI_PROVIDERS: Dict[str, Tuple[str, str]] = { + "CrowdSec": ("crowdsec", "CrowdSec"), "OTX": ("alienvault_otx", "OTX"), "AzSTI": ("azure_sent_byoti", "AzSTI"), "GreyNoise": ("greynoise", "GreyNoise"), diff --git a/msticpy/context/tiproviders/crowdsec.py b/msticpy/context/tiproviders/crowdsec.py new file mode 100644 index 000000000..f80dcdd4f --- /dev/null +++ b/msticpy/context/tiproviders/crowdsec.py @@ -0,0 +1,75 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +""" +CrowdSec Provider. + +Input can be a single IoC observable or a pandas DataFrame containing +multiple observables. Processing may require an API key and +processing performance may be limited to a specific number of +requests per minute for the account type that you have. + +""" +from typing import Any, Dict, Tuple + +from ..._version import VERSION +from ..http_provider import APILookupParams +from .ti_http_provider import HttpTIProvider +from .ti_provider_base import ResultSeverity + +__version__ = VERSION +__author__ = "Shivam Sandbhor" + + +class CrowdSec(HttpTIProvider): + """CrowdSec CTI Smoke Lookup.""" + + _BASE_URL = "https://cti.api.crowdsec.net" + + _QUERIES = { + "ipv4": APILookupParams( + path="/v2/smoke/{observable}", + headers={ + "x-api-key": "{AuthKey}", + "User-Agent": "crowdsec-msticpy-tiprovider/v1.0.0", + }, + ), + } + _QUERIES["ipv6"] = _QUERIES["ipv4"] + + def parse_results(self, response: Dict) -> Tuple[bool, ResultSeverity, Any]: + """Return the details of the response.""" + if self._failed_response(response): + return False, ResultSeverity.information, response["RawResult"]["message"] + + if response["RawResult"]["scores"]["overall"]["total"] <= 2: + result_severity = ResultSeverity.information + elif response["RawResult"]["scores"]["overall"]["total"] <= 3: + result_severity = ResultSeverity.warning + else: + result_severity = ResultSeverity.high + + return ( + True, + result_severity, + { + "Background Noise": response["RawResult"]["background_noise_score"], + "Overall Score": response["RawResult"]["scores"]["overall"]["total"], + "First Seen": response["RawResult"]["history"]["first_seen"], + "Last Seen": response["RawResult"]["history"]["last_seen"], + "Attack Details": ",".join( + [ + attack_detail["label"] + for attack_detail in response["RawResult"]["attack_details"] + ] + ), + "Behaviors": ",".join( + [ + behavior["name"] + for behavior in response["RawResult"]["behaviors"] + ] + ), + }, + ) diff --git a/tests/context/test_tiproviders.py b/tests/context/test_tiproviders.py index 5e76f9457..5927b11af 100644 --- a/tests/context/test_tiproviders.py +++ b/tests/context/test_tiproviders.py @@ -157,7 +157,15 @@ def _format_json_response(resp_data, **kwargs): "www.microsoft.com": ("hostname", "whois"), } -_TI_PROVIDER_TESTS = ["XForce", "OTX", "VirusTotal", "GreyNoise", "RiskIQ", "IntSights"] +_TI_PROVIDER_TESTS = [ + "XForce", + "OTX", + "VirusTotal", + "GreyNoise", + "RiskIQ", + "IntSights", + "CrowdSec", +] @pytest.mark.parametrize("provider_name", _TI_PROVIDER_TESTS) @@ -218,7 +226,15 @@ def verify_result(result, ti_lookup): for lu_result in result.to_dict(orient="records"): check.is_in( lu_result["Provider"], - ["OTX", "XForce", "VirusTotal", "GreyNoise", "RiskIQ", "IntSights"], + [ + "OTX", + "XForce", + "VirusTotal", + "GreyNoise", + "RiskIQ", + "IntSights", + "CrowdSec", + ], ) check.is_not_none(lu_result["Ioc"]) check.is_not_none(lu_result["IocType"]) @@ -887,6 +903,118 @@ def _get_riskiq_classification(): "Tags": ["tag"], }, }, + "https://cti.api.crowdsec.net": { + "response": { + "ip_range_score": 1, + "ip": "167.248.133.133", + "ip_range": "167.248.133.0/24", + "as_name": "CENSYS-ARIN-03", + "as_num": 398722, + "location": { + "country": "US", + "city": None, + "latitude": 1.751, + "longitude": -97.822, + }, + "reverse_dns": "scanner-03.ch1.censys-scanner.com", + "behaviors": [ + { + "name": "sip:bruteforce", + "label": "SIP Bruteforce", + "description": "IP has been reported for performing a SIP (VOIP) brute force attack.", + }, + { + "name": "tcp:scan", + "label": "TCP Scan", + "description": "IP has been reported for performing TCP port scanning.", + }, + ], + "history": { + "first_seen": f"{dt.datetime.now().isoformat(timespec='seconds')}+00:00", + "last_seen": f"{dt.datetime.now().isoformat(timespec='seconds')}+00:00", + "full_age": 490, + "days_age": 489, + }, + "classifications": { + "false_positives": [], + "classifications": [ + { + "name": "scanner:legit", + "label": "Legit scanner", + "description": "IP belongs to a company that scans the internet", + }, + { + "name": "scanner:censys", + "label": "Known Security Company", + "description": "IP belongs to a company that scans the internet: Censys.", + }, + { + "name": "community-blocklist", + "label": "CrowdSec Community Blocklist", + "description": "IP belongs to the CrowdSec Community Blocklist", + }, + ], + }, + "attack_details": [ + { + "name": "crowdsecurity/opensips-request", + "label": "SIP Bruteforce", + "description": "Detect brute force on VOIP/SIP services", + "references": [], + }, + { + "name": "firewallservices/pf-scan-multi_ports", + "label": "Port Scanner", + "description": "Detect tcp port scan", + "references": [], + }, + ], + "target_countries": { + "DE": 27, + "FR": 19, + "US": 16, + "EE": 16, + "HK": 5, + "DK": 2, + "GB": 2, + "FI": 2, + "KR": 2, + "SG": 2, + }, + "background_noise_score": 10, + "scores": { + "overall": { + "aggressiveness": 1, + "threat": 4, + "trust": 5, + "anomaly": 0, + "total": 3, + }, + "last_day": { + "aggressiveness": 0, + "threat": 0, + "trust": 0, + "anomaly": 0, + "total": 0, + }, + "last_week": { + "aggressiveness": 0, + "threat": 4, + "trust": 5, + "anomaly": 0, + "total": 3, + }, + "last_month": { + "aggressiveness": 0, + "threat": 4, + "trust": 5, + "anomaly": 0, + "total": 3, + }, + }, + "references": [], + }, + }, } _TOR_NODES = [ diff --git a/tests/msticpyconfig-test.yaml b/tests/msticpyconfig-test.yaml index a6bd04c78..715c650e6 100644 --- a/tests/msticpyconfig-test.yaml +++ b/tests/msticpyconfig-test.yaml @@ -90,6 +90,11 @@ TIProviders: AuthKey: "[PLACEHOLDER]" Primary: True Provider: Pulsedive + CrowdSec: + Args: + AuthKey: "[PLACEHOLDER]" + Primary: True + Provider: CrowdSec ContextProviders: ServiceNow: Primary: True diff --git a/tests/testdata/msticpyconfig.yaml b/tests/testdata/msticpyconfig.yaml index 35b7b01db..b9833b84a 100644 --- a/tests/testdata/msticpyconfig.yaml +++ b/tests/testdata/msticpyconfig.yaml @@ -67,6 +67,11 @@ TIProviders: AuthKey: "INTSIGHTS_KEY" Primary: False Provider: "IntSights" + CrowdSec: + Args: + AuthKey: "[PLACEHOLDER]" + Primary: True + Provider: CrowdSec OtherProviders: GeoIPLite: Args: