diff --git a/docs/user/bots.rst b/docs/user/bots.rst index b0054227e..d46b7d776 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3194,6 +3194,7 @@ Threshold **Information** +* **Cache parameters** (see section :ref:`common-parameters`) * `name`: `intelmq.bots.experts.threshold.expert` * `lookup`: redis cache * `public`: no @@ -3226,6 +3227,31 @@ This bot has certain limitations and is not a true threshold filter (yet). It wo Please note: Even if a message is sent, any further identical messages are dropped, if the time difference to the last message is less than the timeout! The counter is not reset if the threshold is reached. +.. _intelmq.bots.experts.time_filter.expert: + +Time Filter +^^^^^^^^^ + +Filtering records that do not exceed the specified deadline. + +**Information** + +* `name:` `intelmq.bots.experts.time_filter.expert` +* `lookup:` no +* `public:` yes +* `cache (redis db):` none +* `description:` Time based filtering + +**Configuration Parameters** + +* `field`: the event field which should be filtered on. Needs to be in ISO 8601 formatted. Default: `time.source` +* `timespan`: time interval for filtering records. Default: `24 hours` + +**Description** + +The bot filters new records, for example when only records from the last 2 days (48 hours) are needed. Also allows entries without a date. + + .. _intelmq.bots.experts.tor_nodes.expert: Tor Nodes diff --git a/intelmq/bots/experts/time_filter/__init__.py b/intelmq/bots/experts/time_filter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py new file mode 100644 index 000000000..ce0739566 --- /dev/null +++ b/intelmq/bots/experts/time_filter/expert.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +""" +Time based filtering + +SPDX-FileCopyrightText: 2021 Marius Karotkis +SPDX-License-Identifier: AGPL-3.0-or-later +""" + +from datetime import datetime, timedelta +from dateutil import parser +from intelmq.lib.bot import Bot +from datetime import timezone +from intelmq.lib.utils import parse_relative + + +class TimeFilterExpertBot(Bot): + """ Time based filtering """ + field: str = 'time.source' + timespan: str = '24 hours' + + __delta = None + + def init(self): + if self.field: + self.__delta = datetime.now(tz=timezone.utc) - timedelta(minutes=parse_relative(self.timespan)) + + def process(self): + event = self.receive_message() + event_time = self.__delta + + if self.field in event: + try: + event_time = parser.parse(str(event.get(self.field))) + except ValueError: + self.process_message(event_time, event) + return + else: + self.process_message(event_time, event) + return + else: + # not found field + self.process_message(event_time, event) + return + + def process_message(self, event_time, event): + event_time = event_time.replace(tzinfo=None) + self.__delta = self.__delta.replace(tzinfo=None) + + if event_time > self.__delta: + self.send_message(event) + else: + self.logger.debug(f"Filtered out event with search field {self.field} and event time {event_time} .") + + self.acknowledge_message() + + +BOT = TimeFilterExpertBot diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index f1ebbcd9c..959848b7f 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -857,7 +857,8 @@ def list_all_bots() -> dict: base_path = resource_filename('intelmq', 'bots') - botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if botfile.is_file() and botfile.name != '__init__.py'] + botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if + botfile.is_file() and botfile.name != '__init__.py'] for file in botfiles: file = Path(file.as_posix().replace(base_path, 'intelmq/bots')) try: @@ -881,7 +882,8 @@ def list_all_bots() -> dict: bots[file.parts[2].capitalize()[:-1]][name] = { "module": mod.__name__, - "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent(mod.BOT.__doc__).strip(), + "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent( + mod.BOT.__doc__), "parameters": keys, } return bots diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt new file mode 100644 index 000000000..b00113cde --- /dev/null +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2021 Marius Karotkis +# SPDX-License-Identifier: AGPL-3.0-or-later + +time-machine diff --git a/intelmq/tests/bots/experts/time_filter/__init__.py b/intelmq/tests/bots/experts/time_filter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py new file mode 100644 index 000000000..5f47b96b0 --- /dev/null +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +""" +Time based filtering + +SPDX-FileCopyrightText: 2021 Marius Karotkis +SPDX-License-Identifier: AGPL-3.0-or-later +""" + +import unittest + +import intelmq.lib.test as test +from intelmq.bots.experts.time_filter.expert import TimeFilterExpertBot +from intelmq.lib.exceptions import MissingDependencyError + +try: + import time_machine +except ImportError: + time_machine = None + +EXAMPLE_INPUT_DROP = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "2020-08-13T00:00:00+00:00", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline1": "2020-09-13T00:00:00+00:00", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS_2 = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS_3 = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "2020-09-13", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} + + +@test.skip_exotic() +class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for TimeFilterExpertBot handling Reports. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = TimeFilterExpertBot + cls.input_message = EXAMPLE_INPUT_DROP + cls.sysconfig = { + 'search_field': 'extra.lastonline', + 'search_from': "72 hours" + } + + if time_machine: + @time_machine.travel("2021-05-05") + def test_expert_drop(self): + self.run_bot() + self.assertOutputQueueLen(0) + + @time_machine.travel("2020-09-09") + def test_expert_pass(self): + self.input_message = EXAMPLE_INPUT_PASS + self.run_bot() + self.assertOutputQueueLen(1) + + @time_machine.travel("2020-09-09") + def test_expert_pass_2(self): + self.input_message = EXAMPLE_INPUT_PASS_2 + self.run_bot() + self.assertOutputQueueLen(1) + + @time_machine.travel("2020-09-09") + def test_expert_pass_3(self): + self.input_message = EXAMPLE_INPUT_PASS_3 + self.run_bot() + self.assertOutputQueueLen(1) + + +if __name__ == '__main__': # pragma: no cover + unittest.main()