From 0405205f87f6effb080d1e299b0673f62c44f820 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 26 May 2021 14:50:05 +0300 Subject: [PATCH 01/17] Add new bot: time filter --- intelmq/bots/experts/time_filter/__init__.py | 0 intelmq/bots/experts/time_filter/expert.py | 55 +++++++++ .../bots/experts/time_filter/__init__.py | 0 .../bots/experts/time_filter/test_expert.py | 106 ++++++++++++++++++ 4 files changed, 161 insertions(+) create mode 100644 intelmq/bots/experts/time_filter/__init__.py create mode 100644 intelmq/bots/experts/time_filter/expert.py create mode 100644 intelmq/tests/bots/experts/time_filter/__init__.py create mode 100644 intelmq/tests/bots/experts/time_filter/test_expert.py diff --git a/intelmq/bots/experts/time_filter/__init__.py b/intelmq/bots/experts/time_filter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py new file mode 100644 index 000000000..bd6d6e8bb --- /dev/null +++ b/intelmq/bots/experts/time_filter/expert.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +from datetime import datetime, timedelta +from dateutil import parser +from intelmq.lib.utils import get_timedelta +from intelmq.lib.bot import Bot +from datetime import timezone + + +class TimeFilterExpertBot(Bot): + search_field: str = 'time.source' + search_from: str = '1d' + + not_after = None + + def init(self): + self.search_field = self.search_field + if self.search_from: + timedelta_params = get_timedelta(self.search_from) + self.not_after = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) + + def process(self): + event = self.receive_message() + # time based filtering + if self.search_field in event: + try: + event_time = parser.parse(str(event.get(self.search_field))) + except ValueError: + event_time = self.not_after + self.process_message(event_time, event) + return + else: + self.process_message(event_time, event) + return + else: + # not found field + event_time = self.not_after + self.process_message(event_time, event) + return + + def process_message(self, event_time, event): + event_time = event_time.replace(tzinfo=None) + self.not_after = self.not_after.replace(tzinfo=None) + + if event_time < self.not_after: + self.acknowledge_message() + self.logger.debug( + f"Filtered out event with search field {self.search_field} and event time {event_time} .") + return + else: + self.send_message(event) + self.acknowledge_message() + return + + +BOT = TimeFilterExpertBot diff --git a/intelmq/tests/bots/experts/time_filter/__init__.py b/intelmq/tests/bots/experts/time_filter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py new file mode 100644 index 000000000..e8ff0ede0 --- /dev/null +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import unittest +from freezegun import freeze_time + +import intelmq.lib.test as test +from intelmq.bots.experts.time_filter.expert import TimeFilterExpertBot + +EXAMPLE_INPUT_DROP = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "2020-08-13T00:00:00+00:00", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline1": "2020-09-13T00:00:00+00:00", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS_2 = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS_3 = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "2020-09-13", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} + + +class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for TimeFilterExpertBot handling Reports. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = TimeFilterExpertBot + cls.input_message = EXAMPLE_INPUT_DROP + cls.sysconfig = { + 'search_field': 'extra.lastonline', + 'search_from': "1d" + } + + @freeze_time("2021-05-05") + def test_expert_drop(self): + self.run_bot() + self.assertOutputQueueLen(0) + + @freeze_time("2020-09-09") + def test_expert_pass(self): + self.input_message = EXAMPLE_INPUT_PASS + self.run_bot() + self.assertOutputQueueLen(1) + + @freeze_time("2020-09-09") + def test_expert_pass_2(self): + self.input_message = EXAMPLE_INPUT_PASS_2 + self.run_bot() + self.assertOutputQueueLen(1) + + @freeze_time("2020-09-09") + def test_expert_pass_3(self): + self.input_message = EXAMPLE_INPUT_PASS_3 + self.run_bot() + self.assertOutputQueueLen(1) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() From ebd9e99f18214f2dde355637e42da2a6dc63c0c8 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Mon, 14 Jun 2021 16:30:18 +0300 Subject: [PATCH 02/17] Small changes --- intelmq/bots/experts/time_filter/expert.py | 40 ++++++++++------------ 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index bd6d6e8bb..14e2305a7 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -7,25 +7,25 @@ class TimeFilterExpertBot(Bot): - search_field: str = 'time.source' - search_from: str = '1d' + """ Time based filtering """ + field: str = 'time.source' + timespan: str = '1d' - not_after = None + __delta = None def init(self): - self.search_field = self.search_field - if self.search_from: - timedelta_params = get_timedelta(self.search_from) - self.not_after = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) + if self.field: + timedelta_params = get_timedelta(self.timespan) + self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) def process(self): event = self.receive_message() - # time based filtering - if self.search_field in event: + event_time = self.__delta + + if self.field in event: try: - event_time = parser.parse(str(event.get(self.search_field))) + event_time = parser.parse(str(event.get(self.field))) except ValueError: - event_time = self.not_after self.process_message(event_time, event) return else: @@ -33,23 +33,19 @@ def process(self): return else: # not found field - event_time = self.not_after self.process_message(event_time, event) return def process_message(self, event_time, event): event_time = event_time.replace(tzinfo=None) - self.not_after = self.not_after.replace(tzinfo=None) + self.__delta = self.__delta.replace(tzinfo=None) - if event_time < self.not_after: - self.acknowledge_message() - self.logger.debug( - f"Filtered out event with search field {self.search_field} and event time {event_time} .") - return - else: + if event_time > self.__delta: self.send_message(event) - self.acknowledge_message() - return - + else: + self.logger.debug(f"Filtered out event with search field {self.field} and event time {event_time} .") + + self.acknowledge_message() + BOT = TimeFilterExpertBot From 86056a60d8e91f3c6c3f445d8dc9fb7266fb7648 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 4 Aug 2021 14:43:53 +0300 Subject: [PATCH 03/17] Update bot and add documentation --- docs/user/bots.rst | 18 ++++++++++++++++++ intelmq/bots/experts/time_filter/expert.py | 3 +++ 2 files changed, 21 insertions(+) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 4e0a269c0..519a74cdb 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3097,6 +3097,24 @@ Please note: Even if a message is sent, any further identical messages are dropp .. _intelmq.bots.experts.tor_nodes.expert: +Time Filter +^^^^^^^^^ + +**Information** + +* `name:` `intelmq.bots.experts.time_filter.expert` +* `lookup:` no +* `public:` yes +* `cache (redis db):` none +* `description:` Time based filtering + +**Configuration Parameters** + +* `field`: `time.source` +* `timespan`: `1d` + +.. _intelmq.bots.experts.time_filter.expert: + Tor Nodes ^^^^^^^^^ diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 14e2305a7..21df1af70 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +""" +Time based filtering +""" from datetime import datetime, timedelta from dateutil import parser from intelmq.lib.utils import get_timedelta From 167fb370fb1b0d22b268282d5acfc1b36322f39d Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 6 Aug 2021 09:33:23 +0300 Subject: [PATCH 04/17] Add license --- intelmq/bots/experts/time_filter/expert.py | 4 ++++ intelmq/tests/bots/experts/time_filter/test_expert.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 21df1af70..0f02017ce 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- """ Time based filtering + +SPDX-FileCopyrightText: 2021 Marius Karotkis +SPDX-License-Identifier: AGPL-3.0-or-later """ + from datetime import datetime, timedelta from dateutil import parser from intelmq.lib.utils import get_timedelta diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index e8ff0ede0..0929fd385 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -1,4 +1,10 @@ # -*- coding: utf-8 -*- +""" +Time based filtering + +SPDX-FileCopyrightText: 2021 Marius Karotkis +SPDX-License-Identifier: AGPL-3.0-or-later +""" import unittest from freezegun import freeze_time From 760ec91fe70b66e6e4fa7fc7c85b08e2bea3746d Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 09:51:27 +0300 Subject: [PATCH 05/17] Add requirements, small fix --- docs/user/bots.rst | 5 +++-- .../bots/experts/time_filter/REQUIREMENTS.txt | 1 + intelmq/bots/experts/time_filter/expert.py | 16 ++++++++++++---- .../bots/experts/time_filter/REQUIREMENTS.txt | 1 + .../bots/experts/time_filter/test_expert.py | 13 ++++++++++++- 5 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 intelmq/bots/experts/time_filter/REQUIREMENTS.txt create mode 100644 intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 519a74cdb..31f2ff43d 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3095,7 +3095,7 @@ This bot has certain limitations and is not a true threshold filter (yet). It wo Please note: Even if a message is sent, any further identical messages are dropped, if the time difference to the last message is less than the timeout! The counter is not reset if the threshold is reached. -.. _intelmq.bots.experts.tor_nodes.expert: +.. _intelmq.bots.experts.time_filter.expert: Time Filter ^^^^^^^^^ @@ -3113,7 +3113,8 @@ Time Filter * `field`: `time.source` * `timespan`: `1d` -.. _intelmq.bots.experts.time_filter.expert: + +.. _intelmq.bots.experts.tor_nodes.expert: Tor Nodes ^^^^^^^^^ diff --git a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt new file mode 100644 index 000000000..39ca0b49b --- /dev/null +++ b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt @@ -0,0 +1 @@ +get_timedelta diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 0f02017ce..eecb6335a 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -8,20 +8,28 @@ from datetime import datetime, timedelta from dateutil import parser -from intelmq.lib.utils import get_timedelta from intelmq.lib.bot import Bot from datetime import timezone +from intelmq.lib.exceptions import MissingDependencyError + +try: + from intelmq.lib.utils import get_timedelta +except ImportError: + get_timedelta = None class TimeFilterExpertBot(Bot): """ Time based filtering """ field: str = 'time.source' - timespan: str = '1d' + timespan: str = '1d' __delta = None def init(self): if self.field: + if get_timedelta is None: + raise MissingDependencyError("get_timedelta") + timedelta_params = get_timedelta(self.timespan) self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) @@ -51,8 +59,8 @@ def process_message(self, event_time, event): self.send_message(event) else: self.logger.debug(f"Filtered out event with search field {self.field} and event time {event_time} .") - + self.acknowledge_message() - + BOT = TimeFilterExpertBot diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt new file mode 100644 index 000000000..fa5ae1fef --- /dev/null +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -0,0 +1 @@ +freeze_time \ No newline at end of file diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 0929fd385..97a964cd1 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -7,10 +7,15 @@ """ import unittest -from freezegun import freeze_time import intelmq.lib.test as test from intelmq.bots.experts.time_filter.expert import TimeFilterExpertBot +from intelmq.lib.exceptions import MissingDependencyError + +try: + from freezegun import freeze_time +except ImportError: + freeze_time = None EXAMPLE_INPUT_DROP = { "__type": "Event", @@ -70,11 +75,17 @@ } +@test.skip_exotic() class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): """ A TestCase for TimeFilterExpertBot handling Reports. """ + @classmethod + def init(cls): + if freeze_time is None: + raise MissingDependencyError("freeze_time") + @classmethod def set_bot(cls): cls.bot_reference = TimeFilterExpertBot From b6cdad4cca13eb8cc1386f7b28ad8fe9be79735e Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 09:57:32 +0300 Subject: [PATCH 06/17] Add license to requirements --- intelmq/bots/experts/time_filter/REQUIREMENTS.txt | 3 +++ intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt index 39ca0b49b..71d1b21b1 100644 --- a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt @@ -1 +1,4 @@ +# SPDX-FileCopyrightText: 2021 Marius Karotkis +# SPDX-License-Identifier: AGPL-3.0-or-later + get_timedelta diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index fa5ae1fef..7ae9c7038 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1 +1,4 @@ -freeze_time \ No newline at end of file +# SPDX-FileCopyrightText: 2021 Marius Karotkis +# SPDX-License-Identifier: AGPL-3.0-or-later + +freeze_time From 0f615a8b60d341eed23ea4652680596643a1e82a Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 10:06:47 +0300 Subject: [PATCH 07/17] Add to utils get_timedelta --- intelmq/bots/experts/time_filter/REQUIREMENTS.txt | 4 ---- intelmq/bots/experts/time_filter/expert.py | 10 +--------- intelmq/lib/utils.py | 10 ++++++++++ 3 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 intelmq/bots/experts/time_filter/REQUIREMENTS.txt diff --git a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt deleted file mode 100644 index 71d1b21b1..000000000 --- a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Marius Karotkis -# SPDX-License-Identifier: AGPL-3.0-or-later - -get_timedelta diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index eecb6335a..2b431ec51 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -10,12 +10,7 @@ from dateutil import parser from intelmq.lib.bot import Bot from datetime import timezone -from intelmq.lib.exceptions import MissingDependencyError - -try: - from intelmq.lib.utils import get_timedelta -except ImportError: - get_timedelta = None +from intelmq.lib.utils import get_timedelta class TimeFilterExpertBot(Bot): @@ -27,9 +22,6 @@ class TimeFilterExpertBot(Bot): def init(self): if self.field: - if get_timedelta is None: - raise MissingDependencyError("get_timedelta") - timedelta_params = get_timedelta(self.timespan) self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index d9699ff67..824235191 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -910,3 +910,13 @@ def get_bots_settings(bot_id: str = None) -> dict: if 'global' in runtime_conf: del runtime_conf['global'] return runtime_conf + +def get_timedelta(search_last): + m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) + if not m: raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") + date_mapping = { + "d": "days", + "h": "hours", + "m": "minutes" + } + return {date_mapping[m.group('map')]: int(m.group("number"))} \ No newline at end of file From 2e35b650ba0777f6e372ebfa087dec6360efe722 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 10:09:10 +0300 Subject: [PATCH 08/17] Fix codestyle --- intelmq/lib/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 824235191..98fb31bb9 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -850,7 +850,8 @@ def list_all_bots() -> dict: base_path = resource_filename('intelmq', 'bots') - botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if botfile.is_file() and botfile.name != '__init__.py'] + botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if + botfile.is_file() and botfile.name != '__init__.py'] for file in botfiles: file = Path(file.as_posix().replace(base_path, 'intelmq/bots')) mod = importlib.import_module('.'.join(file.with_suffix('').parts)) @@ -870,7 +871,8 @@ def list_all_bots() -> dict: bots[file.parts[2].capitalize()[:-1]][name] = { "module": mod.__name__, - "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent(mod.BOT.__doc__), + "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent( + mod.BOT.__doc__), "parameters": keys, } return bots @@ -911,6 +913,7 @@ def get_bots_settings(bot_id: str = None) -> dict: del runtime_conf['global'] return runtime_conf + def get_timedelta(search_last): m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) if not m: raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") @@ -919,4 +922,4 @@ def get_timedelta(search_last): "h": "hours", "m": "minutes" } - return {date_mapping[m.group('map')]: int(m.group("number"))} \ No newline at end of file + return {date_mapping[m.group('map')]: int(m.group("number"))} From dd4e1387cc37b52496c53e850027de791d207e1d Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 16:10:55 +0300 Subject: [PATCH 09/17] Fix --- intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index 7ae9c7038..f81fab6da 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2021 Marius Karotkis # SPDX-License-Identifier: AGPL-3.0-or-later -freeze_time +freezegun From 9d60d5b2923abc822e7649f292a73a5d6f3d71e4 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:00:41 +0300 Subject: [PATCH 10/17] Fix --- intelmq/lib/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 0b0b739c4..79b5dd836 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -922,7 +922,8 @@ def get_bots_settings(bot_id: str = None) -> dict: def get_timedelta(search_last): m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) - if not m: raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") + if not m: + raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") date_mapping = { "d": "days", "h": "hours", From f111f9e2c6d3053f2223acb28955e2ac37922353 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:19:19 +0300 Subject: [PATCH 11/17] Fix --- intelmq/tests/bots/experts/time_filter/test_expert.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 97a964cd1..c9566745a 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -95,23 +95,27 @@ def set_bot(cls): 'search_from': "1d" } + @test.skip_exotic() @freeze_time("2021-05-05") def test_expert_drop(self): self.run_bot() self.assertOutputQueueLen(0) + @test.skip_exotic() @freeze_time("2020-09-09") def test_expert_pass(self): self.input_message = EXAMPLE_INPUT_PASS self.run_bot() self.assertOutputQueueLen(1) + @test.skip_exotic() @freeze_time("2020-09-09") def test_expert_pass_2(self): self.input_message = EXAMPLE_INPUT_PASS_2 self.run_bot() self.assertOutputQueueLen(1) + @test.skip_exotic() @freeze_time("2020-09-09") def test_expert_pass_3(self): self.input_message = EXAMPLE_INPUT_PASS_3 From 811871b478cad0af52dc0e3133fb00febc7feb27 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:48:53 +0300 Subject: [PATCH 12/17] Change freezegun to time_machine --- .../bots/experts/time_filter/REQUIREMENTS.txt | 2 +- .../bots/experts/time_filter/test_expert.py | 24 +++++++------------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index f81fab6da..3e3e4172c 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2021 Marius Karotkis # SPDX-License-Identifier: AGPL-3.0-or-later -freezegun +time_machine diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index c9566745a..6d9a471dd 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -13,9 +13,9 @@ from intelmq.lib.exceptions import MissingDependencyError try: - from freezegun import freeze_time + import time_machine except ImportError: - freeze_time = None + time_machine = None EXAMPLE_INPUT_DROP = { "__type": "Event", @@ -81,13 +81,11 @@ class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): A TestCase for TimeFilterExpertBot handling Reports. """ - @classmethod - def init(cls): - if freeze_time is None: - raise MissingDependencyError("freeze_time") - @classmethod def set_bot(cls): + if time_machine is None: + raise MissingDependencyError("time_machine") + cls.bot_reference = TimeFilterExpertBot cls.input_message = EXAMPLE_INPUT_DROP cls.sysconfig = { @@ -95,28 +93,24 @@ def set_bot(cls): 'search_from': "1d" } - @test.skip_exotic() - @freeze_time("2021-05-05") + @time_machine.travel("2021-05-05") def test_expert_drop(self): self.run_bot() self.assertOutputQueueLen(0) - @test.skip_exotic() - @freeze_time("2020-09-09") + @time_machine.travel("2020-09-09") def test_expert_pass(self): self.input_message = EXAMPLE_INPUT_PASS self.run_bot() self.assertOutputQueueLen(1) - @test.skip_exotic() - @freeze_time("2020-09-09") + @time_machine.travel("2020-09-09") def test_expert_pass_2(self): self.input_message = EXAMPLE_INPUT_PASS_2 self.run_bot() self.assertOutputQueueLen(1) - @test.skip_exotic() - @freeze_time("2020-09-09") + @time_machine.travel("2020-09-09") def test_expert_pass_3(self): self.input_message = EXAMPLE_INPUT_PASS_3 self.run_bot() From 9d41b4a2d4b8031ec0eec77b655eb5450ff26cc7 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:58:01 +0300 Subject: [PATCH 13/17] fix time-machine --- intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index 3e3e4172c..b00113cde 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2021 Marius Karotkis # SPDX-License-Identifier: AGPL-3.0-or-later -time_machine +time-machine From 2d8f636ed080242e78d5e2cc6ee765ed5f7ce9d4 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 25 Aug 2021 15:20:17 +0300 Subject: [PATCH 14/17] Fix tests --- .../bots/experts/time_filter/test_expert.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 6d9a471dd..5c0538148 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -93,28 +93,29 @@ def set_bot(cls): 'search_from': "1d" } - @time_machine.travel("2021-05-05") - def test_expert_drop(self): - self.run_bot() - self.assertOutputQueueLen(0) + if time_machine: + @time_machine.travel("2021-05-05") + def test_expert_drop(self): + self.run_bot() + self.assertOutputQueueLen(0) - @time_machine.travel("2020-09-09") - def test_expert_pass(self): - self.input_message = EXAMPLE_INPUT_PASS - self.run_bot() - self.assertOutputQueueLen(1) + @time_machine.travel("2020-09-09") + def test_expert_pass(self): + self.input_message = EXAMPLE_INPUT_PASS + self.run_bot() + self.assertOutputQueueLen(1) - @time_machine.travel("2020-09-09") - def test_expert_pass_2(self): - self.input_message = EXAMPLE_INPUT_PASS_2 - self.run_bot() - self.assertOutputQueueLen(1) + @time_machine.travel("2020-09-09") + def test_expert_pass_2(self): + self.input_message = EXAMPLE_INPUT_PASS_2 + self.run_bot() + self.assertOutputQueueLen(1) - @time_machine.travel("2020-09-09") - def test_expert_pass_3(self): - self.input_message = EXAMPLE_INPUT_PASS_3 - self.run_bot() - self.assertOutputQueueLen(1) + @time_machine.travel("2020-09-09") + def test_expert_pass_3(self): + self.input_message = EXAMPLE_INPUT_PASS_3 + self.run_bot() + self.assertOutputQueueLen(1) if __name__ == '__main__': # pragma: no cover From a20be810d4704bfe13c9d375735ce60a9164b2dc Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 25 Aug 2021 15:34:54 +0300 Subject: [PATCH 15/17] Fix tests --- intelmq/tests/bots/experts/time_filter/test_expert.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 5c0538148..8239ae34d 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -83,9 +83,6 @@ class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(cls): - if time_machine is None: - raise MissingDependencyError("time_machine") - cls.bot_reference = TimeFilterExpertBot cls.input_message = EXAMPLE_INPUT_DROP cls.sysconfig = { From adb68ee2e93df31ff7b06305ee77f139ebbf2d41 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 27 Aug 2021 13:04:24 +0300 Subject: [PATCH 16/17] Add description --- docs/user/bots.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index dcacf1764..a2f09d1d9 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3205,7 +3205,6 @@ Threshold **Information** - * **Cache parameters** (see section :ref:`common-parameters`) * `name`: `intelmq.bots.experts.threshold.expert` * `lookup`: redis cache @@ -3244,6 +3243,8 @@ Please note: Even if a message is sent, any further identical messages are dropp Time Filter ^^^^^^^^^ +Filtering records that do not exceed the specified deadline. + **Information** * `name:` `intelmq.bots.experts.time_filter.expert` @@ -3254,8 +3255,12 @@ Time Filter **Configuration Parameters** -* `field`: `time.source` -* `timespan`: `1d` +* `field`: the event field which should be filtered on. Needs to be in ISO 8601 formatted. Default: `time.source` +* `timespan`: time interval for filtering records. Default: `1d` + +**Description** + +The bot filters new records, for example when only records from the last 2 days are needed. Also allows entries without a date. .. _intelmq.bots.experts.tor_nodes.expert: From cd41494affd907a20032021772709c807349c1d0 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 27 Aug 2021 13:23:39 +0300 Subject: [PATCH 17/17] Remove similar function --- docs/user/bots.rst | 4 ++-- intelmq/bots/experts/time_filter/expert.py | 7 +++---- intelmq/lib/utils.py | 12 ------------ .../tests/bots/experts/time_filter/test_expert.py | 2 +- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index a2f09d1d9..bf4c528a5 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3256,11 +3256,11 @@ Filtering records that do not exceed the specified deadline. **Configuration Parameters** * `field`: the event field which should be filtered on. Needs to be in ISO 8601 formatted. Default: `time.source` -* `timespan`: time interval for filtering records. Default: `1d` +* `timespan`: time interval for filtering records. Default: `24 hours` **Description** -The bot filters new records, for example when only records from the last 2 days are needed. Also allows entries without a date. +The bot filters new records, for example when only records from the last 2 days (48 hours) are needed. Also allows entries without a date. .. _intelmq.bots.experts.tor_nodes.expert: diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 2b431ec51..ce0739566 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -10,20 +10,19 @@ from dateutil import parser from intelmq.lib.bot import Bot from datetime import timezone -from intelmq.lib.utils import get_timedelta +from intelmq.lib.utils import parse_relative class TimeFilterExpertBot(Bot): """ Time based filtering """ field: str = 'time.source' - timespan: str = '1d' + timespan: str = '24 hours' __delta = None def init(self): if self.field: - timedelta_params = get_timedelta(self.timespan) - self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) + self.__delta = datetime.now(tz=timezone.utc) - timedelta(minutes=parse_relative(self.timespan)) def process(self): event = self.receive_message() diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 79b5dd836..8f0eb814c 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -918,15 +918,3 @@ def get_bots_settings(bot_id: str = None) -> dict: if 'global' in runtime_conf: del runtime_conf['global'] return runtime_conf - - -def get_timedelta(search_last): - m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) - if not m: - raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") - date_mapping = { - "d": "days", - "h": "hours", - "m": "minutes" - } - return {date_mapping[m.group('map')]: int(m.group("number"))} diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 8239ae34d..5f47b96b0 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -87,7 +87,7 @@ def set_bot(cls): cls.input_message = EXAMPLE_INPUT_DROP cls.sysconfig = { 'search_field': 'extra.lastonline', - 'search_from': "1d" + 'search_from': "72 hours" } if time_machine: