From 797289f9587e527813e8e8330d93abfa17ee223c Mon Sep 17 00:00:00 2001 From: Egor Vasilyev Date: Thu, 28 Apr 2022 16:46:12 +0300 Subject: [PATCH] #6: added ability to pass custom dns servers to resolver (#7) added ability to pass custom dns servers to resolver (it's important because exporter may be ran not in Russia while we still should use Russia resolvers) --- README.md | 2 ++ docker-compose.yml | 1 + handler.py | 16 ++++++++++++++++ kubernetes/config.yaml | 3 ++- kubernetes/templates/deployment.yaml.j2 | 6 ++++-- rkn_exporter.py | 10 +++++++--- tests/test_handler.py | 9 ++++++++- 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ee909a6..3dcc026 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ PORT=8080 ; docker run -it --rm --name rkn-exporter \ -e APP_DOMAINS=/app/inputs/domains.txt \ -e APP_SUBNETS=/app/inputs/blocked_subnets.txt \ -e APP_THREADS_COUNT=10 \ + -e APP_DNS=8.8.8.8 \ -e LOG_LEVEL=DEBUG \ bissquit/rkn-exporter:latest ``` @@ -92,6 +93,7 @@ This command is not needed if you configure env with `make env`. |-d, --domains|`APP_DOMAINS`|Path to a file with domains to check. One domain per line (default: No)| |-s, --blocked_subnets|`APP_SUBNETS`|Path to a file with subnets bloked by RKN. One subnet per line. Or url with json list (default: No)| |-t, --threads_count|`APP_THREADS_COUNT`|Threads count to parallelize computation. Is useful when DNS resolving is slow (default: 10)| +|--dns|`APP_DNS`|DNS servers (default: 8.8.8.8)| |-|`LOG_LEVEL`|Log level based on Python [logging](https://docs.python.org/3/library/logging.html) module. expected values: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO)| diff --git a/docker-compose.yml b/docker-compose.yml index 91d277b..1b8251d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: # rkn-exporter receives file or url - APP_SUBNETS=/app/inputs/blocked_subnets.txt - APP_THREADS_COUNT=10 + - APP_DNS=8.8.8.8 - LOG_LEVEL=DEBUG volumes: - ./inputs:/app/inputs diff --git a/handler.py b/handler.py index fd856ac..6e78d43 100644 --- a/handler.py +++ b/handler.py @@ -270,3 +270,19 @@ async def data_handler(path: str) -> set: raw_data = read_file_to_list(path=path) return set(ip_converter(raw_data)) + + +def normalize_dns(dns_str: str) -> list: + # remove any whitespaces from string first then split string into a list + # comma is default delimiter + dns_servers_list = (dns_str.replace(' ', '')).split(',') + + valid_list = [] + for dns_server_str in dns_servers_list: + # check if string is a valid ipv4 address + if validators.ipv4(dns_server_str): + valid_list.append(dns_server_str) + else: + logger.debug(f'{dns_server_str} is not a valid ip address of DNS server!') + logger.debug(f'DNS server(s) to proceed: {valid_list}') + return valid_list diff --git a/kubernetes/config.yaml b/kubernetes/config.yaml index 82c78b1..32e082c 100644 --- a/kubernetes/config.yaml +++ b/kubernetes/config.yaml @@ -5,7 +5,7 @@ common: k8s_namespace: monitoring # type not only image tag but hash too - rkn_exporter_version: 'v1.3-dev@sha256:df72a1eb422ba07fc42686b0bc7198a4c953ddbbefe6ebb0a5c9469d70d24e59' + rkn_exporter_version: 'v1.6@sha256:65b475b4e3102f901f9d8d5aba1239bf18bc3e268009f6cf528f7d276fe73c14' # default settings. You may rewrite them for certain k8s cluster rkn_exporter: ip: "0.0.0.0" @@ -14,6 +14,7 @@ common: domains: "/app/inputs/domains.txt" subnets: "https://reestr.rublacklist.net/api/v2/ips/json/" threads_count: "10" + dns: "8.8.8.8" log_level: "DEBUG" env-name: diff --git a/kubernetes/templates/deployment.yaml.j2 b/kubernetes/templates/deployment.yaml.j2 index 8f8b97f..f38814c 100644 --- a/kubernetes/templates/deployment.yaml.j2 +++ b/kubernetes/templates/deployment.yaml.j2 @@ -31,14 +31,16 @@ spec: value: "{{ rkn_exporter.subnets }}" - name: APP_THREADS_COUNT value: "{{ rkn_exporter.threads_count }}" + - name: APP_DNS + value: "{{ rkn_exporter.dns }}" - name: LOG_LEVEL value: "{{ rkn_exporter.log_level }}" resources: requests: - cpu: 40m + cpu: 100m memory: 64Mi limits: - cpu: 80m + cpu: 200m memory: 128Mi volumeMounts: - name: inputs diff --git a/rkn_exporter.py b/rkn_exporter.py index c293b82..2ea4568 100755 --- a/rkn_exporter.py +++ b/rkn_exporter.py @@ -13,7 +13,8 @@ return_metrics, \ fill_queue, \ subnets_to_ips, \ - data_handler + data_handler, \ + normalize_dns # possibly it's good idea to use contextvars here data = 'rkn_computation_success 0' @@ -52,6 +53,10 @@ def parse_args(): default=os.getenv("APP_THREADS_COUNT", 10), type=int, help='Threads count to parallelize computation. Is useful when DNS resolving is slow (default: 10)') + parser.add_argument('--dns', + default=os.getenv("APP_DNS", '8.8.8.8'), + type=str, + help='DNS servers (default: 8.8.8.8)') return parser.parse_args() @@ -97,8 +102,7 @@ async def handler(self): blocked_subnets_set = await data_handler(self.args.blocked_subnets) blocked_ips_set = subnets_to_ips(blocked_subnets_set) - # I'll add variables later - resolver = self.initialize_resolver(nameservers=['8.8.8.8'], + resolver = self.initialize_resolver(nameservers=normalize_dns(self.args.dns), timeout=20, lifetime=20, retry_servfail=False) diff --git a/tests/test_handler.py b/tests/test_handler.py index 7feaffb..dffb292 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -17,7 +17,7 @@ return_domain_metrics, \ read_file_to_list, \ validate_domains, \ - fill_queue, subnet_to_ips, subnets_to_ips, get_data, ip_converter, data_handler + fill_queue, subnet_to_ips, subnets_to_ips, get_data, ip_converter, data_handler, normalize_dns # taken from https://github.com/aio-libs/aiohttp/blob/master/tests/test_resolver.py @@ -282,3 +282,10 @@ async def test_data_handler(mocker): mocker.patch('handler.read_file_to_list', return_value=await mock_awaitable_obj(raw_ips_list)) blocked_subnets_set = await data_handler('./fake/path/to/file') assert blocked_subnets_set == valid_subnets_set + + +def test_normalize_dns(): + input_str = ' 8.8.8.8 ,8.8.4.4, 8.8.1.1111' + + valid_dns_list = normalize_dns(input_str) + assert valid_dns_list == ['8.8.8.8', '8.8.4.4']