Skip to content

Commit 3bca21a

Browse files
patchback[bot]vbotka
patchback[bot]
and
vbotka
authored
[PR #8456/6f8f12f7 backport][stable-9] Feature filter keep_keys (#8462)
Feature filter keep_keys (#8456) * Add filter keep_keys. Implement feature request #8438 * Fix comment indentation. * Fix regex reference. * Fix indentation. * Fix isinstance list. * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein <[email protected]> * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein <[email protected]> * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein <[email protected]> * Update plugins/plugin_utils/keys_filter.py Co-authored-by: Felix Fontein <[email protected]> * Update plugins/filter/keep_keys.py Co-authored-by: Felix Fontein <[email protected]> * Update documentation, examples, and integration tests. * _keys_filter_target_str returns tuple of unique target strings if target is list. Update documentation, function comments, and error messages. * Sort maintainers. * Update plugins/filter/keep_keys.py Co-authored-by: Felix Fontein <[email protected]> * Update examples with explicit collection. --------- Co-authored-by: Felix Fontein <[email protected]> (cherry picked from commit 6f8f12f) Co-authored-by: Vladimir Botka <[email protected]>
1 parent 1bb3d41 commit 3bca21a

File tree

7 files changed

+374
-0
lines changed

7 files changed

+374
-0
lines changed

‎.github/BOTMETA.yml

+4
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,8 @@ files:
157157
$filters/jc.py:
158158
maintainers: kellyjonbrazil
159159
$filters/json_query.py: {}
160+
$filters/keep_keys.py:
161+
maintainers: vbotka
160162
$filters/lists.py:
161163
maintainers: cfiehe
162164
$filters/lists_difference.yml:
@@ -1417,6 +1419,8 @@ files:
14171419
ignore: matze
14181420
labels: zypper
14191421
maintainers: $team_suse
1422+
$plugin_utils/keys_filter.py:
1423+
maintainers: vbotka
14201424
$plugin_utils/unsafe.py:
14211425
maintainers: felixfontein
14221426
$tests/a_module.py:

‎plugins/filter/keep_keys.py

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# -*- coding: utf-8 -*-
2+
# Copyright (c) 2024 Vladimir Botka <[email protected]>
3+
# Copyright (c) 2024 Felix Fontein <[email protected]>
4+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
5+
# SPDX-License-Identifier: GPL-3.0-or-later
6+
7+
from __future__ import (absolute_import, division, print_function)
8+
__metaclass__ = type
9+
10+
DOCUMENTATION = '''
11+
name: keep_keys
12+
short_description: Keep specific keys from dictionaries in a list
13+
version_added: "9.1.0"
14+
author:
15+
- Vladimir Botka (@vbotka)
16+
- Felix Fontein (@felixfontein)
17+
description: This filter keeps only specified keys from a provided list of dictionaries.
18+
options:
19+
_input:
20+
description:
21+
- A list of dictionaries.
22+
- Top level keys must be strings.
23+
type: list
24+
elements: dictionary
25+
required: true
26+
target:
27+
description:
28+
- A single key or key pattern to keep, or a list of keys or keys patterns to keep.
29+
- If O(matching_parameter=regex) there must be exactly one pattern provided.
30+
type: raw
31+
required: true
32+
matching_parameter:
33+
description: Specify the matching option of target keys.
34+
type: str
35+
default: equal
36+
choices:
37+
equal: Matches keys of exactly one of the O(target) items.
38+
starts_with: Matches keys that start with one of the O(target) items.
39+
ends_with: Matches keys that end with one of the O(target) items.
40+
regex:
41+
- Matches keys that match the regular expresion provided in O(target).
42+
- In this case, O(target) must be a regex string or a list with single regex string.
43+
'''
44+
45+
EXAMPLES = '''
46+
l:
47+
- {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
48+
- {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
49+
50+
# 1) By default match keys that equal any of the items in the target.
51+
t: [k0_x0, k1_x1]
52+
r: "{{ l | community.general.keep_keys(target=t) }}"
53+
54+
# 2) Match keys that start with any of the items in the target.
55+
t: [k0, k1]
56+
r: "{{ l | community.general.keep_keys(target=t, matching_parameter='starts_with') }}"
57+
58+
# 3) Match keys that end with any of the items in target.
59+
t: [x0, x1]
60+
r: "{{ l | community.general.keep_keys(target=t, matching_parameter='ends_with') }}"
61+
62+
# 4) Match keys by the regex.
63+
t: ['^.*[01]_x.*$']
64+
r: "{{ l | community.general.keep_keys(target=t, matching_parameter='regex') }}"
65+
66+
# 5) Match keys by the regex.
67+
t: '^.*[01]_x.*$'
68+
r: "{{ l | community.general.keep_keys(target=t, matching_parameter='regex') }}"
69+
70+
# The results of above examples 1-5 are all the same.
71+
r:
72+
- {k0_x0: A0, k1_x1: B0}
73+
- {k0_x0: A1, k1_x1: B1}
74+
75+
# 6) By default match keys that equal the target.
76+
t: k0_x0
77+
r: "{{ l | community.general.keep_keys(target=t) }}"
78+
79+
# 7) Match keys that start with the target.
80+
t: k0
81+
r: "{{ l | community.general.keep_keys(target=t, matching_parameter='starts_with') }}"
82+
83+
# 8) Match keys that end with the target.
84+
t: x0
85+
r: "{{ l | community.general.keep_keys(target=t, matching_parameter='ends_with') }}"
86+
87+
# 9) Match keys by the regex.
88+
t: '^.*0_x.*$'
89+
r: "{{ l | community.general.keep_keys(target=t, matching_parameter='regex') }}"
90+
91+
# The results of above examples 6-9 are all the same.
92+
r:
93+
- {k0_x0: A0}
94+
- {k0_x0: A1}
95+
'''
96+
97+
RETURN = '''
98+
_value:
99+
description: The list of dictionaries with selected keys.
100+
type: list
101+
elements: dictionary
102+
'''
103+
104+
from ansible_collections.community.general.plugins.plugin_utils.keys_filter import (
105+
_keys_filter_params,
106+
_keys_filter_target_str)
107+
108+
109+
def keep_keys(data, target=None, matching_parameter='equal'):
110+
"""keep specific keys from dictionaries in a list"""
111+
112+
# test parameters
113+
_keys_filter_params(data, target, matching_parameter)
114+
# test and transform target
115+
tt = _keys_filter_target_str(target, matching_parameter)
116+
117+
if matching_parameter == 'equal':
118+
def keep_key(key):
119+
return key in tt
120+
elif matching_parameter == 'starts_with':
121+
def keep_key(key):
122+
return key.startswith(tt)
123+
elif matching_parameter == 'ends_with':
124+
def keep_key(key):
125+
return key.endswith(tt)
126+
elif matching_parameter == 'regex':
127+
def keep_key(key):
128+
return tt.match(key) is not None
129+
130+
return [dict((k, v) for k, v in d.items() if keep_key(k)) for d in data]
131+
132+
133+
class FilterModule(object):
134+
135+
def filters(self):
136+
return {
137+
'keep_keys': keep_keys,
138+
}

‎plugins/plugin_utils/keys_filter.py

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# Copyright (c) 2024 Vladimir Botka <[email protected]>
2+
# Copyright (c) 2024 Felix Fontein <[email protected]>
3+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
4+
# SPDX-License-Identifier: GPL-3.0-or-later
5+
6+
from __future__ import (absolute_import, division, print_function)
7+
__metaclass__ = type
8+
9+
import re
10+
11+
from ansible.errors import AnsibleFilterError
12+
from ansible.module_utils.six import string_types
13+
from ansible.module_utils.common._collections_compat import Mapping, Sequence
14+
15+
16+
def _keys_filter_params(data, target, matching_parameter):
17+
"""test parameters:
18+
* data must be a list of dictionaries. All keys must be strings.
19+
* target must be a non-empty sequence.
20+
* matching_parameter is member of a list.
21+
"""
22+
23+
mp = matching_parameter
24+
ml = ['equal', 'starts_with', 'ends_with', 'regex']
25+
26+
if not isinstance(data, Sequence):
27+
msg = "First argument must be a list. %s is %s"
28+
raise AnsibleFilterError(msg % (data, type(data)))
29+
30+
for elem in data:
31+
if not isinstance(elem, Mapping):
32+
msg = "The data items must be dictionaries. %s is %s"
33+
raise AnsibleFilterError(msg % (elem, type(elem)))
34+
35+
for elem in data:
36+
if not all(isinstance(item, string_types) for item in elem.keys()):
37+
msg = "Top level keys must be strings. keys: %s"
38+
raise AnsibleFilterError(msg % elem.keys())
39+
40+
if not isinstance(target, Sequence):
41+
msg = ("The target must be a string or a list. target is %s.")
42+
raise AnsibleFilterError(msg % target)
43+
44+
if len(target) == 0:
45+
msg = ("The target can't be empty.")
46+
raise AnsibleFilterError(msg)
47+
48+
if mp not in ml:
49+
msg = ("The matching_parameter must be one of %s. matching_parameter is %s")
50+
raise AnsibleFilterError(msg % (ml, mp))
51+
52+
return
53+
54+
55+
def _keys_filter_target_str(target, matching_parameter):
56+
"""test:
57+
* If target is list all items are strings
58+
* If matching_parameter=regex target is a string or list with single string
59+
convert and return:
60+
* tuple of unique target items, or
61+
* tuple with single item, or
62+
* compiled regex if matching_parameter=regex
63+
"""
64+
65+
if isinstance(target, list):
66+
for elem in target:
67+
if not isinstance(elem, string_types):
68+
msg = "The target items must be strings. %s is %s"
69+
raise AnsibleFilterError(msg % (elem, type(elem)))
70+
71+
if matching_parameter == 'regex':
72+
if isinstance(target, string_types):
73+
r = target
74+
else:
75+
if len(target) > 1:
76+
msg = ("Single item is required in the target list if matching_parameter is regex.")
77+
raise AnsibleFilterError(msg)
78+
else:
79+
r = target[0]
80+
try:
81+
tt = re.compile(r)
82+
except re.error:
83+
msg = ("The target must be a valid regex if matching_parameter is regex."
84+
" target is %s")
85+
raise AnsibleFilterError(msg % r)
86+
elif isinstance(target, string_types):
87+
tt = (target, )
88+
else:
89+
tt = tuple(set(target))
90+
91+
return tt
92+
93+
94+
def _keys_filter_target_dict(target, matching_parameter):
95+
"""test:
96+
* target is a list of dictionaries
97+
* ...
98+
"""
99+
100+
# TODO: Complete and use this in filter replace_keys
101+
102+
if isinstance(target, list):
103+
for elem in target:
104+
if not isinstance(elem, Mapping):
105+
msg = "The target items must be dictionary. %s is %s"
106+
raise AnsibleFilterError(msg % (elem, type(elem)))
107+
108+
return

‎tests/integration/targets/filter_keep_keys/aliases

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Copyright (c) Ansible Project
2+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
3+
# SPDX-License-Identifier: GPL-3.0-or-later
4+
5+
azp/posix/2

‎tests/integration/targets/filter_keep_keys/tasks/keep_keys.yml

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
---
2+
# Copyright (c) Ansible Project
3+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
4+
# SPDX-License-Identifier: GPL-3.0-or-later
5+
6+
- name: Debug ansible_version
7+
ansible.builtin.debug:
8+
var: ansible_version
9+
when: not quite_test | d(true) | bool
10+
tags: ansible_version
11+
12+
- name: Test keep keys equal (default)
13+
ansible.builtin.assert:
14+
that:
15+
- (rr | difference(result1) | length) == 0
16+
success_msg: |
17+
[OK] result:
18+
{{ rr | to_yaml }}
19+
fail_msg: |
20+
[ERR] result:
21+
{{ rr | to_yaml }}
22+
quiet: "{{ quiet_test | d(true) | bool }}"
23+
vars:
24+
rr: "{{ list1 | community.general.keep_keys(target=tt) }}"
25+
tt: [k0_x0, k1_x1]
26+
tags: equal_default
27+
28+
- name: Test keep keys regex string
29+
ansible.builtin.assert:
30+
that:
31+
- (rr | difference(result1) | length) == 0
32+
success_msg: |
33+
[OK] result:
34+
{{ rr | to_yaml }}
35+
fail_msg: |
36+
[ERR] result:
37+
{{ rr | to_yaml }}
38+
quiet: "{{ quiet_test | d(true) | bool }}"
39+
vars:
40+
rr: "{{ list1 | community.general.keep_keys(target=tt, matching_parameter=mp) }}"
41+
mp: regex
42+
tt: '^.*[01]_x.*$'
43+
tags: regex_string
44+
45+
- name: Test keep keys targets1
46+
ansible.builtin.assert:
47+
that:
48+
- (rr | difference(result1) | length) == 0
49+
success_msg: |
50+
[OK] result:
51+
{{ rr | to_yaml }}
52+
fail_msg: |
53+
[ERR] result:
54+
{{ rr | to_yaml }}
55+
quiet: "{{ quiet_test | d(true) | bool }}"
56+
loop: "{{ targets1 }}"
57+
loop_control:
58+
label: "{{ item.mp }}: {{ item.tt }}"
59+
vars:
60+
rr: "{{ list1 | community.general.keep_keys(target=item.tt, matching_parameter=item.mp) }}"
61+
tags: targets1
62+
63+
- name: Test keep keys targets2
64+
ansible.builtin.assert:
65+
that:
66+
- (rr | difference(result2) | length) == 0
67+
success_msg: |
68+
[OK] result:
69+
{{ rr | to_yaml }}
70+
fail_msg: |
71+
[ERR] result:
72+
{{ rr | to_yaml }}
73+
quiet: "{{ quiet_test | d(true) | bool }}"
74+
loop: "{{ targets2 }}"
75+
loop_control:
76+
label: "{{ item.mp }}: {{ item.tt }}"
77+
vars:
78+
rr: "{{ list2 | community.general.keep_keys(target=item.tt, matching_parameter=item.mp) }}"
79+
tags: targets2

‎tests/integration/targets/filter_keep_keys/tasks/main.yml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
# Copyright (c) Ansible Project
3+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
4+
# SPDX-License-Identifier: GPL-3.0-or-later
5+
6+
- name: Test keep_keys
7+
import_tasks: keep_keys.yml

‎tests/integration/targets/filter_keep_keys/vars/main.yml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
---
2+
# Copyright (c) Ansible Project
3+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
4+
# SPDX-License-Identifier: GPL-3.0-or-later
5+
6+
targets1:
7+
- {mp: equal, tt: [k0_x0, k1_x1]}
8+
- {mp: starts_with, tt: [k0, k1]}
9+
- {mp: ends_with, tt: [x0, x1]}
10+
- {mp: regex, tt: ['^.*[01]_x.*$']}
11+
- {mp: regex, tt: '^.*[01]_x.*$'}
12+
13+
list1:
14+
- {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
15+
- {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
16+
17+
result1:
18+
- {k0_x0: A0, k1_x1: B0}
19+
- {k0_x0: A1, k1_x1: B1}
20+
21+
targets2:
22+
- {mp: equal, tt: k0_x0}
23+
- {mp: starts_with, tt: k0}
24+
- {mp: ends_with, tt: x0}
25+
- {mp: regex, tt: '^.*0_x.*$'}
26+
27+
list2:
28+
- {k0_x0: A0, k1_x1: B0, k2_x2: [C0], k3_x3: foo}
29+
- {k0_x0: A1, k1_x1: B1, k2_x2: [C1], k3_x3: bar}
30+
31+
result2:
32+
- {k0_x0: A0}
33+
- {k0_x0: A1}

0 commit comments

Comments
 (0)