Skip to content

Commit 5fa9230

Browse files
hkagehkage
committed
Merge branch 'development'
2 parents a58acc5 + 66d4133 commit 5fa9230

File tree

8 files changed

+181
-4
lines changed

8 files changed

+181
-4
lines changed

‎.github/workflows/test.yml

+11-1
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,18 @@ jobs:
3434
- name: Install dependencies
3535
run: |
3636
python -m pip install --upgrade pip
37-
pip install tox "coverage<5"
37+
pip install --use-pep517 tox "coverage<5"
3838
- name: Run tests
3939
run: |
4040
export TOXENV=$(echo "py${{ matrix.python-version }}" | sed 's/\.//g')
4141
tox -- -p no:warnings
42+
- name: Generate coverage report
43+
run: coverage html
44+
if: ${{ success() }}
45+
- name: Upload coverage data
46+
uses: actions/upload-artifact@v3
47+
with:
48+
name: coverage
49+
path: htmlcov
50+
if-no-files-found: ignore
51+
if: ${{ success() }}

‎CHANGELOG.rst

+5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ Changelog
44
Development
55
-----------
66

7+
0.11.0 (2024-02-29)
8+
-------------------
9+
10+
* `#52 <https://github.com/rheinwerk-verlag/pganonymize/pull/52>`_: Add update_json provider (`bobslee <https://github.com/bobslee>`_)
11+
712
0.10.0 (2022-11-29)
813
-------------------
914

‎docs/schema.rst

+31
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,37 @@ This provider will replace values with a unique UUID4.
423423
provider:
424424
name: uuid4
425425
426+
``update_json``
427+
~~~~~~~~~~~~~~~
428+
429+
**Arguments:**
430+
431+
* ``update_values_type``
432+
433+
This provider will replace json and jsonb data values with a specified provider configuration per data type.
434+
435+
436+
**Example usage**:
437+
438+
.. code-block:: yaml
439+
440+
tables:
441+
- payment_transaction:
442+
fields:
443+
- data:
444+
provider:
445+
name: update_json
446+
update_values_type:
447+
str:
448+
provider:
449+
name: uuid4
450+
int:
451+
provider:
452+
name: fake.pyint
453+
float:
454+
provider:
455+
name: fake.pyfloat
456+
426457
.. _Faker: https://github.com/joke2k/faker
427458
.. _Faker documentation: http://faker.rtfd.org/
428459
.. _native UUIDs: https://www.postgresql.org/docs/current/datatype-uuid.html

‎pganonymize/providers.py

+26
Original file line numberDiff line numberDiff line change
@@ -246,3 +246,29 @@ class UUID4Provider(Provider):
246246
@classmethod
247247
def alter_value(cls, original_value, **kwargs):
248248
return uuid4()
249+
250+
251+
@register('update_json')
252+
class UpdateJSONProvider(Provider):
253+
"""Provider to update JSON data (currently values) by providers."""
254+
255+
@classmethod
256+
def alter_value(cls, original_value, **kwargs):
257+
def update_dict(input_dict, update_values_type={}):
258+
"""Update dictionary with recursion (nested dictionaries)."""
259+
if not update_values_type:
260+
return
261+
for key, val in input_dict.items():
262+
if isinstance(val, dict):
263+
update_dict(val, update_values_type=update_values_type)
264+
else:
265+
val_type = type(val).__name__
266+
val_update = update_values_type.get(val_type)
267+
if val_update:
268+
if val_update.get('provider'):
269+
provider_config = val_update.get('provider')
270+
provider_class = provider_registry.get_provider(provider_config['name'])
271+
provider_value = provider_class.alter_value(val, **provider_config)
272+
input_dict[key] = provider_value
273+
update_dict(original_value, update_values_type=kwargs.get('update_values_type', {}))
274+
return original_value

‎pganonymize/utils.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ def escape_str_replace(value):
323323
:return: Escaped value
324324
"""
325325
if isinstance(value, dict):
326-
return json.dumps(value).encode()
326+
return json.dumps(value, default=str).encode()
327327
return value
328328

329329

‎pganonymize/version.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# -*- coding: utf-8 -*-
22

3-
__version__ = '0.10.0'
3+
__version__ = '0.11.0'

‎pytest.ini

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
[pytest]
2-
addopts = --cov=pganonymize --cov-report term-missing --cov-config setup.cfg
2+
addopts = --cov=pganonymize --cov-append --cov-report term-missing --cov-config setup.cfg
33
testpaths = tests pganonymize
44
python_paths = pganonymize

‎tests/test_providers.py

+105
Original file line numberDiff line numberDiff line change
@@ -211,3 +211,108 @@ class TestUUID4Provider(object):
211211
@pytest.mark.parametrize('value, expected', [(None, uuid.UUID), ('Foo', uuid.UUID)])
212212
def test_alter_value(self, value, expected):
213213
assert type(providers.UUID4Provider.alter_value(value)) == expected
214+
215+
216+
class TestUpdateJSONProvider(object):
217+
218+
@pytest.mark.parametrize('value, update_values_type, expected', [
219+
({'foo': 'bar'}, {'str': {'provider': {'name': 'set', 'value': 'foobar'}}}, {'foo': 'foobar'}),
220+
({'chef': 'cuisine'}, {'str': {'provider': {'name': 'uuid4'}}}, {'chef': uuid.UUID})
221+
])
222+
def test_str_type(self, value, update_values_type, expected):
223+
res = providers.UpdateJSONProvider.alter_value(value, update_values_type=update_values_type)
224+
if res.get('foo'):
225+
assert res['foo'] == expected['foo']
226+
if res.get('chef'):
227+
assert type(res['chef']) is expected['chef']
228+
229+
def test_int_type(self):
230+
test_dict = {'foo': 123}
231+
232+
update_values_type = {'int': {'provider': {'name': 'set', 'value': 999}}}
233+
res = providers.UpdateJSONProvider.alter_value(test_dict, update_values_type=update_values_type)
234+
assert res['foo'] == 999
235+
236+
update_values_type = {'int': {'provider': {'name': 'clear'}}}
237+
res = providers.UpdateJSONProvider.alter_value(test_dict, update_values_type=update_values_type)
238+
assert res['foo'] is None
239+
240+
def test_float_type(self):
241+
test_dict = {'foo': 123.45, 'bar': 234.77}
242+
243+
update_values_type = {'float': {'provider': {'name': 'set', 'value': 999.99}}}
244+
res = providers.UpdateJSONProvider.alter_value(test_dict, update_values_type=update_values_type)
245+
assert res['foo'] == 999.99
246+
assert res['bar'] == 999.99
247+
248+
update_values_type = {'float': {'provider': {'name': 'clear'}}}
249+
res = providers.UpdateJSONProvider.alter_value(test_dict, update_values_type=update_values_type)
250+
assert res['foo'] is None
251+
252+
def test_multi_types(self):
253+
test_dict = {
254+
'fooInt': 123,
255+
'fooFloat': 123.45,
256+
'fooStr': 'some foo',
257+
'barStr': 'some bar'
258+
}
259+
update_values_type = {
260+
'int': {'provider': {'name': 'set', 'value': 999}},
261+
'float': {'provider': {'name': 'set', 'value': 999.99}},
262+
'str': {'provider': {'name': 'set', 'value': 'foobar'}},
263+
}
264+
res = providers.UpdateJSONProvider.alter_value(test_dict, update_values_type=update_values_type)
265+
assert res['fooInt'] == 999
266+
assert res['fooFloat'] == 999.99
267+
assert res['fooStr'] == 'foobar'
268+
assert res['barStr'] == 'foobar'
269+
270+
def test_type_not_specified(self):
271+
test_dict = {
272+
'fooInt': 123,
273+
'fooFloat': 123.45,
274+
'fooStr': 'some foo',
275+
'barStr': 'some bar'
276+
}
277+
update_values_type = {
278+
'int': {'provider': {'name': 'set', 'value': 999}},
279+
}
280+
res = providers.UpdateJSONProvider.alter_value(test_dict, update_values_type=update_values_type)
281+
assert res['fooInt'] == 999
282+
assert res['fooFloat'] == 123.45
283+
assert res['fooStr'] == 'some foo'
284+
assert res['barStr'] == 'some bar'
285+
286+
def test_nested_json(self):
287+
test_dict = {
288+
'fooInt': 123,
289+
'fooFloat': 123.45,
290+
'fooStr': 'some foo',
291+
'barStr': 'some bar',
292+
'nested': {
293+
'fooInt': 444,
294+
'fooFloat': 50.9,
295+
'fooStr': 'abc',
296+
'anotherNested': {
297+
'fooInt': 555,
298+
'fooFloat': 6000.123,
299+
'fooStr': 'xyz',
300+
}
301+
}
302+
}
303+
update_values_type = {
304+
'int': {'provider': {'name': 'set', 'value': 999}},
305+
'float': {'provider': {'name': 'set', 'value': 999.99}},
306+
'str': {'provider': {'name': 'set', 'value': 'foobar'}},
307+
}
308+
res = providers.UpdateJSONProvider.alter_value(test_dict, update_values_type=update_values_type)
309+
assert res['fooInt'] == 999
310+
assert res['fooFloat'] == 999.99
311+
assert res['fooStr'] == 'foobar'
312+
assert res['barStr'] == 'foobar'
313+
assert res['nested']['fooInt'] == 999
314+
assert res['nested']['fooFloat'] == 999.99
315+
assert res['nested']['fooStr'] == 'foobar'
316+
assert res['nested']['anotherNested']['fooInt'] == 999
317+
assert res['nested']['anotherNested']['fooFloat'] == 999.99
318+
assert res['nested']['anotherNested']['fooStr'] == 'foobar'

0 commit comments

Comments
 (0)