Merge branch 'development'

Author: hkage

.editorconfig

@@ -3,19 +3,21 @@
 root = true
 
 [*]
-indent_style = space
-indent_size = 4
-trim_trailing_whitespace = true
-insert_final_newline = true
 charset = utf-8
 end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
 
-[*.bat]
-indent_style = tab
-end_of_line = crlf
+[*.{py,rst,ini}]
+indent_style = space
+indent_size = 4
+
+[*.{html,css,scss,json,yml}]
+indent_style = space
+indent_size = 2
 
-[LICENSE]
-insert_final_newline = false
+[*.md]
+trim_trailing_whitespace = false
 
 [Makefile]
 indent_style = tab

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Development
 
+## 0.5.0 (2021-06-30)
+
+* [#22](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/22): Fix table and column name quotes in cursor.copy_from call ([nurikk](https://github.com/nurikk))
+* [#23](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/23): Allow uniq faker ([nurikk](https://github.com/nurikk))
+
 ## 0.4.1 (2021-05-27)
 
 * [#19](https://github.com/rheinwerk-verlag/postgresql-anonymizer/pull/19): Make chunk size in the table definition dynamic ([halilkaya](https://github.com/halilkaya))

README.rst

@@ -1,13 +1,13 @@
 PostgreSQL Anonymizer
 =====================
 
-A commandline tool to anonymize PostgreSQL databases for GDPR purposes.
+A commandline tool to anonymize PostgreSQL databases for DSGVO/GDPR purposes.
 
-It uses a YAML definition file to define which tables and fields should be anonymized and provides various methods of anonymization.
+It uses a YAML file to define which tables and fields should be anonymized and provides various methods of anonymization. The tool requires a direct PostgreSQL connection to perform the anonymization.
 
 .. class:: no-web no-pdf
 
-    |python| |license| |pypi| |downloads| |build|
+    |python| |license| |pypi| |downloads| |build| |health|
 
 .. contents::
 
@@ -34,9 +34,13 @@ Features
 +----------------+----------------------+-----------------------+----------------------------------+
 | ``email``      | [email protected] | ``md5``               | 0cba00ca3da1b283a57287bcceb17e35 |
 +----------------+----------------------+-----------------------+----------------------------------+
+| ``email``      | [email protected] | ``faker.unique.email``| [email protected]                 |
++----------------+----------------------+-----------------------+----------------------------------+
 | ``ip``         | 157.50.1.20          | ``set``               | 127.0.0.1                        |
 +----------------+----------------------+-----------------------+----------------------------------+
 
+Note: `faker.unique.[provider]` only supported on python3.5+ (Faker library min supported python version)
+
 See the `documentation`_ for a more detailed description of the provided anonymization methods.
 
 Installation
@@ -161,3 +165,8 @@ After that you can pass a schema file to the container, using Docker volumes, an
 
 .. |build| image:: https://github.com/rheinwerk-verlag/postgresql-anonymizer/workflows/Test/badge.svg
     :target: https://github.com/rheinwerk-verlag/postgresql-anonymizer/actions
+    
+.. |health| image:: https://snyk.io/advisor/python/pganonymize/badge.svg
+  :target: https://snyk.io/advisor/python/pganonymize
+  :alt: pganonymize
+

docs/schema.rst

@@ -216,8 +216,8 @@ The ``clear`` provider will set a database field to ``null``.
 
 **Arguments:** none
 
-``pganonymize`` supports all providers from the Python library Faker_. All you have to do is prefix
-the provider with ``fake`` and then use the function name from the Faker library, e.g:
+``pganonymize`` supports all providers from the Python library `Faker`_. All you have to do is prefix the provider with
+``fake`` and then use the function name from the Faker library, e.g:
 
 * ``fake.first_name``
 * ``fake.street_name``
@@ -235,6 +235,8 @@ the provider with ``fake`` and then use the function name from the Faker library
             provider:
               name: fake.email
 
+See the `Faker documentation`_ for a full set of providers.
+
 ``mask``
 ~~~~~~~~
 
@@ -298,3 +300,6 @@ The value can also be a dictionary for JSONB columns::
             provider:
               name: set
               value: '{"foo": "bar", "baz": 1}'
+
+.. _Faker: https://github.com/joke2k/faker
+.. _Faker documentation: http://faker.rtfd.org/

pganonymizer/providers.py

@@ -1,3 +1,4 @@
+import operator
 import random
 from hashlib import md5
 
@@ -87,9 +88,9 @@ def matches(cls, name):
         return cls.id.lower() == name.split('.')[0].lower()
 
     def alter_value(self, value):
-        func_name = self.kwargs['name'].split('.')[1]
+        func_name = self.kwargs['name'].split('.', 1)[1]
         try:
-            func = getattr(fake_data, func_name)
+            func = operator.attrgetter(func_name)(fake_data)
         except AttributeError as exc:
             raise InvalidProviderArgument(exc)
         return func()

pganonymizer/utils.py

@@ -81,7 +81,7 @@ def build_data(connection, table, columns, excludes, search, total_count, chunk_
                     row[key] = value
             if verbose:
                 progress_bar.next()
-            table_columns = ['"{}"'.format(column) for column in row.keys()]
+            table_columns = [column for column in row.keys()]
             if not row_column_dict:
                 continue
             data.append(row.values())
@@ -97,9 +97,11 @@ def row_matches_excludes(row, excludes=None):
 
     :param list row: The data row
     :param list excludes: A list of field exclusion roles, e.g.:
-        [
-            {'email': ['\\S.*@example.com', '\\S.*@foobar.com', ]}
-        ]
+
+    >>> [
+    >>>     {'email': ['\\S.*@example.com', '\\S.*@foobar.com', ]}
+    >>> ]
+
     :return: True or False
     :rtype: bool
     """
@@ -126,7 +128,8 @@ def copy_from(connection, data, table, columns):
     new_data = data2csv(data)
     cursor = connection.cursor()
     try:
-        cursor.copy_from(new_data, table, sep=COPY_DB_DELIMITER, null='\\N', columns=columns)
+        quoted_cols = ['"{}"'.format(column) for column in columns]
+        cursor.copy_from(new_data, table, sep=COPY_DB_DELIMITER, null='\\N', columns=quoted_cols)
     except (BadCopyFileFormat, InvalidTextRepresentation) as exc:
         raise BadDataFormat(exc)
     cursor.close()
@@ -145,9 +148,9 @@ def import_data(connection, column_dict, source_table, table_columns, primary_ke
     :param list data: The table data.
     """
     primary_key = primary_key if primary_key else DEFAULT_PRIMARY_KEY
-    temp_table = '"tmp_{table}"'.format(table=source_table)
+    temp_table = 'tmp_{table}'.format(table=source_table)
     cursor = connection.cursor()
-    cursor.execute('CREATE TEMP TABLE %s (LIKE %s INCLUDING ALL) ON COMMIT DROP;' % (temp_table, source_table))
+    cursor.execute('CREATE TEMP TABLE "%s" (LIKE %s INCLUDING ALL) ON COMMIT DROP;' % (temp_table, source_table))
     copy_from(connection, data, temp_table, table_columns)
     set_columns = ', '.join(['{column} = s.{column}'.format(column='"{}"'.format(key)) for key in column_dict.keys()])
     sql = (
@@ -221,10 +224,12 @@ def get_column_dict(columns):
     Return a dictionary with all fields from the table definition and None as value.
 
     :param list columns: A list of field definitions from the YAML schema, e.g.:
-        [
-            {'first_name': {'provider': 'set', 'value': 'Foo'}},
-            {'guest_email': {'append': '@localhost', 'provider': 'md5'}},
-        ]
+
+    >>> [
+    >>>     {'first_name': {'provider': 'set', 'value': 'Foo'}},
+    >>>     {'guest_email': {'append': '@localhost', 'provider': 'md5'}},
+    >>> ]
+
     :return: A dictionary containing all fields to be altered with a default value of None, e.g.::
         {'guest_email': None}
     :rtype: dict
@@ -242,9 +247,11 @@ def get_column_values(row, columns):
 
     :param psycopg2.extras.DictRow row: A data row from the current table to be altered
     :param list columns: A list of table columns with their provider rules, e.g.:
-        [
-            {'guest_email': {'append': '@localhost', 'provider': 'md5'}}
-        ]
+
+    >>> [
+    >>>     {'guest_email': {'append': '@localhost', 'provider': 'md5'}}
+    >>> ]
+
     :return: A dictionary with all fields that have to be altered and their value for a single data row, e.g.:
         {'guest_email': '12faf5a9bb6f6f067608dca3027c8fcb@localhost'}
     :rtype: dict

pganonymizer/version.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 
-__version__ = '0.4.1'
+__version__ = '0.5.0'