Merge branch 'feature/nmig_3'

Author: AnatolyUss

.gitignore

@@ -0,0 +1,4 @@
+.idea
+node_modules
+logs_directory
+temporary_directory

README.md

@@ -23,7 +23,7 @@ from MySQL to PostgreSQL as easy and smooth as possible.</p>
 
 <h3>SYSTEM REQUIREMENTS</h3>
 <ul>
-<li> <b>Node.js 5 or higher</b></li>
+<li> <b>Node.js 7 or higher</b></li>
 </ul>
 
 <h3>USAGE</h3>
@@ -44,11 +44,11 @@ from MySQL to PostgreSQL as easy and smooth as possible.</p>
     &nbsp;&nbsp;&nbsp;&nbsp;<b>Sample:</b><br />
     <pre>$ cd /path/to/nmig</pre><br />
     <pre>$ npm install</pre><br />
-    <pre>$ node nmig.js</pre><br />
+    <pre>$ npm start</pre><br />
 </p>
 
 <p><b>5.</b> If a disaster took place during migration (for what ever reason) - simply restart the process
-<code>$ node nmig.js</code><br>&nbsp;&nbsp;&nbsp;&nbsp;NMIG will restart from the point it was stopped at.
+<code>$ npm start</code><br>&nbsp;&nbsp;&nbsp;&nbsp;NMIG will restart from the point it was stopped at.
 </p>
 
 <p><b>6.</b> At the end of migration check log files, if necessary.<br />&nbsp;&nbsp;&nbsp;
@@ -61,23 +61,9 @@ from MySQL to PostgreSQL as easy and smooth as possible.</p>
    <a href="mailto:[email protected]?subject=NMIG">[email protected]</a></p>
 
 <h3>VERSION</h3>
-<p>Current version is 2.6.1<br />
+<p>Current version is 3.0.0<br />
 (major version . improvements . bug fixes)</p>
 
-
-<h3>TEST</h3>
-<p>Tested using MySQL Community Server (5.7.17) and PostgreSQL (9.5).<br />
-The entire process of migration 59.9 MB database (53 tables, 581275 rows),<br />
-which includes data types mapping, creation of tables, constraints, indexes, <br />
-PKs, FKs, migration of data, garbage-collection (VACUUM) and analyzing the newly created <br />
-PostgreSQL database took 1 minute 2 seconds.</p>
-<p>
-<b>Note:</b>&nbsp; it is highly recommended to VACUUM newly created database! <br />
-Just keep in mind, that VACUUM is a very time-consuming procedure. <br />
-So if you are short in time - disable VACUUM via config.json ("no_vacuum" parameter). <br />
-Such step will save you ~25% of migration time. <br />
-</p>
-
 <h3>REMARKS</h3>
 <p>Errors/Exceptions are not passed silently.<br />
 Any error will be immediately written into the error log file.</p>

config.json

@@ -1,4 +1,9 @@
 {
+    "README" : [
+        "This JSON document represents the main configuration options.",
+        "Please, be mindful, read each option's description before setting its value."
+    ],
+
     "source_description" : [
         "Connection parameters to your MySQL database",
         "Please ensure, that you have defined all parameters properly.",
@@ -13,11 +18,6 @@
         "password" : "0123456789"
     },
 
-    "max_pool_size_source_description" : [
-        "Maximal amount of simultaneous connections to your MySQL database during migration"
-    ],
-    "max_pool_size_source" : 10,
-
     "target_description" : [
         "Connection parameters to your PostgreSQL database",
         "Please ensure, that you have defined all parameters properly.",
@@ -32,28 +32,19 @@
         "password" : "0123456789"
     },
 
-    "max_pool_size_target_description" : [
-        "Maximal amount of simultaneous connections to your PostgreSQL database during migration"
+    "max_db_connection_pool_size_description" : [
+        "Maximal amount of simultaneous connections to your MySQL and PostgreSQL servers.",
+        "For example: 10 means, that nmig will create a pool of 10 connections to MySQL server",
+        "and 10 connections to PostgreSQL server.",
+        "20 connections to both servers."
     ],
-    "max_pool_size_target" : 10,
-
-    "pipe_width_description" : [
-        "Maximal amount of data-chunks, processed in a single 'DataPipe' iteration.",
-        "Note: 'pipe_width' should not be greater than 'max_pool_size_target'."
-    ],
-    "pipe_width" : 10,
-
-    "max_loader_processes_description" : [
-        "Each 'DataPipe' iteration processed by a separate process (o.s. thread).",
-        "max_loader_processes is a maximal amount of simultaneously running loader processes.",
-        "If not supplied, then max_loader_processes will be equals to one."
-    ],
-    "max_loader_processes" : 1,
+    "max_db_connection_pool_size" : 10,
 
     "loader_max_old_space_size_description" : [
         "V8 memory limit of the loader process.",
-        "Possible values are: any number, representing memory limit (in MB),",
-        "or DEFAULT, representing V8 default limit for your current hardware."
+        "Possible values are:",
+        "1. any number, representing memory limit (in MB).",
+        "2. 'DEFAULT', representing V8 default memory limit for your current hardware."
     ],
     "loader_max_old_space_size" : "DEFAULT",
 
@@ -64,19 +55,20 @@
     "encoding" : "utf8",
 
     "schema_description" : [
-        "schema - a name of the schema, that will contain all migrated tables.",
+        "A name of the schema, that will contain all migrated tables.",
         "If not supplied, then a new schema will be created automatically."
     ],
     "schema" : "public",
 
     "data_chunk_size_description" : [
-        "During migration each table's data will be split into chunks of data_chunk_size (in MB).",
-        "If not supplied, then 1 MB will be used as a default."
+        "During migration each table's data will be split into chunks not larger than data_chunk_size (in MB).",
+        "If not supplied, then 10 MB will be used as a default."
     ],
-    "data_chunk_size" : 1,
+    "data_chunk_size" : 10,
 
     "no_vacuum_description" : [
-        "PostgreSQL VACUUM reclaims storage occupied by dead tuples. VACUUM is a very time-consuming procedure.",
+        "PostgreSQL VACUUM reclaims storage occupied by dead tuples.",
+        "VACUUM is a very time-consuming procedure.",
         "By default, VACUUM will be performed automatically after migration (recommended)",
         "This behavior can be disabled for tables, included into the list (Array) below.",
         "Table names, included into the list below should be names, you use in your source database (MySQL)."
@@ -96,14 +88,15 @@
     "migrate_only_data" : false,
 
     "delimiter_description" : [
-      "Specifies the character that separates columns within each record.",
+      "Specifies the character, that separates columns within each record.",
       "This must be a single one-byte character.",
       "The default is comma."
     ],
     "delimiter" : ",",
 
     "enable_extra_config_description" : [
-        "In order to enable the additional configuration options - set this parameter true."
+        "In order to enable the additional configuration options, placed at extra_config.json",
+        " - set this parameter true."
     ],
     "enable_extra_config" : false
 }

DataTypesMap.json renamed to data_types_map.json

@@ -1,12 +1,24 @@
 {
     "README" : [
         "This JSON document represents an additional configuration options.",
-        "These options (currently only renaming) will be applied during migration if, and only if",
+        "Currently, following options are available:",
+        "1. tables and columns renaming 'on the fly'.",
+        "2. generating foreign keys 'on the fly'.",
+        "These options will be applied during migration if, and only if",
         "the 'enable_extra_config' attribute from 'config.json' is set true.",
         "By default the 'enable_extra_config' is false, hence the options",
-        "described here will not take effect.",
+        "described here will not take effect."
+    ],
+
+    "README_RENAMING" : [
         "Following lines are an example of expected renaming format.",
-        "Feel free to modify them."
+        "Feel free to modify them.",
+        "Note, the 'name' attribute is mandatory.",
+        "'columns' attribute is optional.",
+        "If you don't need to rename columns, then remove the 'columns' attribute.",
+        "If you don't need to rename tables,",
+        "then set both 'name' object's attributes ('original' and 'new') to an actual table's name.",
+        "If you don't need to rename neither tables nor columns, then remove the 'tables' section."
     ],
 
     "tables" : [
@@ -28,16 +40,16 @@
                 },
 
                 {
-                    "original" : "password",
-                    "new" : "renamed_password"
+                    "original" : "group_id",
+                    "new" : "renamed_group_id"
                 }
             ]
         },
 
         {
             "name" : {
-                "original" : "logs",
-                "new" : "renamed_logs"
+                "original" : "groups",
+                "new" : "renamed_groups"
             },
 
             "columns" : [
@@ -47,5 +59,56 @@
                 }
             ]
         }
+    ],
+
+    "README_FKs" : [
+        "Following lines are an example of expected foreign keys format.",
+        "Feel free to modify them.",
+        "Note, if one of the following: 'column_name', 'referenced_table_name', 'referenced_column_name'",
+        "has being renamed, then a new name must be used.",
+        "Note, possible values for 'update_rule' and 'delete_rule' are 'restrict' or 'cascade'.",
+        "If you don't need to create foreign keys 'on the fly', then remove the 'foreign_keys' section."
+    ],
+
+    "foreign_keys" : [
+        {
+            "constraint_name" : "renamed_admins_renamed_group_id_foreign",
+            "table_name" : "renamed_admins",
+            "column_name" : "renamed_group_id",
+            "referenced_table_name" : "renamed_groups",
+            "referenced_column_name" : "renamed_id",
+            "update_rule" : "restrict",
+            "delete_rule" : "restrict"
+        },
+
+        {
+            "constraint_name" : "logs_test_3_merchant_id_foreign",
+            "table_name" : "logs_test_3",
+            "column_name" : "merchant_id",
+            "referenced_table_name" : "merchants",
+            "referenced_column_name" : "id",
+            "update_rule" : "restrict",
+            "delete_rule" : "cascade"
+        },
+
+        {
+            "constraint_name" : "composite_foreign_key_example",
+            "table_name" : "loGs_Test-6",
+            "column_name" : "id1",
+            "referenced_table_name" : "logs_test_4",
+            "referenced_column_name" : "id1",
+            "update_rule" : "cascade",
+            "delete_rule" : "cascade"
+        },
+
+        {
+            "constraint_name" : "composite_foreign_key_example",
+            "table_name" : "loGs_Test-6",
+            "column_name" : "id2",
+            "referenced_table_name" : "logs_test_4",
+            "referenced_column_name" : "id2",
+            "update_rule" : "cascade",
+            "delete_rule" : "cascade"
+        }
     ]
 }