From 0390d78efdfc4154bc431471ec49f4c6dcd9a9a1 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Wed, 27 Oct 2010 13:45:51 -0400 Subject: [PATCH 01/69] treat mysql float as pg real --- mysql2psql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mysql2psql b/mysql2psql index e75141c..3892936 100755 --- a/mysql2psql +++ b/mysql2psql @@ -103,7 +103,9 @@ class MysqlReader "varchar" when /char/ "char" - when /(float|decimal)/ + when /float/ + "real" + when /decimal/ "decimal" when /double/ "double precision" @@ -291,7 +293,7 @@ class PostgresWriter < Writer when "boolean" default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default "boolean" - when "float" + when "real" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default "real" when "float unsigned" From a49cf08bd5aa23a771518c8bc2f837014344f187 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Wed, 27 Oct 2010 14:11:25 -0400 Subject: [PATCH 02/69] dont guess at precision --- mysql2psql | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql2psql b/mysql2psql index 3892936..9e9f681 100755 --- a/mysql2psql +++ b/mysql2psql @@ -121,6 +121,7 @@ class MysqlReader fields = [] @reader.mysql.query("EXPLAIN `#{name}`") do |res| while field = res.fetch_row do + length = -1 length = field[1][/\((\d+)\)/, 1] if field[1] =~ /\((\d+)\)/ length = field[1][/\((\d+),(\d+)\)/, 1] if field[1] =~ /\((\d+),(\d+)\)/ desc = { From 61bcbbf269b86373a39824b67a3168c4dc2b49a8 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Wed, 27 Oct 2010 14:37:57 -0400 Subject: [PATCH 03/69] check number of tuples inserted on copy --- mysql2psql | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mysql2psql b/mysql2psql index 9e9f681..d65f873 100755 --- a/mysql2psql +++ b/mysql2psql @@ -651,14 +651,22 @@ class PostgresDbWriter < PostgresWriter if counter % 5000 == 0 @conn.put_copy_end + res = @conn.get_result + if res.cmdtuples != 5000 + puts "\nWARNING: #{table.name} expected 5000 tuple inserts got #{res.cmdtuples} at row #{counter}\n" + end @conn.exec(copy_line) end end - _time2 = Time.now - puts "\n#{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s" # @conn.putline(".\n") @conn.put_copy_end + res = @conn.get_result + if res.cmdtuples != (_counter % 5000) + puts "\nWARNING: table #{table.name} expected #{_counter % 5000} tuple inserts got #{res.cmdtuples}\n" + end + _time2 = Time.now + puts "\n#{table.name} #{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s" end def close From 9dd8a2f75b8995befdc01c557ed2d3626e3db5f1 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Thu, 28 Oct 2010 07:20:55 -0400 Subject: [PATCH 04/69] fix error on counting inserts for empty tables --- mysql2psql | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mysql2psql b/mysql2psql index d65f873..25cc2bb 100755 --- a/mysql2psql +++ b/mysql2psql @@ -661,9 +661,11 @@ class PostgresDbWriter < PostgresWriter end # @conn.putline(".\n") @conn.put_copy_end - res = @conn.get_result - if res.cmdtuples != (_counter % 5000) - puts "\nWARNING: table #{table.name} expected #{_counter % 5000} tuple inserts got #{res.cmdtuples}\n" + if _counter && (_counter % 5000) > 0 + res = @conn.get_result + if res.cmdtuples != (_counter % 5000) + puts "\nWARNING: table #{table.name} expected #{_counter % 5000} tuple inserts got #{res.cmdtuples}\n" + end end _time2 = Time.now puts "\n#{table.name} #{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s" From 2bfa98f421db7826ecdea0e29db3cfdc77ff6802 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 1 Nov 2010 22:45:17 +0800 Subject: [PATCH 05/69] clean up formatting --- README.rdoc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rdoc b/README.rdoc index ae6a0dc..1b81b80 100644 --- a/README.rdoc +++ b/README.rdoc @@ -11,7 +11,8 @@ to contact me, I'll help you. Mysql2psql is packaged as a gem. Install as usual (use sudo if required). $ gem install mysql2psql - NB: the gem hasn't been formally released on http://rubygems.org/ yet. For now you need to clone the git repo at http://github.com/tardate/mysql2postgres and 'rake install' + +NB: the gem hasn't been formally released on http://rubygems.org/ yet. For now you need to clone the git repo at http://github.com/tardate/mysql2postgres and 'rake install'. After installation, the "mysql2psql" command will be available. When run, it will generate a default configuration file in the current directory if a config @@ -60,6 +61,7 @@ The gem packaged version you are currently looking at has yet to be merged into The gem bundling and refactoring for testing was done by Paul Gallagher and the repository is at http://github.com/tardate/mysql2postgres +Gem packaging is done with Jeweler. Note that on windows this means you must run under mingw or other shell that supports git to do the gem bundling (but you can gem install the built gem in the normal Windows shell). == Running tests If you fork/clone the project, you will want to run the test suite (and add to it if you are developing new features or fixing bugs). @@ -80,6 +82,7 @@ mysql on localhost:3306 - database created called "mysql2psql_test" - user setup for "mysql2psql" with no password - e.g. + mysqladmin -uroot -p create mysql2psql_test mysql -uroot -p -e "grant all on mysql2psql_test.* to 'mysql2psql'@'localhost';" # verify connecction: @@ -89,6 +92,7 @@ postgres on localhost:5432 - database created called "mysql2psql_test" - role (user) access setup for "mysql2psql" with no password - e.g. + psql postgres -c "create role mysql2psql with login" psql postgres -c "create database mysql2psql_test with owner mysql2psql encoding='UTF8'" # verify connection: @@ -96,10 +100,9 @@ postgres on localhost:5432 == Notes, Limitations, Outstanding Issues.. -Todo: +=== Todo - more tests - release gem -- a windows cmd shim === note from mgkimsal I'm still having trouble with bit(1)/boolean fields From 6a4a8b7e71f118476d1d04f9622969256cd8492e Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 1 Nov 2010 22:48:10 +0800 Subject: [PATCH 06/69] factor out set serial sequence code to postgres_writer --- lib/mysql2psql/postgres_db_writer.rb | 2 +- lib/mysql2psql/postgres_file_writer.rb | 5 ++--- lib/mysql2psql/postgres_writer.rb | 9 ++++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index f636c9f..5dbbf67 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -65,7 +65,7 @@ def write_table(table) CACHE 1 EOF - @conn.exec "SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true)" + @conn.exec sqlfor_set_serial_sequence(table,serial_key,maxval) end if @conn.server_version < 80200 diff --git a/lib/mysql2psql/postgres_file_writer.rb b/lib/mysql2psql/postgres_file_writer.rb index 12d39f4..5fc2ad7 100644 --- a/lib/mysql2psql/postgres_file_writer.rb +++ b/lib/mysql2psql/postgres_file_writer.rb @@ -33,7 +33,7 @@ def truncate(table) EOF if serial_key @f << <<-EOF -SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true); +#{sqlfor_reset_serial_sequence(table,serial_key,maxval)} EOF end end @@ -69,8 +69,7 @@ def write_table(table) NO MINVALUE CACHE 1; - -SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true); +#{sqlfor_set_serial_sequence(table,serial_key,maxval)} EOF end diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 9609ed4..6dc2f20 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -137,7 +137,14 @@ def process_row(table, row) def truncate(table) end - + + def sqlfor_set_serial_sequence(table,serial_key,maxval) + "SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true);" + end + def sqlfor_reset_serial_sequence(table,serial_key,maxval) + "SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true);" + end + end end \ No newline at end of file From 9727863d945c0e738324730e3a07a59d944b27df Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Thu, 4 Nov 2010 09:54:47 -0400 Subject: [PATCH 07/69] remerge of bug fixes --- lib/mysql2psql/mysql_reader.rb | 7 +++++-- lib/mysql2psql/postgres_db_writer.rb | 19 ++++++++++++++----- lib/mysql2psql/postgres_writer.rb | 4 ++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb index 98dc5e8..c343650 100644 --- a/lib/mysql2psql/mysql_reader.rb +++ b/lib/mysql2psql/mysql_reader.rb @@ -43,7 +43,9 @@ def convert_type(type) "varchar" when /char/ "char" - when /(float|decimal)/ + when /float/ + "real" + when /decimal/ "decimal" when /double/ "double precision" @@ -59,6 +61,7 @@ def load_columns fields = [] @reader.mysql.query("EXPLAIN `#{name}`") do |res| while field = res.fetch_row do + length = -1 length = field[1][/\((\d+)\)/, 1] if field[1] =~ /\((\d+)\)/ length = field[1][/\((\d+),(\d+)\)/, 1] if field[1] =~ /\((\d+),(\d+)\)/ desc = { @@ -188,4 +191,4 @@ def paginated_read(table, page_size) end end -end \ No newline at end of file +end diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index f636c9f..a861e21 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -164,16 +164,25 @@ def write_contents(table, reader) if counter % 5000 == 0 @conn.put_copy_end + res = @conn.get_result + if res.cmdtuples != 5000 + puts "\nWARNING: #{table.name} expected 5000 tuple inserts got #{res.cmdtuples} at row #{counter}\n" + end @conn.exec(copy_line) end end - _time2 = Time.now - puts "\n#{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s" -# @conn.putline(".\n") @conn.put_copy_end + if _counter && (_counter % 5000) > 0 + res = @conn.get_result + if res.cmdtuples != (_counter % 5000) + puts "\nWARNING: table #{table.name} expected #{_counter % 5000} tuple inserts got #{res.cmdtuples}\n" + end + end + _time2 = Time.now + puts "\n#{table.name} #{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s" end - + end -end \ No newline at end of file +end diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 9609ed4..96da0a4 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -46,7 +46,7 @@ def column_type_info(column) when "boolean" default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default "boolean" - when "float" + when "real" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default "real" when "float unsigned" @@ -140,4 +140,4 @@ def truncate(table) end -end \ No newline at end of file +end From 7b48c1caf99ac90286f4ab2f355d3f568dd95373 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 00:12:05 +0800 Subject: [PATCH 08/69] allow default of no socket to be entered --- lib/mysql2psql/config.rb | 2 +- lib/mysql2psql/mysql_reader.rb | 2 +- test/fixtures/config_all_options.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index 07f6c69..6733f73 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -32,7 +32,7 @@ def self.template(to_filename = nil, include_tables = [], exclude_tables = [], s mysql: hostname: localhost port: 3306 - socket: /tmp/mysql.sock + socket: username: mysql2psql password: database: mysql2psql_test diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb index 98dc5e8..fab00ae 100644 --- a/lib/mysql2psql/mysql_reader.rb +++ b/lib/mysql2psql/mysql_reader.rb @@ -162,7 +162,7 @@ def initialize(options) @host, @user, @passwd, @db, @port, @sock, @flag = options.mysqlhostname('localhost'), options.mysqlusername, options.mysqlpassword, options.mysqldatabase, - options.mysqlport, options.mysqlsocket + options.mysqlport, options.mysqlsocket(nil) connect end diff --git a/test/fixtures/config_all_options.yml b/test/fixtures/config_all_options.yml index 06d0b83..cdb1be2 100644 --- a/test/fixtures/config_all_options.yml +++ b/test/fixtures/config_all_options.yml @@ -1,7 +1,7 @@ mysql: hostname: localhost port: 3306 - socket: /tmp/mysql.sock + socket: username: somename password: secretpassword database: somename From fc54c746b2bd7f6ad02a186c9927bce327fc4424 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 00:13:30 +0800 Subject: [PATCH 09/69] ignore redcar, rvmrc configurations --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d3a0169..684c0dd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ configs test/fixtures/test*.sql pkg +.redcar +.rvmrc + From e01f5df3d6965327479eb94033e798d6d725c817 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 00:14:10 +0800 Subject: [PATCH 10/69] add basic auto-increment test (for file coversions) --- test/fixtures/seed_integration_tests.sql | 10 ++++++++++ test/integration/convert_to_file_test.rb | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 8163255..59a2dea 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -22,3 +22,13 @@ INSERT INTO numeric_types_basics VALUES (23,23,23,23,23,23,23,23,23,23,23,23); +DROP TABLE IF EXISTS basic_autoincrement; +CREATE TABLE basic_autoincrement ( + auto_id INT(11) NOT NULL AUTO_INCREMENT, + auto_dummy INT, + PRIMARY KEY (auto_id) +); + +INSERT INTO basic_autoincrement(auto_dummy) VALUES +(1),(2),(23); + diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index f7c440c..0a2db69 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -5,6 +5,12 @@ class ConvertToFileTest < Test::Unit::TestCase class << self + + # This is a suite of tests to verify conversion of full schema and data to file. + # The export is done once in the class setup. + # Tests inspect specific features of the converted file, + # the contents of which are preloaded as the :content attribute of this class + # def startup seed_test_database @@options=get_test_config_by_label(:localmysql_to_file_convert_all) @@ -23,12 +29,14 @@ def teardown def content @@content end - + + # verify table creation def test_table_creation assert_not_nil content.match('DROP TABLE IF EXISTS "numeric_types_basics" CASCADE') assert_not_nil content.match(/CREATE TABLE "numeric_types_basics"/) end + # tests for the conversion of numeric types def test_basic_numerics_tinyint assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE).match( content ) end @@ -63,4 +71,8 @@ def test_basic_numerics_numeric assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match( content ) end + # test autoincrement handling + def test_autoincrement + assert_not_nil Regexp.new('CREATE TABLE "basic_autoincrement".*"auto_id" integer DEFAULT.*\)', Regexp::MULTILINE).match( content ) + end end \ No newline at end of file From 8d8883b8338958b9c8b80cab3e2d8b902d0e72d6 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 00:14:21 +0800 Subject: [PATCH 11/69] fix indenting --- README.rdoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rdoc b/README.rdoc index 1b81b80..39eed70 100644 --- a/README.rdoc +++ b/README.rdoc @@ -110,11 +110,11 @@ workaround I've found is to put output in file then in VIM on file, search/replace the true/false binary fields with t/f specifically -(reversed on 3/23 - should be ^A gets f) -:%s/^@/t/g -:%s/^A/f/g -keystrokes are ctrl-v ctrl-shift-@ to get the 'true' binary field -keystrokes are ctrl-v ctrl-shift-A to get the 'false' binary field + (reversed on 3/23 - should be ^A gets f) + :%s/^@/t/g + :%s/^A/f/g + keystrokes are ctrl-v ctrl-shift-@ to get the 'true' binary field + keystrokes are ctrl-v ctrl-shift-A to get the 'false' binary field == Contributors Project founded by Max Lapshin From c4b20c20b3e30dc4ba8ff2a2f5d551b81aeb2ec9 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 00:52:28 +0800 Subject: [PATCH 12/69] floats and reals should migrate to double precision (although floats _may_ be single - depending on precision - currently taking the safe route and migrate all to double) --- lib/mysql2psql/mysql_reader.rb | 6 ++++-- lib/mysql2psql/postgres_writer.rb | 5 ++--- test/fixtures/seed_integration_tests.sql | 15 ++++++++++++--- test/integration/convert_to_db_test.rb | 2 ++ test/integration/convert_to_file_test.rb | 3 ++- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb index fab00ae..4f47e68 100644 --- a/lib/mysql2psql/mysql_reader.rb +++ b/lib/mysql2psql/mysql_reader.rb @@ -43,9 +43,11 @@ def convert_type(type) "varchar" when /char/ "char" - when /(float|decimal)/ + when /decimal/ "decimal" - when /double/ + when /float/ + "float" + when /real|double/ "double precision" else type diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 6dc2f20..cfda61b 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -29,7 +29,6 @@ def column_type_info(column) "character(#{column[:length]})" when "varchar" default = default + "::character varying" if default - # puts "VARCHAR: #{column.inspect}" "character varying(#{column[:length]})" # Integer and numeric types @@ -48,10 +47,10 @@ def column_type_info(column) "boolean" when "float" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default - "real" + "double precision" when "float unsigned" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default - "real" + "double precision" when "decimal" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 59a2dea..dc89e15 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -12,14 +12,15 @@ CREATE TABLE numeric_types_basics ( f_real REAL, f_double DOUBLE, f_float FLOAT, + f_ufloat FLOAT UNSIGNED, f_decimal DECIMAL, f_numeric NUMERIC ); INSERT INTO numeric_types_basics VALUES -(1,1,1,1,1,1,1,1,1,1,1,1), -(2,2,2,2,2,2,2,2,2,2,2,2), -(23,23,23,23,23,23,23,23,23,23,23,23); +(1,1,1,1,1,1,1,1,1,1,1,1,1), +(2,2,2,2,2,2,2,2,2,2,2,2,2), +(23,23,23,23,23,23,23,23,23,23,23,23,23); DROP TABLE IF EXISTS basic_autoincrement; @@ -32,3 +33,11 @@ CREATE TABLE basic_autoincrement ( INSERT INTO basic_autoincrement(auto_dummy) VALUES (1),(2),(23); +DROP TABLE IF EXISTS numeric_type_floats; +CREATE TABLE numeric_type_floats ( + latitude FLOAT, + longitude FLOAT +); + +INSERT INTO numeric_type_floats(latitude,longitude) VALUES +(1.1,2.2); diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 249c45e..47190a0 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -24,6 +24,8 @@ def teardown def test_table_creation assert_true @@mysql2psql.writer.exists?('numeric_types_basics') + assert_true @@mysql2psql.writer.exists?('basic_autoincrement') + assert_true @@mysql2psql.writer.exists?('numeric_type_floats') end end \ No newline at end of file diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index 0a2db69..58087ec 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -62,7 +62,8 @@ def test_basic_numerics_double assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE).match( content ) end def test_basic_numerics_float - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" numeric\(20, 0\),.*\)', Regexp::MULTILINE).match( content ) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision.*\)', Regexp::MULTILINE).match( content ) + assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_ufloat" double precision.*\)', Regexp::MULTILINE).match( content ) end def test_basic_numerics_decimal assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE).match( content ) From 1d44286ca464d6c7db8cde93bd72858b5190448b Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 00:58:41 +0800 Subject: [PATCH 13/69] readme format fixup --- README.rdoc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.rdoc b/README.rdoc index 39eed70..7d3b476 100644 --- a/README.rdoc +++ b/README.rdoc @@ -81,23 +81,27 @@ Running the integration tests *will* rewrite data and schema for the "mysql2psql mysql on localhost:3306 - database created called "mysql2psql_test" - user setup for "mysql2psql" with no password -- e.g. + +e.g. mysqladmin -uroot -p create mysql2psql_test mysql -uroot -p -e "grant all on mysql2psql_test.* to 'mysql2psql'@'localhost';" # verify connecction: mysql -umysql2psql -e "select database(),'yes' as connected" mysql2psql_test - + + postgres on localhost:5432 - database created called "mysql2psql_test" - role (user) access setup for "mysql2psql" with no password -- e.g. + +e.g. psql postgres -c "create role mysql2psql with login" psql postgres -c "create database mysql2psql_test with owner mysql2psql encoding='UTF8'" # verify connection: psql mysql2psql_test -U mysql2psql -c "\c" + == Notes, Limitations, Outstanding Issues.. === Todo From ee7cb8ecfb33c5e72dc355ca11d04c74c858dc7c Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 04:05:15 +0800 Subject: [PATCH 14/69] revised/corrected integer signed/unsigned conversions --- lib/mysql2psql/mysql_reader.rb | 12 +++-- lib/mysql2psql/postgres_writer.rb | 5 +-- test/fixtures/seed_integration_tests.sql | 31 +++++++++++-- test/integration/convert_to_file_test.rb | 56 +++++++++++++++++++----- test/integration/mysql_reader_test.rb | 2 +- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb index 9bc8471..0c4a182 100644 --- a/lib/mysql2psql/mysql_reader.rb +++ b/lib/mysql2psql/mysql_reader.rb @@ -27,15 +27,21 @@ def columns def convert_type(type) case type - when /int.* unsigned/ + when /^int.* unsigned/ "bigint" when /bigint/ "bigint" when "bit(1)" "boolean" + when /smallint.* unsigned/ + "integer" + when /smallint/ + "smallint" when "tinyint(1)" "boolean" - when /tinyint/ + when "tinyint(3) unsigned" + "tinyint" + when "tinyint(4)" "tinyint" when /int/ "integer" @@ -74,7 +80,7 @@ def load_columns :primary_key => field[3] == "PRI", :auto_increment => field[5] == "auto_increment" } - desc[:default] = field[4] unless field[4].nil? + desc[:default] = field[4] unless field[4].nil? fields << desc end end diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 1dd504a..847478f 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -16,8 +16,7 @@ def column_type(column) def column_type_info(column) if column[:auto_increment] return "integer DEFAULT nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass) NOT NULL" - end - + end default = column[:default] ? " DEFAULT #{column[:default] == nil ? 'NULL' : "'"+PGconn.escape(column[:default])+"'"}" : nil null = column[:null] ? "" : " NOT NULL" type = @@ -38,7 +37,7 @@ def column_type_info(column) when "bigint" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default "bigint" - when "tinyint" + when /tinyint|smallint/ default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default "smallint" diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index dc89e15..58dc8a7 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -4,23 +4,31 @@ DROP TABLE IF EXISTS numeric_types_basics; CREATE TABLE numeric_types_basics ( id int, f_tinyint TINYINT, + f_tinyint_u TINYINT UNSIGNED, f_smallint SMALLINT, + f_smallint_u SMALLINT UNSIGNED, f_mediumint MEDIUMINT, + f_mediumint_u MEDIUMINT UNSIGNED, f_int INT, + f_int_u INT UNSIGNED, f_integer INTEGER, + f_integer_u INTEGER UNSIGNED, f_bigint BIGINT, + f_bigint_u BIGINT UNSIGNED, f_real REAL, f_double DOUBLE, f_float FLOAT, - f_ufloat FLOAT UNSIGNED, + f_float_u FLOAT UNSIGNED, f_decimal DECIMAL, f_numeric NUMERIC ); INSERT INTO numeric_types_basics VALUES -(1,1,1,1,1,1,1,1,1,1,1,1,1), -(2,2,2,2,2,2,2,2,2,2,2,2,2), -(23,23,23,23,23,23,23,23,23,23,23,23,23); +( 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19), +( 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), +( 3,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23), +( 4, -128, 0,-32768, 0,-8388608, 0,-2147483648, 0,-2147483648, 0,-9223372036854775808, 0, 1, 1, 1, 1, 1, 1), +( 5, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 4294967295, 9223372036854775807, 18446744073709551615, 1, 1, 1, 1, 1, 1); DROP TABLE IF EXISTS basic_autoincrement; @@ -33,6 +41,7 @@ CREATE TABLE basic_autoincrement ( INSERT INTO basic_autoincrement(auto_dummy) VALUES (1),(2),(23); +-- see GH#22 float conversion error DROP TABLE IF EXISTS numeric_type_floats; CREATE TABLE numeric_type_floats ( latitude FLOAT, @@ -41,3 +50,17 @@ CREATE TABLE numeric_type_floats ( INSERT INTO numeric_type_floats(latitude,longitude) VALUES (1.1,2.2); + +-- see GH#18 smallint error +DROP TABLE IF EXISTS gh18_smallint; +CREATE TABLE gh18_smallint ( + s_smallint SMALLINT, + u_smallint SMALLINT UNSIGNED +); + +INSERT INTO gh18_smallint(s_smallint,u_smallint) VALUES +(-32768,32767), +(-1,0), +(32767,65535); + + diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index 58087ec..8364701 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -37,39 +37,71 @@ def test_table_creation end # tests for the conversion of numeric types + def get_basic_numerics_match(column) + Regexp.new('CREATE TABLE "numeric_types_basics".*"' + column + '" ([^\n]*).*\)', Regexp::MULTILINE).match( content ) + end def test_basic_numerics_tinyint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_tinyint' ) + assert_match /smallint/,match[1] + end + def test_basic_numerics_tinyint_u + match = get_basic_numerics_match( 'f_tinyint_u' ) + assert_match /smallint/,match[1] end def test_basic_numerics_smallint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_smallint" integer,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_smallint' ) + assert_match /smallint/,match[1] + end + def test_basic_numerics_smallint_u + match = get_basic_numerics_match( 'f_smallint_u' ) + assert_match /integer/,match[1] end def test_basic_numerics_mediumint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_mediumint" integer,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_mediumint' ) + assert_match /integer/,match[1] end def test_basic_numerics_int - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_int" integer,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_int' ) + assert_match /integer/,match[1] end def test_basic_numerics_integer - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_integer" integer,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_integer' ) + assert_match /integer/,match[1] end def test_basic_numerics_bigint - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_bigint' ) + assert_match /bigint/,match[1] + #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE).match( content ) end def test_basic_numerics_real - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_real' ) + assert_match /double precision/,match[1] + #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE).match( content ) end def test_basic_numerics_double - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_double' ) + assert_match /double precision/,match[1] + #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE).match( content ) end def test_basic_numerics_float - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision.*\)', Regexp::MULTILINE).match( content ) - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_ufloat" double precision.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_float' ) + assert_match /double precision/,match[1] + #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision.*\)', Regexp::MULTILINE).match( content ) + end + def test_basic_numerics_float_u + match = get_basic_numerics_match( 'f_float_u' ) + assert_match /double precision/,match[1] + #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float_u" double precision.*\)', Regexp::MULTILINE).match( content ) end def test_basic_numerics_decimal - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_decimal' ) + assert_match /numeric/,match[1] + #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE).match( content ) end def test_basic_numerics_numeric - assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match( content ) + match = get_basic_numerics_match( 'f_numeric' ) + assert_match /numeric/,match[1] + #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match( content ) end # test autoincrement handling diff --git a/test/integration/mysql_reader_test.rb b/test/integration/mysql_reader_test.rb index 87c1be3..b6506e5 100644 --- a/test/integration/mysql_reader_test.rb +++ b/test/integration/mysql_reader_test.rb @@ -31,7 +31,7 @@ def test_tables_collection assert_equal 'numeric_types_basics', values[0].name end def test_paginated_read - expected_rows=3 + expected_rows=5 page_size=2 expected_pages=(1.0 * expected_rows / page_size).ceil From 4e6f1e500e2699bc075aa45c03b8a2b965e7746a Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 21 Nov 2010 04:15:58 +0800 Subject: [PATCH 15/69] simplify tinyint type matching --- lib/mysql2psql/mysql_reader.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb index 0c4a182..5333845 100644 --- a/lib/mysql2psql/mysql_reader.rb +++ b/lib/mysql2psql/mysql_reader.rb @@ -39,9 +39,7 @@ def convert_type(type) "smallint" when "tinyint(1)" "boolean" - when "tinyint(3) unsigned" - "tinyint" - when "tinyint(4)" + when /tinyint/ "tinyint" when /int/ "integer" @@ -80,7 +78,7 @@ def load_columns :primary_key => field[3] == "PRI", :auto_increment => field[5] == "auto_increment" } - desc[:default] = field[4] unless field[4].nil? + desc[:default] = field[4] unless field[4].nil? fields << desc end end From 2b7664a747bb00b24331655ace1624c9faa703c9 Mon Sep 17 00:00:00 2001 From: Andy Koch Date: Fri, 8 Apr 2011 17:06:12 -0700 Subject: [PATCH 16/69] testing this with pg 0.10.+ --- mysql2psql.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index fb52551..530208f 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -75,16 +75,16 @@ Gem::Specification.new do |s| if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then s.add_runtime_dependency(%q, ["= 2.8.1"]) - s.add_runtime_dependency(%q, ["= 0.9.0"]) + s.add_runtime_dependency(%q, ["~> 0.10.0"]) s.add_development_dependency(%q, [">= 2.1.1"]) else s.add_dependency(%q, ["= 2.8.1"]) - s.add_dependency(%q, ["= 0.9.0"]) + s.add_dependency(%q, ["~> 0.10.0"]) s.add_dependency(%q, [">= 2.1.1"]) end else s.add_dependency(%q, ["= 2.8.1"]) - s.add_dependency(%q, ["= 0.9.0"]) + s.add_dependency(%q, ["~> 0.10.0"]) s.add_dependency(%q, [">= 2.1.1"]) end end From a1d175598c7e576d257b4abbcc1c75f78852a5fc Mon Sep 17 00:00:00 2001 From: Andy Koch Date: Fri, 8 Apr 2011 17:14:37 -0700 Subject: [PATCH 17/69] remove deprecated feature per gem -v 1.7.2 --- mysql2psql.gemspec | 1 - 1 file changed, 1 deletion(-) diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index 530208f..42f2ff4 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -10,7 +10,6 @@ Gem::Specification.new do |s| s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher "] s.date = %q{2010-09-19} - s.default_executable = %q{mysql2psql} s.description = %q{It can create postgresql dump from mysql database or directly load data from mysql to postgresql (at about 100 000 records per minute). Translates most data types and indexes.} s.email = %q{gallagher.paul@gmail.com} From cada55b2aa2eab523d31464aebd3b7017e74944d Mon Sep 17 00:00:00 2001 From: Andy Koch Date: Mon, 11 Apr 2011 11:05:50 -0700 Subject: [PATCH 18/69] add a check for mysql cmd, on my mac it's mysql5 --- test/lib/test_helper.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lib/test_helper.rb b/test/lib/test_helper.rb index ea49633..4b77519 100644 --- a/test/lib/test_helper.rb +++ b/test/lib/test_helper.rb @@ -12,7 +12,8 @@ def seed_test_database options=get_test_config_by_label(:localmysql_to_file_convert_nothing) seedfilepath = "#{File.dirname(__FILE__)}/../fixtures/seed_integration_tests.sql" - rc=system("mysql -u#{options.mysqlusername} #{options.mysqldatabase} < #{seedfilepath}") + mysql_cmd = `which mysql`.empty? ? 'mysql5' : 'mysql' + rc=system("#{mysql_cmd} -u#{options.mysqlusername} #{options.mysqldatabase} < #{seedfilepath}") raise StandardError unless rc return true rescue From 12f0b435509ca67292b15326487809643690caca Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sat, 30 Apr 2011 20:49:54 +0800 Subject: [PATCH 19/69] resync gemspec and rakefile --- Rakefile | 2 +- mysql2psql.gemspec | 87 ++++++++++++++++++++++------------------------ 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/Rakefile b/Rakefile index c7193f3..933ba46 100644 --- a/Rakefile +++ b/Rakefile @@ -30,7 +30,7 @@ begin "Paul Gallagher " ] gem.add_dependency "mysql", "= 2.8.1" - gem.add_dependency "pg", "= 0.9.0" + gem.add_dependency "pg", "~> 0.10.0" gem.add_development_dependency "test-unit", ">= 2.1.1" # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings end diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index 42f2ff4..1b3baa1 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -1,6 +1,6 @@ # Generated by jeweler # DO NOT EDIT THIS FILE DIRECTLY -# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command +# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' # -*- encoding: utf-8 -*- Gem::Specification.new do |s| @@ -9,7 +9,7 @@ Gem::Specification.new do |s| s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher "] - s.date = %q{2010-09-19} + s.date = %q{2011-04-30} s.description = %q{It can create postgresql dump from mysql database or directly load data from mysql to postgresql (at about 100 000 records per minute). Translates most data types and indexes.} s.email = %q{gallagher.paul@gmail.com} @@ -18,58 +18,55 @@ Gem::Specification.new do |s| "README.rdoc" ] s.files = [ - ".gitignore", - "MIT-LICENSE", - "README.rdoc", - "Rakefile", - "bin/mysql2psql", - "lib/mysql2psql.rb", - "lib/mysql2psql/config.rb", - "lib/mysql2psql/config_base.rb", - "lib/mysql2psql/converter.rb", - "lib/mysql2psql/errors.rb", - "lib/mysql2psql/mysql_reader.rb", - "lib/mysql2psql/postgres_db_writer.rb", - "lib/mysql2psql/postgres_file_writer.rb", - "lib/mysql2psql/postgres_writer.rb", - "lib/mysql2psql/version.rb", - "lib/mysql2psql/writer.rb", - "mysql2psql.gemspec", - "test/fixtures/config_all_options.yml", - "test/fixtures/seed_integration_tests.sql", - "test/integration/convert_to_db_test.rb", - "test/integration/convert_to_file_test.rb", - "test/integration/converter_test.rb", - "test/integration/mysql_reader_base_test.rb", - "test/integration/mysql_reader_test.rb", - "test/integration/postgres_db_writer_base_test.rb", - "test/lib/ext_test_unit.rb", - "test/lib/test_helper.rb", - "test/units/config_base_test.rb", - "test/units/config_test.rb", - "test/units/postgres_file_writer_test.rb" + "MIT-LICENSE", + "README.rdoc", + "Rakefile", + "bin/mysql2psql", + "lib/mysql2psql.rb", + "lib/mysql2psql/config.rb", + "lib/mysql2psql/config_base.rb", + "lib/mysql2psql/converter.rb", + "lib/mysql2psql/errors.rb", + "lib/mysql2psql/mysql_reader.rb", + "lib/mysql2psql/postgres_db_writer.rb", + "lib/mysql2psql/postgres_file_writer.rb", + "lib/mysql2psql/postgres_writer.rb", + "lib/mysql2psql/version.rb", + "lib/mysql2psql/writer.rb", + "mysql2psql.gemspec", + "test/fixtures/config_all_options.yml", + "test/fixtures/seed_integration_tests.sql", + "test/integration/convert_to_db_test.rb", + "test/integration/convert_to_file_test.rb", + "test/integration/converter_test.rb", + "test/integration/mysql_reader_base_test.rb", + "test/integration/mysql_reader_test.rb", + "test/integration/postgres_db_writer_base_test.rb", + "test/lib/ext_test_unit.rb", + "test/lib/test_helper.rb", + "test/units/config_base_test.rb", + "test/units/config_test.rb", + "test/units/postgres_file_writer_test.rb" ] s.homepage = %q{http://github.com/tardate/mysql2postgresql} - s.rdoc_options = ["--charset=UTF-8"] s.require_paths = ["lib"] - s.rubygems_version = %q{1.3.7} + s.rubygems_version = %q{1.7.2} s.summary = %q{Tool for converting mysql database to postgresql} s.test_files = [ "test/integration/convert_to_db_test.rb", - "test/integration/convert_to_file_test.rb", - "test/integration/converter_test.rb", - "test/integration/mysql_reader_base_test.rb", - "test/integration/mysql_reader_test.rb", - "test/integration/postgres_db_writer_base_test.rb", - "test/lib/ext_test_unit.rb", - "test/lib/test_helper.rb", - "test/units/config_base_test.rb", - "test/units/config_test.rb", - "test/units/postgres_file_writer_test.rb" + "test/integration/convert_to_file_test.rb", + "test/integration/converter_test.rb", + "test/integration/mysql_reader_base_test.rb", + "test/integration/mysql_reader_test.rb", + "test/integration/postgres_db_writer_base_test.rb", + "test/lib/ext_test_unit.rb", + "test/lib/test_helper.rb", + "test/units/config_base_test.rb", + "test/units/config_test.rb", + "test/units/postgres_file_writer_test.rb" ] if s.respond_to? :specification_version then - current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION s.specification_version = 3 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then From 2bae286677359254c73eebaa26b86888871dae8f Mon Sep 17 00:00:00 2001 From: Andy Koch Date: Mon, 2 May 2011 10:27:10 -0700 Subject: [PATCH 20/69] update pg version to ~> 0.11.0 all tests passing --- mysql2psql.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index 1b3baa1..035260d 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -71,16 +71,16 @@ Gem::Specification.new do |s| if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then s.add_runtime_dependency(%q, ["= 2.8.1"]) - s.add_runtime_dependency(%q, ["~> 0.10.0"]) + s.add_runtime_dependency(%q, ["~> 0.11.0"]) s.add_development_dependency(%q, [">= 2.1.1"]) else s.add_dependency(%q, ["= 2.8.1"]) - s.add_dependency(%q, ["~> 0.10.0"]) + s.add_dependency(%q, ["~> 0.11.0"]) s.add_dependency(%q, [">= 2.1.1"]) end else s.add_dependency(%q, ["= 2.8.1"]) - s.add_dependency(%q, ["~> 0.10.0"]) + s.add_dependency(%q, ["~> 0.11.0"]) s.add_dependency(%q, [">= 2.1.1"]) end end From 3cce62510cb2cff8eb7fc41f43b1ebe76080dcaf Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Tue, 3 May 2011 06:13:18 +0800 Subject: [PATCH 21/69] resync rakefile + gemspec for pg ~> 0.11.0 --- Rakefile | 2 +- mysql2psql.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index 933ba46..c774b1d 100644 --- a/Rakefile +++ b/Rakefile @@ -30,7 +30,7 @@ begin "Paul Gallagher " ] gem.add_dependency "mysql", "= 2.8.1" - gem.add_dependency "pg", "~> 0.10.0" + gem.add_dependency "pg", "~> 0.11.0" gem.add_development_dependency "test-unit", ">= 2.1.1" # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings end diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index 035260d..f7f29b6 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -9,7 +9,7 @@ Gem::Specification.new do |s| s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher "] - s.date = %q{2011-04-30} + s.date = %q{2011-05-02} s.description = %q{It can create postgresql dump from mysql database or directly load data from mysql to postgresql (at about 100 000 records per minute). Translates most data types and indexes.} s.email = %q{gallagher.paul@gmail.com} From a554cae0f51ce116274aa07a71f7164d39c2c6c2 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Tue, 3 May 2011 06:22:08 +0800 Subject: [PATCH 22/69] add CHANGELOG and step gem version to 0.1.1 --- CHANGELOG | 11 +++++++++++ lib/mysql2psql/version.rb | 2 +- mysql2psql.gemspec | 3 ++- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..d969bba --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,11 @@ +0.1.1 May 3, 2011 +=== + +* updated for pg driver ~> 0.11.0 +* add a check for mysql cmd, e.g. mysql or mysql5 +* remove deprecated feature per gem -v 1.7.2 + +0.1.0 September 18, 2010 +=== + +* Initial packaging as a gem \ No newline at end of file diff --git a/lib/mysql2psql/version.rb b/lib/mysql2psql/version.rb index c61d5c1..b791a1a 100644 --- a/lib/mysql2psql/version.rb +++ b/lib/mysql2psql/version.rb @@ -2,7 +2,7 @@ class Mysql2psql module Version MAJOR = 0 MINOR = 1 - PATCH = 0 + PATCH = 1 STRING = [MAJOR, MINOR, PATCH].compact.join('.') end diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index f7f29b6..8ad2931 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -5,7 +5,7 @@ Gem::Specification.new do |s| s.name = %q{mysql2psql} - s.version = "0.1.0" + s.version = "0.1.1" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher "] @@ -18,6 +18,7 @@ Gem::Specification.new do |s| "README.rdoc" ] s.files = [ + "CHANGELOG", "MIT-LICENSE", "README.rdoc", "Rakefile", From dbef1077aab8c66ef6be76e54d7d4e5d1b834281 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Tue, 3 May 2011 06:39:42 +0800 Subject: [PATCH 23/69] Update README to reflect gem availability --- README.rdoc | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/README.rdoc b/README.rdoc index 7d3b476..0c2d493 100644 --- a/README.rdoc +++ b/README.rdoc @@ -12,9 +12,7 @@ Mysql2psql is packaged as a gem. Install as usual (use sudo if required). $ gem install mysql2psql -NB: the gem hasn't been formally released on http://rubygems.org/ yet. For now you need to clone the git repo at http://github.com/tardate/mysql2postgres and 'rake install'. - -After installation, the "mysql2psql" command will be available. +After installation, the "mysql2psql" command will be available. When run, it will generate a default configuration file in the current directory if a config file is not already found. The default configuration filename is mysql2psql.yml. @@ -43,7 +41,7 @@ After editing the configuration file and launching tool, you will see something == Database driver dependencies -Mysql2psql uses the 'mysql' and 'pg' gems for database connectivity. +Mysql2psql uses the 'mysql' and 'pg' gems for database connectivity. Mysql2psql gem installation should automatically install these too. If you encounter any issues with db connectivity, verify that the 'mysql' and 'pg' gems are installed and working correctly first. @@ -53,15 +51,25 @@ The 'mysql' gem in particular may need a hint on where to find the mysql headers NB: With Ruby 1.8.x, the gem install will usually complain about "No definition for ..." errors. These are non-fatal and just affect the documentation install. This doesn't happen with Ruby 1.9.2. +== Contributing to Mysql2psql + +* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet +* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it +* Fork the project +* Start a feature/bugfix branch +* Commit and push until you are happy with your contribution +* Make sure to add tests for it. This is important so no-one unintentionally breaks it in a future version. +* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so the gem maintainers can cherry-pick around it. +* Send a pull request + == Getting the source -Mysql2psql was first produced by Max Lapshin who maintains the master +Mysql2psql was first produced by Max Lapshin who maintains the master repository at http://github.com/maxlapshin/mysql2postgres -The gem packaged version you are currently looking at has yet to be merged into the master repo. The gem bundling and refactoring for testing was done by Paul Gallagher and the repository is at http://github.com/tardate/mysql2postgres -Gem packaging is done with Jeweler. Note that on windows this means you must run under mingw or other shell that supports git to do the gem bundling (but you can gem install the built gem in the normal Windows shell). +Gem packaging is done with Jeweler. Note that on windows this means you must run under mingw or other shell that supports git to do the gem bundling (but you can gem install the built gem in the normal Windows shell). == Running tests If you fork/clone the project, you will want to run the test suite (and add to it if you are developing new features or fixing bugs). @@ -69,18 +77,18 @@ If you fork/clone the project, you will want to run the test suite (and add to i A suite of tests are provided and are run using rake (rake -T for more info) rake test:units rake test:integration - rake test + rake test Rake without parameters (or "rake test") will run both the unit and integration tests. Unit tests are small standalone tests of the mysql2psql codebase that have no external dependencies (like a database) -Integration tests require suitable mysql and postgres databases to be setup in advance. +Integration tests require suitable mysql and postgres databases to be setup in advance. Running the integration tests *will* rewrite data and schema for the "mysql2psql_test" databases in mysql and postgres. Don't use this database for anything else! -mysql on localhost:3306 -- database created called "mysql2psql_test" -- user setup for "mysql2psql" with no password +Setting up mysql on localhost:3306 +- create a database called "mysql2psql_test" +- setup a user "mysql2psql" with no password e.g. @@ -90,9 +98,9 @@ e.g. mysql -umysql2psql -e "select database(),'yes' as connected" mysql2psql_test -postgres on localhost:5432 -- database created called "mysql2psql_test" -- role (user) access setup for "mysql2psql" with no password +Setting up PostgreSQL on localhost:5432 +- create a database called "mysql2psql_test" +- setup a role (user) "mysql2psql" with no password e.g. @@ -106,7 +114,6 @@ e.g. === Todo - more tests -- release gem === note from mgkimsal I'm still having trouble with bit(1)/boolean fields @@ -136,7 +143,3 @@ Other contributors (in git log order): - Neszt Tibor - Miroslav Kratochvil - Paul Gallagher - - - - From 8c6a1feff6155c5f47c8ee68d290b33aabe0f0e0 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Fri, 26 Aug 2011 14:58:11 -0700 Subject: [PATCH 24/69] make tests pass by moving startup/shutdown into setup/teardown. fixes uninitialized class variable errors: https://gist.github.com/4c76c2894b51ac911fba --- test/integration/convert_to_db_test.rb | 26 +++++++------------ test/integration/convert_to_file_test.rb | 26 +++++-------------- test/integration/converter_test.rb | 12 +++------ test/integration/mysql_reader_base_test.rb | 20 +++++--------- test/integration/mysql_reader_test.rb | 16 ++++-------- .../postgres_db_writer_base_test.rb | 18 ++++--------- 6 files changed, 36 insertions(+), 82 deletions(-) diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 47190a0..d652902 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -4,28 +4,22 @@ class ConvertToDbTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_db_convert_all) - @@mysql2psql = Mysql2psql.new([@@options.filepath]) - @@mysql2psql.convert - @@mysql2psql.writer.open - end - def shutdown - @@mysql2psql.writer.close - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options=get_test_config_by_label(:localmysql_to_db_convert_all) + @mysql2psql = Mysql2psql.new([@options.filepath]) + @mysql2psql.convert + @mysql2psql.writer.open end def teardown + @mysql2psql.writer.close + delete_files_for_test_config(@options) end def test_table_creation - assert_true @@mysql2psql.writer.exists?('numeric_types_basics') - assert_true @@mysql2psql.writer.exists?('basic_autoincrement') - assert_true @@mysql2psql.writer.exists?('numeric_type_floats') + assert_true @mysql2psql.writer.exists?('numeric_types_basics') + assert_true @mysql2psql.writer.exists?('basic_autoincrement') + assert_true @mysql2psql.writer.exists?('numeric_type_floats') end end \ No newline at end of file diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index 8364701..1f12a6a 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -4,30 +4,18 @@ class ConvertToFileTest < Test::Unit::TestCase - class << self - - # This is a suite of tests to verify conversion of full schema and data to file. - # The export is done once in the class setup. - # Tests inspect specific features of the converted file, - # the contents of which are preloaded as the :content attribute of this class - # - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_file_convert_all) - @@mysql2psql = Mysql2psql.new([@@options.filepath]) - @@mysql2psql.convert - @@content = IO.read(@@mysql2psql.options.destfile) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options=get_test_config_by_label(:localmysql_to_file_convert_all) + @mysql2psql = Mysql2psql.new([@options.filepath]) + @mysql2psql.convert + @content = IO.read(@mysql2psql.options.destfile) end def teardown + delete_files_for_test_config(@options) end def content - @@content + @content end # verify table creation diff --git a/test/integration/converter_test.rb b/test/integration/converter_test.rb index a362d9d..df3947a 100644 --- a/test/integration/converter_test.rb +++ b/test/integration/converter_test.rb @@ -4,18 +4,12 @@ class ConverterTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_file_convert_nothing) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @@options=get_test_config_by_label(:localmysql_to_file_convert_nothing) end def teardown + delete_files_for_test_config(@@options) end def options @@options diff --git a/test/integration/mysql_reader_base_test.rb b/test/integration/mysql_reader_base_test.rb index fd03133..4abe0ff 100644 --- a/test/integration/mysql_reader_base_test.rb +++ b/test/integration/mysql_reader_base_test.rb @@ -4,31 +4,23 @@ class MysqlReaderBaseTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options = get_test_config_by_label(:localmysql_to_file_convert_nothing) end + def teardown - end - def options - @@options + delete_files_for_test_config(@options) end def test_mysql_connection assert_nothing_raised do - reader = Mysql2psql::MysqlReader.new(options) + reader = Mysql2psql::MysqlReader.new(@options) end end def test_mysql_reconnect assert_nothing_raised do - reader = Mysql2psql::MysqlReader.new(options) + reader = Mysql2psql::MysqlReader.new(@options) reader.reconnect end end diff --git a/test/integration/mysql_reader_test.rb b/test/integration/mysql_reader_test.rb index b6506e5..e904ed6 100644 --- a/test/integration/mysql_reader_test.rb +++ b/test/integration/mysql_reader_test.rb @@ -2,22 +2,16 @@ class MysqlReaderTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) - @@reader=get_test_reader(@@options) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options = get_test_config_by_label(:localmysql_to_file_convert_nothing) + @reader=get_test_reader(@options) end def teardown + delete_files_for_test_config(@options) end def reader - @@reader + @reader end def test_db_connection diff --git a/test/integration/postgres_db_writer_base_test.rb b/test/integration/postgres_db_writer_base_test.rb index 8e25129..56d0fa8 100644 --- a/test/integration/postgres_db_writer_base_test.rb +++ b/test/integration/postgres_db_writer_base_test.rb @@ -4,26 +4,18 @@ class PostgresDbWriterBaseTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_db_convert_nothing) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options = get_test_config_by_label(:localmysql_to_db_convert_nothing) end + def teardown - end - def options - @@options + delete_files_for_test_config(@options) end def test_pg_connection assert_nothing_raised do - reader = Mysql2psql::PostgresDbWriter.new(options) + reader = Mysql2psql::PostgresDbWriter.new(@options) end end From 3db085bf0e19f402959eee91843decd99275a62e Mon Sep 17 00:00:00 2001 From: James Coleman Date: Wed, 7 Sep 2011 14:46:18 -0400 Subject: [PATCH 25/69] Fix indentation and remove a tab. --- lib/mysql2psql/postgres_writer.rb | 57 +++++++++++++++---------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 847478f..26dd9ce 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -8,11 +8,11 @@ class PostgresWriter < Writer def column_description(column) "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}" end - + def column_type(column) column_type_info(column).split(" ").first end - + def column_type_info(column) if column[:auto_increment] return "integer DEFAULT nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass) NOT NULL" @@ -21,7 +21,7 @@ def column_type_info(column) null = column[:null] ? "" : " NOT NULL" type = case column[:type] - + # String types when "char" default = default + "::char" if default @@ -29,7 +29,7 @@ def column_type_info(column) when "varchar" default = default + "::character varying" if default "character varying(#{column[:length]})" - + # Integer and numeric types when "integer" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default @@ -40,7 +40,7 @@ def column_type_info(column) when /tinyint|smallint/ default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default "smallint" - + when "boolean" default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default "boolean" @@ -104,35 +104,34 @@ def column_type_info(column) end "#{type}#{default}#{null}" end - + def process_row(table, row) - table.columns.each_with_index do |column, index| + table.columns.each_with_index do |column, index| + if column[:type] == "time" + row[index] = "%02d:%02d:%02d" % [row[index].hour, row[index].minute, row[index].second] + end - if column[:type] == "time" - row[index] = "%02d:%02d:%02d" % [row[index].hour, row[index].minute, row[index].second] - end - - if row[index].is_a?(Mysql::Time) - row[index] = row[index].to_s.gsub('0000-00-00 00:00', '1970-01-01 00:00') - row[index] = row[index].to_s.gsub('0000-00-00 00:00:00', '1970-01-01 00:00:00') - end - - if column_type(column) == "boolean" - row[index] = row[index] == 1 ? 't' : row[index] == 0 ? 'f' : row[index] - end - - if row[index].is_a?(String) - if column_type(column) == "bytea" - row[index] = PGconn.escape_bytea(row[index]) - else - row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r').gsub(/\0/, '') - end + if row[index].is_a?(Mysql::Time) + row[index] = row[index].to_s.gsub('0000-00-00 00:00', '1970-01-01 00:00') + row[index] = row[index].to_s.gsub('0000-00-00 00:00:00', '1970-01-01 00:00:00') + end + + if column_type(column) == "boolean" + row[index] = row[index] == 1 ? 't' : row[index] == 0 ? 'f' : row[index] + end + + if row[index].is_a?(String) + if column_type(column) == "bytea" + row[index] = PGconn.escape_bytea(row[index]) + else + row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r').gsub(/\0/, '') end - - row[index] = '\N' if !row[index] end + + row[index] = '\N' if !row[index] + end end - + def truncate(table) end From 8e624aebb2171c364467abdee9d116e98daac11d Mon Sep 17 00:00:00 2001 From: James Coleman Date: Wed, 7 Sep 2011 15:29:15 -0400 Subject: [PATCH 26/69] Remove dreadful nested ternary. --- lib/mysql2psql/postgres_writer.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 26dd9ce..ef28cfe 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -117,7 +117,16 @@ def process_row(table, row) end if column_type(column) == "boolean" - row[index] = row[index] == 1 ? 't' : row[index] == 0 ? 'f' : row[index] + current_value = row[index] + row[index] = ( + if current_value == 1 + 't' + elsif current_value == 0 + 'f' + else + current_value + end + ) end if row[index].is_a?(String) From a1fdd91f584220f7ed038fdd16a9bbac9670c31c Mon Sep 17 00:00:00 2001 From: James Coleman Date: Wed, 7 Sep 2011 15:30:48 -0400 Subject: [PATCH 27/69] Fix boolean parsing bug. The value of a boolean column may actually be the string containing the 0 or 1 byte (note: not character) rather than the integer 0 or 1. The row processing should respect that and still output a 't' or 'f' rather than the original value string. --- lib/mysql2psql/postgres_writer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index ef28cfe..11a818e 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -119,9 +119,9 @@ def process_row(table, row) if column_type(column) == "boolean" current_value = row[index] row[index] = ( - if current_value == 1 + if current_value == 1 || current_value == "\1" 't' - elsif current_value == 0 + elsif current_value == 0 || current_value == "\0" 'f' else current_value From 10f92f5dac17d2f4f4f6b590c97a56193255d4b5 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Fri, 26 Aug 2011 14:58:11 -0700 Subject: [PATCH 28/69] make tests pass by moving startup/shutdown into setup/teardown. fixes uninitialized class variable errors: https://gist.github.com/4c76c2894b51ac911fba --- test/integration/convert_to_db_test.rb | 26 +++++++------------ test/integration/convert_to_file_test.rb | 26 +++++-------------- test/integration/converter_test.rb | 12 +++------ test/integration/mysql_reader_base_test.rb | 20 +++++--------- test/integration/mysql_reader_test.rb | 16 ++++-------- .../postgres_db_writer_base_test.rb | 18 ++++--------- 6 files changed, 36 insertions(+), 82 deletions(-) diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 47190a0..d652902 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -4,28 +4,22 @@ class ConvertToDbTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_db_convert_all) - @@mysql2psql = Mysql2psql.new([@@options.filepath]) - @@mysql2psql.convert - @@mysql2psql.writer.open - end - def shutdown - @@mysql2psql.writer.close - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options=get_test_config_by_label(:localmysql_to_db_convert_all) + @mysql2psql = Mysql2psql.new([@options.filepath]) + @mysql2psql.convert + @mysql2psql.writer.open end def teardown + @mysql2psql.writer.close + delete_files_for_test_config(@options) end def test_table_creation - assert_true @@mysql2psql.writer.exists?('numeric_types_basics') - assert_true @@mysql2psql.writer.exists?('basic_autoincrement') - assert_true @@mysql2psql.writer.exists?('numeric_type_floats') + assert_true @mysql2psql.writer.exists?('numeric_types_basics') + assert_true @mysql2psql.writer.exists?('basic_autoincrement') + assert_true @mysql2psql.writer.exists?('numeric_type_floats') end end \ No newline at end of file diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index 8364701..1f12a6a 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -4,30 +4,18 @@ class ConvertToFileTest < Test::Unit::TestCase - class << self - - # This is a suite of tests to verify conversion of full schema and data to file. - # The export is done once in the class setup. - # Tests inspect specific features of the converted file, - # the contents of which are preloaded as the :content attribute of this class - # - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_file_convert_all) - @@mysql2psql = Mysql2psql.new([@@options.filepath]) - @@mysql2psql.convert - @@content = IO.read(@@mysql2psql.options.destfile) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options=get_test_config_by_label(:localmysql_to_file_convert_all) + @mysql2psql = Mysql2psql.new([@options.filepath]) + @mysql2psql.convert + @content = IO.read(@mysql2psql.options.destfile) end def teardown + delete_files_for_test_config(@options) end def content - @@content + @content end # verify table creation diff --git a/test/integration/converter_test.rb b/test/integration/converter_test.rb index a362d9d..df3947a 100644 --- a/test/integration/converter_test.rb +++ b/test/integration/converter_test.rb @@ -4,18 +4,12 @@ class ConverterTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_file_convert_nothing) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @@options=get_test_config_by_label(:localmysql_to_file_convert_nothing) end def teardown + delete_files_for_test_config(@@options) end def options @@options diff --git a/test/integration/mysql_reader_base_test.rb b/test/integration/mysql_reader_base_test.rb index fd03133..4abe0ff 100644 --- a/test/integration/mysql_reader_base_test.rb +++ b/test/integration/mysql_reader_base_test.rb @@ -4,31 +4,23 @@ class MysqlReaderBaseTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options = get_test_config_by_label(:localmysql_to_file_convert_nothing) end + def teardown - end - def options - @@options + delete_files_for_test_config(@options) end def test_mysql_connection assert_nothing_raised do - reader = Mysql2psql::MysqlReader.new(options) + reader = Mysql2psql::MysqlReader.new(@options) end end def test_mysql_reconnect assert_nothing_raised do - reader = Mysql2psql::MysqlReader.new(options) + reader = Mysql2psql::MysqlReader.new(@options) reader.reconnect end end diff --git a/test/integration/mysql_reader_test.rb b/test/integration/mysql_reader_test.rb index b6506e5..e904ed6 100644 --- a/test/integration/mysql_reader_test.rb +++ b/test/integration/mysql_reader_test.rb @@ -2,22 +2,16 @@ class MysqlReaderTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) - @@reader=get_test_reader(@@options) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options = get_test_config_by_label(:localmysql_to_file_convert_nothing) + @reader=get_test_reader(@options) end def teardown + delete_files_for_test_config(@options) end def reader - @@reader + @reader end def test_db_connection diff --git a/test/integration/postgres_db_writer_base_test.rb b/test/integration/postgres_db_writer_base_test.rb index 8e25129..56d0fa8 100644 --- a/test/integration/postgres_db_writer_base_test.rb +++ b/test/integration/postgres_db_writer_base_test.rb @@ -4,26 +4,18 @@ class PostgresDbWriterBaseTest < Test::Unit::TestCase - class << self - def startup - seed_test_database - @@options=get_test_config_by_label(:localmysql_to_db_convert_nothing) - end - def shutdown - delete_files_for_test_config(@@options) - end - end def setup + seed_test_database + @options = get_test_config_by_label(:localmysql_to_db_convert_nothing) end + def teardown - end - def options - @@options + delete_files_for_test_config(@options) end def test_pg_connection assert_nothing_raised do - reader = Mysql2psql::PostgresDbWriter.new(options) + reader = Mysql2psql::PostgresDbWriter.new(@options) end end From 1e1ed6a82086d30cc2e19c3411ded4a821787a17 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Thu, 8 Sep 2011 14:03:01 -0400 Subject: [PATCH 29/69] Remove unused variable. --- lib/mysql2psql/postgres_db_writer.rb | 1 - lib/mysql2psql/postgres_file_writer.rb | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index 5949e3c..8265e1c 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -149,7 +149,6 @@ def write_contents(table, reader) puts "Loading #{table.name}..." STDOUT.flush _counter = reader.paginated_read(table, 1000) do |row, counter| - line = [] process_row(table, row) @conn.put_copy_data(row.join("\t") + "\n") diff --git a/lib/mysql2psql/postgres_file_writer.rb b/lib/mysql2psql/postgres_file_writer.rb index 5fc2ad7..5350c7d 100644 --- a/lib/mysql2psql/postgres_file_writer.rb +++ b/lib/mysql2psql/postgres_file_writer.rb @@ -125,7 +125,6 @@ def write_contents(table, reader) EOF reader.paginated_read(table, 1000) do |row, counter| - line = [] process_row(table, row) @f << row.join("\t") + "\n" end From 32da4e08bf58f5e60e82adcfd07ca48957fbbb11 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Thu, 8 Sep 2011 14:06:54 -0400 Subject: [PATCH 30/69] Add assert_nil extension to Test::Unit. --- test/lib/ext_test_unit.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/lib/ext_test_unit.rb b/test/lib/ext_test_unit.rb index 025e35e..1e1eabb 100644 --- a/test/lib/ext_test_unit.rb +++ b/test/lib/ext_test_unit.rb @@ -26,5 +26,8 @@ def assert_false(object, message="") def assert_true(object, message="") assert_equal(true, object, message) end + def assert_nil(object, message="") + assert_equal(nil, object, message) + end end From 7432927ba66c4ce141d6134a7b79ae11110eef7f Mon Sep 17 00:00:00 2001 From: James Coleman Date: Thu, 8 Sep 2011 14:07:41 -0400 Subject: [PATCH 31/69] Fix boolean handling of BIT(1) type columns. MySQL reports the defaults of BIT(1) columns as b'0' or b'1' corresponding to true/false instead of 0 or 1. BIT(1) columns return their values as "\0" or "\1" rather than 0 or 1. TINYINT(1) columns with non-zero (assumed non-null) values equivalent to true (rather than just the value 1) and the value 0 as false according to the MySQL documentation at http://dev.mysql.com/doc/refman/5.0/en/numeric-type-overview.html (which states: "BOOL, BOOLEAN: These types are synonyms for TINYINT(1). A value of zero is considered false. Nonzero values are considered true.") Respect these expected value types/ranges when converting BIT(1) and TINYINT(1) for both default value and field value translation. Includes tests. --- lib/mysql2psql/postgres_writer.rb | 29 ++++++++++++++++++------ test/fixtures/seed_integration_tests.sql | 17 ++++++++++++++ test/integration/convert_to_db_test.rb | 29 ++++++++++++++++++++++++ test/integration/convert_to_file_test.rb | 13 +++++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 11a818e..bd4980e 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -42,7 +42,18 @@ def column_type_info(column) "smallint" when "boolean" - default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default + default_value = ( + case column[:default] + when nil + 'NULL' + when 0, '0', "b'0'" + 'false' + else + # Case for 1, '1', "b'1'" (for BIT(1) the data type), or anything non-nil and non-zero (for the TINYINT(1) type) + 'true' + end + ) + default = " DEFAULT #{default_value}" if default "boolean" when "real" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default @@ -117,14 +128,15 @@ def process_row(table, row) end if column_type(column) == "boolean" - current_value = row[index] row[index] = ( - if current_value == 1 || current_value == "\1" - 't' - elsif current_value == 0 || current_value == "\0" + case row[index] + when nil + nil + when 0, "\0" 'f' else - current_value + # Case for 1, "\1" (for the BIT(1) data type), or anything non-nil and non-zero (to handle the TINYINT(1) type) + 't' end ) end @@ -137,7 +149,10 @@ def process_row(table, row) end end - row[index] = '\N' if !row[index] + # Note: '\N' not "\N" is correct here: + # The string containing the literal backslash followed by 'N' + # represents database NULL value in PostgreSQL's text mode. + row[index] = '\N' if row[index].nil? end end diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 58dc8a7..a636e0b 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -63,4 +63,21 @@ INSERT INTO gh18_smallint(s_smallint,u_smallint) VALUES (-1,0), (32767,65535); +-- see https://github.com/maxlapshin/mysql2postgres/issues/27 +DROP TABLE IF EXISTS test_boolean_conversion; +CREATE TABLE test_boolean_conversion ( + test_name VARCHAR(25), + bit_1 BIT(1), + tinyint_1 TINYINT(1), + bit_1_default_0 BIT(1) DEFAULT 0, + bit_1_default_1 BIT(1) DEFAULT 1, + tinyint_1_default_0 TINYINT(1) DEFAULT 0, + tinyint_1_default_1 TINYINT(1) DEFAULT 1, + tinyint_1_default_2 TINYINT(1) DEFAULT 2 -- Test the fact that 1 byte isn't limited to [0,1] +); +INSERT INTO test_boolean_conversion (test_name, bit_1, tinyint_1) +VALUES ('test-null', NULL, NULL), + ('test-false', 0, 0), + ('test-true', 1, 1); +INSERT INTO test_boolean_conversion (test_name, tinyint_1) VALUES ('test-true-nonzero', 2); diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index d652902..491beed 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -16,10 +16,39 @@ def teardown delete_files_for_test_config(@options) end + def exec_sql_on_psql(sql, parameters=nil) + @mysql2psql.writer.conn.exec(sql, parameters) + end + + def get_boolean_test_record(name) + exec_sql_on_psql('SELECT * FROM test_boolean_conversion WHERE test_name = $1', [name]).first + end + def test_table_creation assert_true @mysql2psql.writer.exists?('numeric_types_basics') assert_true @mysql2psql.writer.exists?('basic_autoincrement') assert_true @mysql2psql.writer.exists?('numeric_type_floats') end + def test_boolean_conversion_to_true + true_record = get_boolean_test_record('test-true') + assert_equal 't', true_record['bit_1'] + assert_equal 't', true_record['tinyint_1'] + + true_nonzero_record = get_boolean_test_record('test-true-nonzero') + assert_equal 't', true_nonzero_record['tinyint_1'] + end + + def test_boolean_conversion_to_false + false_record = get_boolean_test_record('test-false') + assert_equal 'f', false_record['bit_1'] + assert_equal 'f', false_record['tinyint_1'] + end + + def test_boolean_conversion_of_null + null_record = get_boolean_test_record('test-null') + assert_nil null_record['bit_1'] + assert_nil null_record['tinyint_1'] + end + end \ No newline at end of file diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index 1f12a6a..c593ffb 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -92,6 +92,19 @@ def test_basic_numerics_numeric #assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match( content ) end + # test boolean conversion + def get_boolean_column_definition_match(column) + Regexp.new('CREATE TABLE "test_boolean_conversion".*"' + column + '" ([^\n]*)[^;]*', Regexp::MULTILINE).match(content)[1] + end + def test_boolean_default_values + assert_match /DEFAULT false/, get_boolean_column_definition_match('bit_1_default_0') + assert_match /DEFAULT false/, get_boolean_column_definition_match('tinyint_1_default_0') + + assert_match /DEFAULT true/, get_boolean_column_definition_match('bit_1_default_1') + assert_match /DEFAULT true/, get_boolean_column_definition_match('tinyint_1_default_1') + assert_match /DEFAULT true/, get_boolean_column_definition_match('tinyint_1_default_2') + end + # test autoincrement handling def test_autoincrement assert_not_nil Regexp.new('CREATE TABLE "basic_autoincrement".*"auto_id" integer DEFAULT.*\)', Regexp::MULTILINE).match( content ) From f032930d4ff4ac16e024ec1940d36d31b1b9b377 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Thu, 8 Sep 2011 15:26:21 -0400 Subject: [PATCH 32/69] Remove incorrectly indented space. --- lib/mysql2psql/mysql_reader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb index 5333845..46c83f3 100644 --- a/lib/mysql2psql/mysql_reader.rb +++ b/lib/mysql2psql/mysql_reader.rb @@ -52,7 +52,7 @@ def convert_type(type) when /float/ "float" when /real|double/ - "double precision" + "double precision" else type end From 7f5d0304bcb894f9bc7f5c71ccb6900ab77620a7 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Thu, 8 Sep 2011 15:27:08 -0400 Subject: [PATCH 33/69] Don't do unnecessary calculation. --- lib/mysql2psql/postgres_writer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index bd4980e..5befc69 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -127,7 +127,7 @@ def process_row(table, row) row[index] = row[index].to_s.gsub('0000-00-00 00:00:00', '1970-01-01 00:00:00') end - if column_type(column) == "boolean" + if column[:type] == 'boolean' row[index] = ( case row[index] when nil From 98d40dd481d44b97f3259a9d8663c24289b254d1 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Thu, 8 Sep 2011 15:27:26 -0400 Subject: [PATCH 34/69] Factor column information methods. --- lib/mysql2psql/postgres_writer.rb | 166 ++++++++++++++---------------- 1 file changed, 77 insertions(+), 89 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 5befc69..7020070 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -13,36 +13,58 @@ def column_type(column) column_type_info(column).split(" ").first end - def column_type_info(column) + def column_type(column) if column[:auto_increment] - return "integer DEFAULT nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass) NOT NULL" - end - default = column[:default] ? " DEFAULT #{column[:default] == nil ? 'NULL' : "'"+PGconn.escape(column[:default])+"'"}" : nil - null = column[:null] ? "" : " NOT NULL" - type = - case column[:type] - - # String types - when "char" - default = default + "::char" if default - "character(#{column[:length]})" - when "varchar" - default = default + "::character varying" if default - "character varying(#{column[:length]})" - - # Integer and numeric types - when "integer" - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default - "integer" - when "bigint" - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default - "bigint" - when /tinyint|smallint/ - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default - "smallint" - - when "boolean" - default_value = ( + 'integer' + else + case column[:type] + when 'char' + "character(#{column[:length]})" + when 'varchar' + "character varying(#{column[:length]})" + when /tinyint|smallint/ + 'smallint' + when 'real', /float/, 'double precision' + 'double precision' + when 'decimal' + "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" + when 'datetime', 'timestamp' + 'timestamp without time zone' + when 'time' + 'time without time zone' + when 'tinyblob', 'mediumblob', 'longblob', 'blob', 'varbinary' + 'bytea' + when 'tinytext', 'mediumtext', 'longtext', 'text' + 'text' + when /^enum/ + enum = column[:type].gsub(/enum|\(|\)/, '') + max_enum_size = enum.split(',').map{ |check| check.size() -2}.sort[-1] + "character varying(#{max_enum_size}) check( #{column[:name]} in (#{enum}))" + when 'integer', 'bigint', 'boolean', 'date' + column[:type] + else + puts "Unknown #{column.inspect}" + '' + end + end + end + + def column_default(column) + if column[:auto_increment] + "nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass)" + elsif column[:default] + case column[:type] + when 'char' + "#{column[:default]}::char" + when 'varchar', /^enum/ + "#{column[:default]}::character varying" + when 'integer', 'bigint', /tinyint|smallint/ + column[:default].to_i + when 'real', /float/ + column[:default].to_f + when 'decimal', 'double precision' + column[:default] + when 'boolean' case column[:default] when nil 'NULL' @@ -52,73 +74,39 @@ def column_type_info(column) # Case for 1, '1', "b'1'" (for BIT(1) the data type), or anything non-nil and non-zero (for the TINYINT(1) type) 'true' end - ) - default = " DEFAULT #{default_value}" if default - "boolean" - when "real" - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default - "double precision" - when /float/ - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default - "double precision" - when "decimal" - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default - "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" - - when "double precision" - default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default - "double precision" - - # Mysql datetime fields - when "datetime" - default = nil - "timestamp without time zone" - when "date" - default = nil - "date" - when "timestamp" - default = " DEFAULT CURRENT_TIMESTAMP" if column[:default] == "CURRENT_TIMESTAMP" - default = " DEFAULT '1970-01-01 00:00'" if column[:default] == "0000-00-00 00:00" - default = " DEFAULT '1970-01-01 00:00:00'" if column[:default] == "0000-00-00 00:00:00" - "timestamp without time zone" - when "time" - default = " DEFAULT NOW()" if default - "time without time zone" + when 'datetime', 'date' + nil # TODO: I have a hard time believing that this is the correct logic. + when 'timestamp' + case column[:default] + when 'CURRENT_TIMESTAMP' + 'CURRENT_TIMESTAMP' + when '0000-00-00 00:00' + "'1970-01-01 00:00'" + when '0000-00-00 00:00:00' + "'1970-01-01 00:00:00'" + end + when 'time' + 'NOW()' # TODO: Check this logic... + else + "#{column[:default] == nil ? 'NULL' : "'"+PGconn.escape(column[:default])+"'"}" + end + end + end - when "tinyblob" - "bytea" - when "mediumblob" - "bytea" - when "longblob" - "bytea" - when "blob" - "bytea" - when "varbinary" - "bytea" - when "tinytext" - "text" - when "mediumtext" - "text" - when "longtext" - "text" - when "text" - "text" - when /^enum/ - default = default + "::character varying" if default - enum = column[:type].gsub(/enum|\(|\)/, '') - max_enum_size = enum.split(',').map{ |check| check.size() -2}.sort[-1] - "character varying(#{max_enum_size}) check( #{column[:name]} in (#{enum}))" + def column_type_info(column) + type = column_type(column) + if type + not_null = !column[:null] || column[:auto_increment] ? ' NOT NULL' : '' + default = column[:default] || column[:auto_increment] ? ' DEFAULT ' + column_default(column) : '' + "#{type}#{default}#{not_null}" else - puts "Unknown #{column.inspect}" - column[:type].inspect - return "" + '' end - "#{type}#{default}#{null}" end def process_row(table, row) table.columns.each_with_index do |column, index| - if column[:type] == "time" + if column[:type] == 'time' row[index] = "%02d:%02d:%02d" % [row[index].hour, row[index].minute, row[index].second] end From f9b6acae437df7ee3ca6d1b0b683b8be141cf936 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Thu, 8 Sep 2011 15:35:11 -0400 Subject: [PATCH 35/69] Note about boolean issues is no longer applicable. Oh, and pretentiously add myself to the contributors list. --- README.rdoc | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/README.rdoc b/README.rdoc index 0c2d493..c7ea95c 100644 --- a/README.rdoc +++ b/README.rdoc @@ -115,17 +115,6 @@ e.g. === Todo - more tests -=== note from mgkimsal -I'm still having trouble with bit(1)/boolean fields -workaround I've found is to put output in file -then in VIM on file, search/replace the true/false binary fields with t/f -specifically - - (reversed on 3/23 - should be ^A gets f) - :%s/^@/t/g - :%s/^A/f/g - keystrokes are ctrl-v ctrl-shift-@ to get the 'true' binary field - keystrokes are ctrl-v ctrl-shift-A to get the 'false' binary field == Contributors Project founded by Max Lapshin @@ -143,3 +132,4 @@ Other contributors (in git log order): - Neszt Tibor - Miroslav Kratochvil - Paul Gallagher +- James Coleman From ad4a92dd13b0ddf18a4977cc3f697ec41bcec282 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Fri, 9 Sep 2011 08:37:57 -0400 Subject: [PATCH 36/69] Don't copy views as if they were tables. For now, just ignore views. The user should have to recreate views on the new PostgrSQL database. --- lib/mysql2psql/mysql_reader.rb | 19 ++++++++++++++++--- test/fixtures/seed_integration_tests.sql | 4 ++++ test/integration/convert_to_file_test.rb | 5 +++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/mysql2psql/mysql_reader.rb b/lib/mysql2psql/mysql_reader.rb index 46c83f3..958d27a 100644 --- a/lib/mysql2psql/mysql_reader.rb +++ b/lib/mysql2psql/mysql_reader.rb @@ -155,7 +155,7 @@ def query_for_pager end def connect - @mysql = ::Mysql.connect(@host, @user, @passwd, @db, @port, @sock, @flag) + @mysql = Mysql.connect(@host, @user, @passwd, @db, @port, @sock, @flag) @mysql.query("SET NAMES utf8") @mysql.query("SET SESSION query_cache_type = OFF") end @@ -174,9 +174,22 @@ def initialize(options) end attr_reader :mysql - + + def views + unless defined? @views + @mysql.query("SELECT t.TABLE_NAME FROM INFORMATION_SCHEMA.TABLES t WHERE t.TABLE_SCHEMA = '#{@db}' AND t.TABLE_TYPE = 'VIEW';") do |res| + @views = [] + res.each { |row| @views << row[0] } + end + end + + @views + end + def tables - @tables ||= @mysql.list_tables.map {|table| Table.new(self, table)} + @tables ||= (@mysql.list_tables - views).map do |table| + Table.new(self, table) + end end def paginated_read(table, page_size) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index a636e0b..36f0c80 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -81,3 +81,7 @@ VALUES ('test-null', NULL, NULL), ('test-false', 0, 0), ('test-true', 1, 1); INSERT INTO test_boolean_conversion (test_name, tinyint_1) VALUES ('test-true-nonzero', 2); + +CREATE OR REPLACE VIEW test_view AS +SELECT b.test_name +FROM test_boolean_conversion b; diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index c593ffb..a9f91d3 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -109,4 +109,9 @@ def test_boolean_default_values def test_autoincrement assert_not_nil Regexp.new('CREATE TABLE "basic_autoincrement".*"auto_id" integer DEFAULT.*\)', Regexp::MULTILINE).match( content ) end + + # test view creation (or lack thereof!) + def test_should_not_copy_views_as_tables + assert_no_match /CREATE TABLE "test_view"/, content + end end \ No newline at end of file From 7cac2762c3f022c84e555328f375fe065f1a275e Mon Sep 17 00:00:00 2001 From: James Coleman Date: Fri, 9 Sep 2011 08:38:41 -0400 Subject: [PATCH 37/69] Cleanup the row processing logic. --- lib/mysql2psql/postgres_writer.rb | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 7020070..14d0a85 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -108,18 +108,14 @@ def process_row(table, row) table.columns.each_with_index do |column, index| if column[:type] == 'time' row[index] = "%02d:%02d:%02d" % [row[index].hour, row[index].minute, row[index].second] - end - - if row[index].is_a?(Mysql::Time) + elsif row[index].is_a?(Mysql::Time) row[index] = row[index].to_s.gsub('0000-00-00 00:00', '1970-01-01 00:00') row[index] = row[index].to_s.gsub('0000-00-00 00:00:00', '1970-01-01 00:00:00') - end - - if column[:type] == 'boolean' + elsif column[:type] == 'boolean' row[index] = ( case row[index] when nil - nil + '\N' # See note below about null values. when 0, "\0" 'f' else @@ -127,20 +123,18 @@ def process_row(table, row) 't' end ) - end - - if row[index].is_a?(String) + elsif row[index].is_a?(String) if column_type(column) == "bytea" row[index] = PGconn.escape_bytea(row[index]) else - row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r').gsub(/\0/, '') + row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r') end + elsif row[index].nil? + # Note: '\N' not "\N" is correct here: + # The string containing the literal backslash followed by 'N' + # represents database NULL value in PostgreSQL's text mode. + row[index] = '\N' end - - # Note: '\N' not "\N" is correct here: - # The string containing the literal backslash followed by 'N' - # represents database NULL value in PostgreSQL's text mode. - row[index] = '\N' if row[index].nil? end end From ad32f856e3186ec9d5113ae2d1d4ad695f7a0271 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Fri, 9 Sep 2011 09:13:40 -0400 Subject: [PATCH 38/69] Cleanup date* types column default processing. --- lib/mysql2psql/postgres_writer.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 14d0a85..3b473a4 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -74,20 +74,24 @@ def column_default(column) # Case for 1, '1', "b'1'" (for BIT(1) the data type), or anything non-nil and non-zero (for the TINYINT(1) type) 'true' end - when 'datetime', 'date' - nil # TODO: I have a hard time believing that this is the correct logic. - when 'timestamp' + when 'timestamp', 'datetime', 'date' case column[:default] when 'CURRENT_TIMESTAMP' 'CURRENT_TIMESTAMP' + when '0000-00-00' + "'1970-01-01'" when '0000-00-00 00:00' "'1970-01-01 00:00'" when '0000-00-00 00:00:00' "'1970-01-01 00:00:00'" + else + "'#{column[:default]}'" end when 'time' - 'NOW()' # TODO: Check this logic... + "'#{column[:default]}'" else + # TODO: column[:default] will never be nil here. + # Perhaps we should also issue a warning if this case is encountered. "#{column[:default] == nil ? 'NULL' : "'"+PGconn.escape(column[:default])+"'"}" end end From 235f1578cd77ba0153813081837a348e9dfcdb29 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Fri, 9 Sep 2011 09:57:36 -0400 Subject: [PATCH 39/69] Fix fixnum to string casting bug. --- lib/mysql2psql/postgres_writer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 3b473a4..f347778 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -101,7 +101,7 @@ def column_type_info(column) type = column_type(column) if type not_null = !column[:null] || column[:auto_increment] ? ' NOT NULL' : '' - default = column[:default] || column[:auto_increment] ? ' DEFAULT ' + column_default(column) : '' + default = column[:default] || column[:auto_increment] ? " DEFAULT #{column_default(column)}" : '' "#{type}#{default}#{not_null}" else '' From d5acb4f55195b3fb72a240a5781bd473df80040c Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:03:17 -0700 Subject: [PATCH 40/69] add two missing tests for null conversion and datetime conversion --- test/fixtures/seed_integration_tests.sql | 15 +++++++++++++++ test/integration/convert_to_db_test.rb | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 36f0c80..e20c426 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -31,6 +31,7 @@ INSERT INTO numeric_types_basics VALUES ( 5, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 4294967295, 9223372036854775807, 18446744073709551615, 1, 1, 1, 1, 1, 1); + DROP TABLE IF EXISTS basic_autoincrement; CREATE TABLE basic_autoincrement ( auto_id INT(11) NOT NULL AUTO_INCREMENT, @@ -85,3 +86,17 @@ INSERT INTO test_boolean_conversion (test_name, tinyint_1) VALUES ('test-true-no CREATE OR REPLACE VIEW test_view AS SELECT b.test_name FROM test_boolean_conversion b; + +DROP TABLE IF EXISTS test_null_conversion; +CREATE TABLE test_null_conversion ( + column_a VARCHAR(10) +); + +INSERT INTO test_null_conversion (column_a) VALUES (NULL); + +DROP TABLE IF EXISTS test_datetime_conversion; +CREATE TABLE test_datetime_conversion ( + column_a DATETIME +); + +INSERT INTO test_datetime_conversion (column_a) VALUES ('0000-00-00 00:00'); \ No newline at end of file diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 491beed..ba90f50 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -50,5 +50,15 @@ def test_boolean_conversion_of_null assert_nil null_record['bit_1'] assert_nil null_record['tinyint_1'] end + + def test_null_conversion + result = exec_sql_on_psql('SELECT column_a FROM test_null_conversion').first + assert_nil result['column_a'] + end + + def test_datetime_conversion + result = exec_sql_on_psql('SELECT column_a FROM test_datetime_conversion').first + assert_equal '1970-01-01 00:00:00', result['column_a'] + end end \ No newline at end of file From 49eac20c2fd253613039e8dc3dcc8ee37709488a Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:15:05 -0700 Subject: [PATCH 41/69] add test coverage for unique indexes, and indexes that have to be renamed because they match the table name --- test/fixtures/seed_integration_tests.sql | 14 ++++++-------- test/integration/convert_to_db_test.rb | 6 +++++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index e20c426..9ae77b7 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -88,15 +88,13 @@ SELECT b.test_name FROM test_boolean_conversion b; DROP TABLE IF EXISTS test_null_conversion; -CREATE TABLE test_null_conversion ( - column_a VARCHAR(10) -); - +CREATE TABLE test_null_conversion (column_a VARCHAR(10)); INSERT INTO test_null_conversion (column_a) VALUES (NULL); DROP TABLE IF EXISTS test_datetime_conversion; -CREATE TABLE test_datetime_conversion ( - column_a DATETIME -); +CREATE TABLE test_datetime_conversion (column_a DATETIME); +INSERT INTO test_datetime_conversion (column_a) VALUES ('0000-00-00 00:00'); -INSERT INTO test_datetime_conversion (column_a) VALUES ('0000-00-00 00:00'); \ No newline at end of file +DROP TABLE IF EXISTS test_index_conversion; +CREATE TABLE test_index_conversion (column_a VARCHAR(10)); +CREATE UNIQUE INDEX test_index_conversion ON test_index_conversion (column_a); \ No newline at end of file diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index ba90f50..30b9f52 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -60,5 +60,9 @@ def test_datetime_conversion result = exec_sql_on_psql('SELECT column_a FROM test_datetime_conversion').first assert_equal '1970-01-01 00:00:00', result['column_a'] end - + + def test_index_conversion + result = exec_sql_on_psql('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = \'test_index_conversion\'::regclass').first + assert_equal "CREATE UNIQUE INDEX test_index_conversion_index ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] + end end \ No newline at end of file From a80fa1f8ac9f71d3908c2c8bffcbb1fc6bb81363 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:28:08 -0700 Subject: [PATCH 42/69] add test coverage for foreign keys --- test/fixtures/seed_integration_tests.sql | 10 +++++++++- test/integration/convert_to_db_test.rb | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 9ae77b7..9ed4378 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -97,4 +97,12 @@ INSERT INTO test_datetime_conversion (column_a) VALUES ('0000-00-00 00:00'); DROP TABLE IF EXISTS test_index_conversion; CREATE TABLE test_index_conversion (column_a VARCHAR(10)); -CREATE UNIQUE INDEX test_index_conversion ON test_index_conversion (column_a); \ No newline at end of file +CREATE UNIQUE INDEX test_index_conversion ON test_index_conversion (column_a); + +DROP TABLE IF EXISTS test_foreign_keys_child; +DROP TABLE IF EXISTS test_foreign_keys_parent; +CREATE TABLE test_foreign_keys_parent (id INT NOT NULL, PRIMARY KEY (id)) ENGINE=INNODB; +CREATE TABLE test_foreign_keys_child (id INT, test_foreign_keys_parent_id INT, + INDEX par_ind (test_foreign_keys_parent_id), + FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id) ON DELETE CASCADE +) ENGINE=INNODB; diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 30b9f52..72c593a 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -65,4 +65,10 @@ def test_index_conversion result = exec_sql_on_psql('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = \'test_index_conversion\'::regclass').first assert_equal "CREATE UNIQUE INDEX test_index_conversion_index ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] end + + def test_foreign_keys + result = exec_sql_on_psql("SELECT conname, pg_catalog.pg_get_constraintdef(r.oid, true) as condef FROM pg_catalog.pg_constraint r WHERE r.conrelid = 'test_foreign_keys_child'::regclass") + expected = {"condef" => "FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id)", "conname" => "test_foreign_keys_child_test_foreign_keys_parent_id_fkey"} + assert_equal expected, result.first + end end \ No newline at end of file From 1640a155654c3c41bfb9015f3a47ce924d012181 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:31:14 -0700 Subject: [PATCH 43/69] fix rcov rake task --- Rakefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index c774b1d..f439ee8 100644 --- a/Rakefile +++ b/Rakefile @@ -63,9 +63,11 @@ end begin require 'rcov/rcovtask' Rcov::RcovTask.new do |test| - test.libs << 'test' + test.libs << 'lib' + test.libs << 'test/lib' test.pattern = 'test/**/*test.rb' test.verbose = true + test.rcov_opts << "--exclude gems/*" end rescue LoadError task :rcov do From 926aa77b8d428ef550da4da804c85615ee12fe23 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:32:15 -0700 Subject: [PATCH 44/69] remove unused method which was accidentally defined twice --- lib/mysql2psql/postgres_writer.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index f347778..73117f3 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -9,10 +9,6 @@ def column_description(column) "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}" end - def column_type(column) - column_type_info(column).split(" ").first - end - def column_type(column) if column[:auto_increment] 'integer' From d6d2bcb663f03001b7abd3097f27f94306514567 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:44:04 -0700 Subject: [PATCH 45/69] add missing test for datetime defaults, increase test coverage to 88% --- test/fixtures/seed_integration_tests.sql | 10 +++++++-- test/integration/convert_to_db_test.rb | 27 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 9ed4378..416c66e 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -92,7 +92,13 @@ CREATE TABLE test_null_conversion (column_a VARCHAR(10)); INSERT INTO test_null_conversion (column_a) VALUES (NULL); DROP TABLE IF EXISTS test_datetime_conversion; -CREATE TABLE test_datetime_conversion (column_a DATETIME); +CREATE TABLE test_datetime_conversion ( + column_a DATETIME, + column_b TIMESTAMP, + column_c DATETIME DEFAULT '0000-00-00', + column_d DATETIME DEFAULT '0000-00-00 00:00', + column_e DATETIME DEFAULT '0000-00-00 00:00:00' +); INSERT INTO test_datetime_conversion (column_a) VALUES ('0000-00-00 00:00'); DROP TABLE IF EXISTS test_index_conversion; @@ -105,4 +111,4 @@ CREATE TABLE test_foreign_keys_parent (id INT NOT NULL, PRIMARY KEY (id)) ENGINE CREATE TABLE test_foreign_keys_child (id INT, test_foreign_keys_parent_id INT, INDEX par_ind (test_foreign_keys_parent_id), FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id) ON DELETE CASCADE -) ENGINE=INNODB; +) ENGINE=INNODB; \ No newline at end of file diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 72c593a..225f5d0 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -61,6 +61,33 @@ def test_datetime_conversion assert_equal '1970-01-01 00:00:00', result['column_a'] end + def test_datetime_defaults + result = exec_sql_on_psql(<<-SQL) + SELECT a.attname, + pg_catalog.format_type(a.atttypid, a.atttypmod), + (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid) for 128) + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) AS default + FROM pg_catalog.pg_attribute a + WHERE a.attrelid = 'test_datetime_conversion'::regclass AND a.attnum > 0 + SQL + + assert_equal 5, result.count + + result.each do |row| + assert_equal "timestamp without time zone", row["format_type"] + + case row["attname"] + when "column_a" + assert_nil row["default"] + when "column_b" + assert_equal "now()", row["default"] + when "column_c", "column_d", "column_e" + assert_equal "'1970-01-01 00:00:00'::timestamp without time zone", row["default"] + end + end + end + def test_index_conversion result = exec_sql_on_psql('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = \'test_index_conversion\'::regclass').first assert_equal "CREATE UNIQUE INDEX test_index_conversion_index ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] From 690b23bf7a5ffd9f8fffc038e107430287f15049 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:48:04 -0700 Subject: [PATCH 46/69] add test for time conversion --- test/fixtures/seed_integration_tests.sql | 5 +++-- test/integration/convert_to_db_test.rb | 11 ++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 416c66e..cfbce68 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -97,9 +97,10 @@ CREATE TABLE test_datetime_conversion ( column_b TIMESTAMP, column_c DATETIME DEFAULT '0000-00-00', column_d DATETIME DEFAULT '0000-00-00 00:00', - column_e DATETIME DEFAULT '0000-00-00 00:00:00' + column_e DATETIME DEFAULT '0000-00-00 00:00:00', + column_f TIME ); -INSERT INTO test_datetime_conversion (column_a) VALUES ('0000-00-00 00:00'); +INSERT INTO test_datetime_conversion (column_a, column_f) VALUES ('0000-00-00 00:00', '08:15:30'); DROP TABLE IF EXISTS test_index_conversion; CREATE TABLE test_index_conversion (column_a VARCHAR(10)); diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 225f5d0..eaf4875 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -57,8 +57,9 @@ def test_null_conversion end def test_datetime_conversion - result = exec_sql_on_psql('SELECT column_a FROM test_datetime_conversion').first + result = exec_sql_on_psql('SELECT column_a, column_f FROM test_datetime_conversion').first assert_equal '1970-01-01 00:00:00', result['column_a'] + assert_equal '08:15:30', result['column_f'] end def test_datetime_defaults @@ -72,10 +73,14 @@ def test_datetime_defaults WHERE a.attrelid = 'test_datetime_conversion'::regclass AND a.attnum > 0 SQL - assert_equal 5, result.count + assert_equal 6, result.count result.each do |row| - assert_equal "timestamp without time zone", row["format_type"] + if row["attname"] == "column_f" + assert_equal "time without time zone", row["format_type"] + else + assert_equal "timestamp without time zone", row["format_type"] + end case row["attname"] when "column_a" From 960041fd1c46737ddf5ead360e2246cc56077d66 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 10:55:42 -0700 Subject: [PATCH 47/69] add test for what the tool prints to stdout --- test/integration/convert_to_db_test.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index eaf4875..3142f8e 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -5,15 +5,22 @@ class ConvertToDbTest < Test::Unit::TestCase def setup + $stdout = StringIO.new + $stderr = StringIO.new + seed_test_database @options=get_test_config_by_label(:localmysql_to_db_convert_all) @mysql2psql = Mysql2psql.new([@options.filepath]) @mysql2psql.convert @mysql2psql.writer.open end + def teardown @mysql2psql.writer.close delete_files_for_test_config(@options) + + $stdout = STDOUT + $stderr = STDERR end def exec_sql_on_psql(sql, parameters=nil) @@ -103,4 +110,11 @@ def test_foreign_keys expected = {"condef" => "FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id)", "conname" => "test_foreign_keys_child_test_foreign_keys_parent_id_fkey"} assert_equal expected, result.first end + + def test_output + $stdout.rewind + actual = $stdout.read + + assert_match /Counting rows of test_foreign_keys_child/, actual + end end \ No newline at end of file From 2637289aa41e78f807068d57c3272dea402f7f32 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 11:04:23 -0700 Subject: [PATCH 48/69] add test for force_truncate option, and fix bug where when option was true, TRUNCATE statements were not getting written out. increase test coverage to 91% --- lib/mysql2psql/converter.rb | 2 +- test/integration/convert_to_file_test.rb | 4 ++++ test/lib/test_helper.rb | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/mysql2psql/converter.rb b/lib/mysql2psql/converter.rb index e708f4a..0bb765a 100644 --- a/lib/mysql2psql/converter.rb +++ b/lib/mysql2psql/converter.rb @@ -29,7 +29,7 @@ def convert _time2 = Time.now tables.each do |table| - writer.truncate(table) if force_truncate && supress_ddl + writer.truncate(table) if force_truncate && !supress_ddl writer.write_contents(table, reader) end unless @supress_data diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index a9f91d3..2e77006 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -114,4 +114,8 @@ def test_autoincrement def test_should_not_copy_views_as_tables assert_no_match /CREATE TABLE "test_view"/, content end + + def test_truncate + assert_match /TRUNCATE/, content + end end \ No newline at end of file diff --git a/test/lib/test_helper.rb b/test/lib/test_helper.rb index 4b77519..79d7673 100644 --- a/test/lib/test_helper.rb +++ b/test/lib/test_helper.rb @@ -72,7 +72,7 @@ def get_test_config_by_label(name) when :localmysql_to_file_convert_nothing get_new_test_config(true, ['unobtainium'], ['kryptonite'], true, true, false) when :localmysql_to_file_convert_all - get_new_test_config(true, [], [], false, false, false) + get_new_test_config(true, [], [], false, false, true) when :localmysql_to_db_convert_all get_new_test_config(false, [], [], false, false, false) when :localmysql_to_db_convert_nothing From af016ae9aeb70640030a30fb81af7e19a6c1ef86 Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Wed, 14 Sep 2011 11:08:57 -0700 Subject: [PATCH 49/69] add missing test for enum conversion --- test/fixtures/seed_integration_tests.sql | 6 +++++- test/integration/convert_to_db_test.rb | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index cfbce68..63123f4 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -112,4 +112,8 @@ CREATE TABLE test_foreign_keys_parent (id INT NOT NULL, PRIMARY KEY (id)) ENGINE CREATE TABLE test_foreign_keys_child (id INT, test_foreign_keys_parent_id INT, INDEX par_ind (test_foreign_keys_parent_id), FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id) ON DELETE CASCADE -) ENGINE=INNODB; \ No newline at end of file +) ENGINE=INNODB; + +DROP TABLE IF EXISTS test_enum; +CREATE TABLE test_enum (name ENUM('small', 'medium', 'large')); +INSERT INTO test_enum (name) VALUES ('medium'); \ No newline at end of file diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 3142f8e..5faaa5f 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -117,4 +117,16 @@ def test_output assert_match /Counting rows of test_foreign_keys_child/, actual end + + def test_enum + result = exec_sql_on_psql(<<-SQL) + SELECT r.conname, pg_catalog.pg_get_constraintdef(r.oid, true) + FROM pg_catalog.pg_constraint r + WHERE r.conrelid = 'test_enum'::regclass AND r.contype = 'c' + ORDER BY 1 + SQL + + assert_equal 1, result.count + assert_equal "CHECK (name::text = ANY (ARRAY['small'::character varying, 'medium'::character varying, 'large'::character varying]::text[]))", result.first["pg_get_constraintdef"] + end end \ No newline at end of file From 54ffe7405aa545179eecc2d601040633f081abea Mon Sep 17 00:00:00 2001 From: dakhota Date: Fri, 16 Sep 2011 20:23:27 +0200 Subject: [PATCH 50/69] quoting default values in table definition --- lib/mysql2psql/postgres_writer.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 73117f3..a2bc869 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -51,9 +51,9 @@ def column_default(column) elsif column[:default] case column[:type] when 'char' - "#{column[:default]}::char" + "'#{PGconn.escape(column[:default])}'::char" when 'varchar', /^enum/ - "#{column[:default]}::character varying" + "'#{PGconn.escape(column[:default])}'::character varying" when 'integer', 'bigint', /tinyint|smallint/ column[:default].to_i when 'real', /float/ @@ -81,10 +81,10 @@ def column_default(column) when '0000-00-00 00:00:00' "'1970-01-01 00:00:00'" else - "'#{column[:default]}'" + "'#{PGconn.escape(column[:default])}'" end when 'time' - "'#{column[:default]}'" + "'#{PGconn.escape(column[:default])}'" else # TODO: column[:default] will never be nil here. # Perhaps we should also issue a warning if this case is encountered. From 4e3feac51507e61f48e5c213d44a9dbb20c6eef5 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Mon, 19 Sep 2011 11:03:40 -0400 Subject: [PATCH 51/69] Refactor string escaping. Use Ruby's built in String escaping to handle more cases. Explicitly handing the special \N and \. COPY command sequences. Strip null characters from the string (and issue a warning) since PostgreSQL's TEXT types don't allow those in any encoding. --- lib/mysql2psql/postgres_writer.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index f347778..0aa54d1 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -131,7 +131,17 @@ def process_row(table, row) if column_type(column) == "bytea" row[index] = PGconn.escape_bytea(row[index]) else - row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r') + if row[index] == '\N' || row[index] == '\.' + row[index] = '\\' + row[index] # Escape our two PostgreSQL-text-mode-special strings. + else + # Awesome side-effect producing conditional. Don't do this at home. + unless row[index].gsub!(/\0/, '').nil? + puts "Removed null bytes from string since PostgreSQL TEXT types don't allow the storage of null bytes." + end + + row[index] = row[index].dump + row[index] = row[index].slice(1, row[index].size-2) + end end elsif row[index].nil? # Note: '\N' not "\N" is correct here: From f284ad2a78f830cc8f9d97b772f5db8a8c5f12b4 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Mon, 19 Sep 2011 15:38:56 -0400 Subject: [PATCH 52/69] Option to create columns with or without timezones. --- lib/mysql2psql/config.rb | 11 ++++++++++- lib/mysql2psql/converter.rb | 3 ++- lib/mysql2psql/postgres_db_writer.rb | 4 ++-- lib/mysql2psql/postgres_file_writer.rb | 4 ++-- lib/mysql2psql/postgres_writer.rb | 14 +++++++------- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index 6733f73..d304f82 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -27,7 +27,7 @@ def reset_configfile(filepath) file.close end - def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, force_truncate = false) + def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, force_truncate = false, use_timezones = false) configtext = < @use_timezones}) end unless @supress_ddl _time2 = Time.now diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index 8265e1c..82fb264 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -33,7 +33,7 @@ def exists?(relname) (!rc.nil?) && (rc.to_a.length==1) && (rc.first.count.to_i==1) end - def write_table(table) + def write_table(table, options) puts "Creating table #{table.name}..." primary_keys = [] serial_key = nil @@ -47,7 +47,7 @@ def write_table(table) if column[:primary_key] primary_keys << column[:name] end - " " + column_description(column) + " " + column_description(column, options) end.join(",\n") if serial_key diff --git a/lib/mysql2psql/postgres_file_writer.rb b/lib/mysql2psql/postgres_file_writer.rb index 5350c7d..cf2d94c 100644 --- a/lib/mysql2psql/postgres_file_writer.rb +++ b/lib/mysql2psql/postgres_file_writer.rb @@ -38,7 +38,7 @@ def truncate(table) end end - def write_table(table) + def write_table(table, options) primary_keys = [] serial_key = nil maxval = nil @@ -51,7 +51,7 @@ def write_table(table) if column[:primary_key] primary_keys << column[:name] end - " " + column_description(column) + " " + column_description(column, options) end.join(",\n") if serial_key diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 24215a3..125ea11 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -5,11 +5,11 @@ class Mysql2psql class PostgresWriter < Writer - def column_description(column) - "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}" + def column_description(column, options) + "#{PGconn.quote_ident(column[:name])} #{column_type_info(column, options)}" end - def column_type(column) + def column_type(column, options={}) if column[:auto_increment] 'integer' else @@ -25,9 +25,9 @@ def column_type(column) when 'decimal' "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" when 'datetime', 'timestamp' - 'timestamp without time zone' + "timestamp with#{options[:use_timezones] ? '' : 'out'} time zone" when 'time' - 'time without time zone' + "time with#{options[:use_timezones] ? '' : 'out'} time zone" when 'tinyblob', 'mediumblob', 'longblob', 'blob', 'varbinary' 'bytea' when 'tinytext', 'mediumtext', 'longtext', 'text' @@ -93,8 +93,8 @@ def column_default(column) end end - def column_type_info(column) - type = column_type(column) + def column_type_info(column, options) + type = column_type(column, options) if type not_null = !column[:null] || column[:auto_increment] ? ' NOT NULL' : '' default = column[:default] || column[:auto_increment] ? " DEFAULT #{column_default(column)}" : '' From 561dec978b41254983a48ff3dd605045adf4dc6e Mon Sep 17 00:00:00 2001 From: James Tippett Date: Tue, 11 Oct 2011 11:53:21 +1100 Subject: [PATCH 53/69] correct projectwide spelling of 'suppress' --- lib/mysql2psql/config.rb | 26 +++++++++++++------------- lib/mysql2psql/converter.rb | 28 ++++++++++++++-------------- test/fixtures/config_all_options.yml | 10 +++++----- test/lib/test_helper.rb | 6 +++--- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index 6733f73..ade8738 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -1,13 +1,13 @@ require 'mysql2psql/config_base' class Mysql2psql - + class Config < ConfigBase - + def initialize(configfilepath, generate_default_if_not_found = true) unless File.exists?(configfilepath) reset_configfile(configfilepath) if generate_default_if_not_found - if File.exists?(configfilepath) + if File.exists?(configfilepath) raise Mysql2psql::ConfigurationFileInitialized.new("\n No configuration file found. A new file has been initialized at: #{configfilepath} @@ -27,14 +27,14 @@ def reset_configfile(filepath) file.close end - def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, force_truncate = false) + def self.template(to_filename = nil, include_tables = [], exclude_tables = [], suppress_data = false, suppress_ddl = false, force_truncate = false) configtext = < Date: Tue, 11 Oct 2011 12:13:39 +1100 Subject: [PATCH 54/69] add support for suppressing indexes --- lib/mysql2psql/config.rb | 8 +++++++- lib/mysql2psql/converter.rb | 5 +++-- test/fixtures/config_all_options.yml | 3 +++ test/lib/test_helper.rb | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index ade8738..8f17240 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -27,7 +27,7 @@ def reset_configfile(filepath) file.close end - def self.template(to_filename = nil, include_tables = [], exclude_tables = [], suppress_data = false, suppress_ddl = false, force_truncate = false) + def self.template(to_filename = nil, include_tables = [], exclude_tables = [], suppress_data = false, suppress_ddl = false, force_truncate = false, suppress_indexes = false) configtext = < Date: Tue, 11 Oct 2011 12:25:14 +1100 Subject: [PATCH 55/69] typo --- lib/mysql2psql/config.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index 8f17240..53b57e8 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -91,6 +91,7 @@ def self.template(to_filename = nil, include_tables = [], exclude_tables = [], s # if force_truncate is true, forces a table truncate before table loading force_truncate: #{force_truncate} EOS + end if !suppress_indexes.nil? configtext += < Date: Tue, 11 Oct 2011 15:12:21 +1100 Subject: [PATCH 56/69] properly escape varchar default value so pgsql does not confuse it with a column name --- lib/mysql2psql/postgres_writer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index f347778..5019b41 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -57,7 +57,7 @@ def column_default(column) when 'char' "#{column[:default]}::char" when 'varchar', /^enum/ - "#{column[:default]}::character varying" + "'#{column[:default]}'::character varying" when 'integer', 'bigint', /tinyint|smallint/ column[:default].to_i when 'real', /float/ From ef9951ed29b8916d28f3d212ffd68eef44818cce Mon Sep 17 00:00:00 2001 From: Aaron Peckham Date: Thu, 13 Oct 2011 16:33:28 -0700 Subject: [PATCH 57/69] Allow Postgres to rename indexes; don't try to preserve MySQL index names because they might not be unique or might collide with table names --- lib/mysql2psql/postgres_db_writer.rb | 15 +-------------- test/integration/convert_to_db_test.rb | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index 8265e1c..c8995a9 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -94,20 +94,7 @@ def write_indexes(table) next if index[:primary] unique = index[:unique] ? "UNIQUE " : nil - #MySQL allows an index name which could be equal to a table name, Postgres doesn't - indexname = index[:name] - if indexname.eql?(table.name) - indexnamenew = "#{indexname}_index" - puts "WARNING: index \"#{indexname}\" equals table name. This is not allowed by postgres and will be renamed to \"#{indexnamenew}\"" - indexname = indexnamenew - end - - if @conn.server_version < 80200 - @conn.exec("DROP INDEX #{PGconn.quote_ident(indexname)} CASCADE;") if exists?(indexname) - else - @conn.exec("DROP INDEX IF EXISTS #{PGconn.quote_ident(indexname)} CASCADE;") - end - @conn.exec("CREATE #{unique}INDEX #{PGconn.quote_ident(indexname)} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});") + @conn.exec("CREATE #{unique}INDEX ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});") end diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 5faaa5f..bef6863 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -102,7 +102,7 @@ def test_datetime_defaults def test_index_conversion result = exec_sql_on_psql('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = \'test_index_conversion\'::regclass').first - assert_equal "CREATE UNIQUE INDEX test_index_conversion_index ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] + assert_equal "CREATE UNIQUE INDEX test_index_conversion_column_a_idx ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] end def test_foreign_keys From d3bee02f213990c4643cbcb0662b44b06b9259d3 Mon Sep 17 00:00:00 2001 From: James Coleman Date: Wed, 2 Nov 2011 15:19:47 -0400 Subject: [PATCH 58/69] Add capability to update sequence values without writing DDL. --- lib/mysql2psql/config.rb | 11 ++++- lib/mysql2psql/converter.rb | 6 ++- lib/mysql2psql/postgres_db_writer.rb | 58 ++++++++++++++----------- lib/mysql2psql/postgres_file_writer.rb | 60 +++++++++++++++----------- lib/mysql2psql/postgres_writer.rb | 8 ++-- 5 files changed, 88 insertions(+), 55 deletions(-) diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index d304f82..b77767a 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -27,7 +27,7 @@ def reset_configfile(filepath) file.close end - def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, force_truncate = false, use_timezones = false) + def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, supress_sequence_update = false, force_truncate = false, use_timezones = false) configtext = < @use_timezones}) diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index 82fb264..9e33d32 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -27,47 +27,55 @@ def open def close @conn.close end - + def exists?(relname) rc = @conn.exec("SELECT COUNT(*) FROM pg_class WHERE relname = '#{relname}'") (!rc.nil?) && (rc.to_a.length==1) && (rc.first.count.to_i==1) end + def write_sequence_update(table, options) + serial_key_column = table.columns.detect do |column| + column[:auto_increment] + end + + if serial_key_column + serial_key = serial_key_column[:name] + max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1 + serial_key_seq = "#{table.name}_#{serial_key}_seq" + + if !options.supress_ddl + if @conn.server_version < 80200 + @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq) + else + @conn.exec("DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE") + end + @conn.exec <<-EOF + CREATE SEQUENCE #{serial_key_seq} + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1 + EOF + end + + if !options.supress_sequence_update + puts "Updated sequence #{serial_key_seq} to current value of #{max_value}" + @conn.exec sqlfor_set_serial_sequence(table, serial_key_seq, max_value) + end + end + end + def write_table(table, options) puts "Creating table #{table.name}..." primary_keys = [] - serial_key = nil - maxval = nil columns = table.columns.map do |column| - if column[:auto_increment] - serial_key = column[:name] - maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 - end if column[:primary_key] primary_keys << column[:name] end " " + column_description(column, options) end.join(",\n") - if serial_key - if @conn.server_version < 80200 - serial_key_seq = "#{table.name}_#{serial_key}_seq" - @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq) - else - @conn.exec("DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE") - end - @conn.exec <<-EOF - CREATE SEQUENCE #{table.name}_#{serial_key}_seq - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1 - EOF - - @conn.exec sqlfor_set_serial_sequence(table,serial_key,maxval) - end - if @conn.server_version < 80200 @conn.exec "DROP TABLE #{PGconn.quote_ident(table.name)} CASCADE;" if exists?(table.name) else diff --git a/lib/mysql2psql/postgres_file_writer.rb b/lib/mysql2psql/postgres_file_writer.rb index cf2d94c..78e6ed3 100644 --- a/lib/mysql2psql/postgres_file_writer.rb +++ b/lib/mysql2psql/postgres_file_writer.rb @@ -38,41 +38,53 @@ def truncate(table) end end - def write_table(table, options) - primary_keys = [] - serial_key = nil - maxval = nil - - columns = table.columns.map do |column| - if column[:auto_increment] - serial_key = column[:name] - maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 - end - if column[:primary_key] - primary_keys << column[:name] - end - " " + column_description(column, options) - end.join(",\n") + def write_sequence_update(table, options) + serial_key_column = table.columns.detect do |column| + column[:auto_increment] + end - if serial_key + if serial_key_column + serial_key = serial_key_column[:name] + serial_key_seq = "#{table.name}_#{serial_key}_seq" + max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1 @f << <<-EOF -- --- Name: #{table.name}_#{serial_key}_seq; Type: SEQUENCE; Schema: public +-- Name: #{serial_key_seq}; Type: SEQUENCE; Schema: public -- +EOF + + if !options.supress_ddl + @f << <<-EOF +DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE; -DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE; - -CREATE SEQUENCE #{table.name}_#{serial_key}_seq +CREATE SEQUENCE #{serial_key_seq} INCREMENT BY 1 NO MAXVALUE NO MINVALUE CACHE 1; - -#{sqlfor_set_serial_sequence(table,serial_key,maxval)} - - EOF +EOF + end + + if !options.supress_sequence_update + @f << <<-EOF +#{sqlfor_set_serial_sequence(table, serial_key_seq, max_value)} +EOF + end end + end + + def write_table(table, options) + primary_keys = [] + serial_key = nil + maxval = nil + + columns = table.columns.map do |column| + if column[:primary_key] + primary_keys << column[:name] + end + " " + column_description(column, options) + end.join(",\n") @f << <<-EOF -- Table: #{table.name} diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 125ea11..fae5e38 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -151,11 +151,11 @@ def process_row(table, row) def truncate(table) end - def sqlfor_set_serial_sequence(table,serial_key,maxval) - "SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true);" + def sqlfor_set_serial_sequence(table, serial_key_seq, max_value) + "SELECT pg_catalog.setval('#{serial_key_seq}', #{max_value}, true);" end - def sqlfor_reset_serial_sequence(table,serial_key,maxval) - "SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true);" + def sqlfor_reset_serial_sequence(table, serial_key, max_value) + "SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{max_value}, true);" end end From f3d5cfa68a0c55b2bfc8cd4ddf728252a507ec24 Mon Sep 17 00:00:00 2001 From: Matthew Soldo Date: Thu, 24 Nov 2011 13:56:12 -0800 Subject: [PATCH 59/69] bumped pg gem to 0.11.0 --- mysql2psql.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index fb52551..8950218 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -75,16 +75,16 @@ Gem::Specification.new do |s| if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then s.add_runtime_dependency(%q, ["= 2.8.1"]) - s.add_runtime_dependency(%q, ["= 0.9.0"]) + s.add_runtime_dependency(%q, ["= 0.11.0"]) s.add_development_dependency(%q, [">= 2.1.1"]) else s.add_dependency(%q, ["= 2.8.1"]) - s.add_dependency(%q, ["= 0.9.0"]) + s.add_dependency(%q, ["= 0.11.0"]) s.add_dependency(%q, [">= 2.1.1"]) end else s.add_dependency(%q, ["= 2.8.1"]) - s.add_dependency(%q, ["= 0.9.0"]) + s.add_dependency(%q, ["= 0.11.0"]) s.add_dependency(%q, [">= 2.1.1"]) end end From 84773336cdfce69105c0b9c91f135b14375d61ed Mon Sep 17 00:00:00 2001 From: Mike Naberezny Date: Mon, 9 Jan 2012 16:04:26 -0800 Subject: [PATCH 60/69] Fix homepage link --- README.rdoc | 10 +++++----- Rakefile | 2 +- mysql2psql.gemspec | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rdoc b/README.rdoc index c7ea95c..9e5e770 100644 --- a/README.rdoc +++ b/README.rdoc @@ -48,11 +48,11 @@ If you encounter any issues with db connectivity, verify that the 'mysql' and 'p The 'mysql' gem in particular may need a hint on where to find the mysql headers and libraries: $ [sudo] gem install mysql -- --with-mysql-dir=/usr/local/mysql - + NB: With Ruby 1.8.x, the gem install will usually complain about "No definition for ..." errors. These are non-fatal and just affect the documentation install. This doesn't happen with Ruby 1.9.2. == Contributing to Mysql2psql - + * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it * Fork the project @@ -64,16 +64,16 @@ NB: With Ruby 1.8.x, the gem install will usually complain about "No definition == Getting the source Mysql2psql was first produced by Max Lapshin who maintains the master -repository at http://github.com/maxlapshin/mysql2postgres +repository at https://github.com/maxlapshin/mysql2postgres The gem bundling and refactoring for testing was done by Paul Gallagher and -the repository is at http://github.com/tardate/mysql2postgres +the repository is at https://github.com/tardate/mysql2postgres Gem packaging is done with Jeweler. Note that on windows this means you must run under mingw or other shell that supports git to do the gem bundling (but you can gem install the built gem in the normal Windows shell). == Running tests If you fork/clone the project, you will want to run the test suite (and add to it if you are developing new features or fixing bugs). - + A suite of tests are provided and are run using rake (rake -T for more info) rake test:units rake test:integration diff --git a/Rakefile b/Rakefile index c774b1d..8e57979 100644 --- a/Rakefile +++ b/Rakefile @@ -13,7 +13,7 @@ begin gem.description = %Q{It can create postgresql dump from mysql database or directly load data from mysql to postgresql (at about 100 000 records per minute). Translates most data types and indexes.} gem.email = "gallagher.paul@gmail.com" - gem.homepage = "http://github.com/tardate/mysql2postgresql" + gem.homepage = "https://github.com/tardate/mysql2postgres" gem.authors = [ "Max Lapshin ", "Anton Ageev ", diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index 8ad2931..0e41783 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -49,7 +49,7 @@ Gem::Specification.new do |s| "test/units/config_test.rb", "test/units/postgres_file_writer_test.rb" ] - s.homepage = %q{http://github.com/tardate/mysql2postgresql} + s.homepage = %q{https://github.com/tardate/mysql2postgres} s.require_paths = ["lib"] s.rubygems_version = %q{1.7.2} s.summary = %q{Tool for converting mysql database to postgresql} From c3272b85b2128e86ecadc76dcbf4153b7e618a31 Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Wed, 11 Jan 2012 13:32:23 -0600 Subject: [PATCH 61/69] Don't fail on nil date/time. --- lib/mysql2psql/converter.rb | 1 + lib/mysql2psql/postgres_writer.rb | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/mysql2psql/converter.rb b/lib/mysql2psql/converter.rb index e708f4a..19e8092 100644 --- a/lib/mysql2psql/converter.rb +++ b/lib/mysql2psql/converter.rb @@ -48,6 +48,7 @@ def convert return 0 rescue => e $stderr.puts "Mysql2psql: conversion failed: #{e.to_s}" + $stderr.puts e.backtrace return -1 end end diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 9609ed4..27bb0f6 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -111,7 +111,11 @@ def process_row(table, row) table.columns.each_with_index do |column, index| if column[:type] == "time" - row[index] = "%02d:%02d:%02d" % [row[index].hour, row[index].minute, row[index].second] + begin + row[index] = "%02d:%02d:%02d" % [row[index].hour, row[index].minute, row[index].second] + rescue + # nil + end end if row[index].is_a?(Mysql::Time) @@ -140,4 +144,4 @@ def truncate(table) end -end \ No newline at end of file +end From bbd31d74599fcac7b922faf753e95e1118922ffd Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Wed, 11 Jan 2012 13:32:42 -0600 Subject: [PATCH 62/69] Default to 5 digit precision. --- lib/mysql2psql/postgres_writer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 27bb0f6..36dd7b3 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -54,7 +54,7 @@ def column_type_info(column) "real" when "decimal" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default - "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" + "numeric(#{column[:length] || 10}, #{column[:decimals] || 5})" when "double precision" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default From db137fa45f90a3d0058133b48ed1337e4b5891e4 Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Wed, 11 Jan 2012 14:09:53 -0600 Subject: [PATCH 63/69] Use real for decimal --- lib/mysql2psql/postgres_writer.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index 36dd7b3..2d0a7b4 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -54,7 +54,8 @@ def column_type_info(column) "real" when "decimal" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default - "numeric(#{column[:length] || 10}, #{column[:decimals] || 5})" + #"numeric(#{column[:length] || 10}, #{column[:decimals] || 5})" + "real" when "double precision" default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default From e2d61b371686a3c4b5a0ddc8671ae8639b6e42c8 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Sun, 12 Feb 2012 10:49:58 +0800 Subject: [PATCH 64/69] fix test_truncate(ConvertToFileTest) failure * refactor config file template options to be more sane * re-align test options and expectations for test_truncate * update config_all_options fixture to match currently available options --- .gitignore | 2 + README.rdoc | 2 +- lib/mysql2psql/config.rb | 46 +++++++++++++------- test/fixtures/config_all_options.yml | 11 ++++- test/integration/convert_to_file_test.rb | 54 ++++++++++++------------ test/lib/test_helper.rb | 49 +++++++++++++++++---- test/units/config_test.rb | 37 ++++++++++++++++ 7 files changed, 149 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index 684c0dd..5262893 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ test/fixtures/test*.sql pkg .redcar .rvmrc +.bundle +*.out diff --git a/README.rdoc b/README.rdoc index 9e5e770..127e2d0 100644 --- a/README.rdoc +++ b/README.rdoc @@ -94,7 +94,7 @@ e.g. mysqladmin -uroot -p create mysql2psql_test mysql -uroot -p -e "grant all on mysql2psql_test.* to 'mysql2psql'@'localhost';" - # verify connecction: + # verify connection: mysql -umysql2psql -e "select database(),'yes' as connected" mysql2psql_test diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index b77767a..e7b7d2e 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -7,7 +7,7 @@ class Config < ConfigBase def initialize(configfilepath, generate_default_if_not_found = true) unless File.exists?(configfilepath) reset_configfile(configfilepath) if generate_default_if_not_found - if File.exists?(configfilepath) + if File.exists?(configfilepath) raise Mysql2psql::ConfigurationFileInitialized.new("\n No configuration file found. A new file has been initialized at: #{configfilepath} @@ -27,24 +27,33 @@ def reset_configfile(filepath) file.close end - def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, supress_sequence_update = false, force_truncate = false, use_timezones = false) + # Returns template file text given +options+ hash which may include: + # :to_filename - default: nil + # :include_tables - default: [] + # :exclude_tables - default: [] + # :suppress_data - default: false + # :suppress_ddl - default: false + # :suppress_sequence_update - default: false + # :force_truncate - default: false + # :use_timezones - default: false + def self.template(options={}) configtext = <0 configtext += "\ntables:\n" include_tables.each do |t| @@ -65,48 +75,54 @@ def self.template(to_filename = nil, include_tables = [], exclude_tables = [], s #- table4 EOS + exclude_tables = options[:exclude_tables] || [] if exclude_tables.length>0 configtext += "\nexclude_tables:\n" exclude_tables.each do |t| configtext += "- #{t}\n" end end - if !supress_data.nil? + + if !options[:suppress_data].nil? configtext += < true, + :include_tables => ['unobtainium'], + :exclude_tables => ['kryptonite'], + :suppress_data => true, + :suppress_ddl => true, + :force_truncate => false + } when :localmysql_to_file_convert_all - get_new_test_config(true, [], [], false, false, true) + { + :to_file => true, + :include_tables => [], + :exclude_tables => [], + :suppress_data => false, + :suppress_ddl => false, + :force_truncate => true + } when :localmysql_to_db_convert_all - get_new_test_config(false, [], [], false, false, false) + { + :to_file => false, + :include_tables => [], + :exclude_tables => [], + :suppress_data => false, + :suppress_ddl => false, + :force_truncate => false + } when :localmysql_to_db_convert_nothing - get_new_test_config(false, ['unobtainium'], ['kryptonite'], true, true, false) + { + :to_file => false, + :include_tables => ['unobtainium'], + :exclude_tables => ['kryptonite'], + :suppress_data => true, + :suppress_ddl => true, + :force_truncate => false + } else raise StandardError.new("Invalid label: #{name}") end + get_new_test_config(options) end def delete_files_for_test_config(config) diff --git a/test/units/config_test.rb b/test/units/config_test.rb index 5d531f6..eb298e5 100644 --- a/test/units/config_test.rb +++ b/test/units/config_test.rb @@ -23,9 +23,46 @@ def test_config_file_not_found value = Mysql2psql::Config.new(configfile_not_found, false) end end + def test_initialize_new_config_file assert_raise(Mysql2psql::ConfigurationFileInitialized) do value = Mysql2psql::Config.new(configfile_new, true) end end + + def test_config_option_pgdatabase_as_array_index + expected = 'somename' + config = Mysql2psql::Config.new(configfile_all_opts, false) + assert_equal expected,config[:pgdatabase] + end + def test_template_option_to_filename + expected = 'test_filename' + value = Mysql2psql::Config.template({ :to_filename => expected }) + assert_match /file: #{expected}/,value + end + def test_template_option_suppress_data + expected = true + value = Mysql2psql::Config.template({ :suppress_data => expected }) + assert_match /supress_data: #{expected}/,value #NB: option spelling needs fixing + end + def test_template_option_suppress_ddl + expected = true + value = Mysql2psql::Config.template({ :suppress_ddl => expected }) + assert_match /supress_ddl: #{expected}/,value #NB: option spelling needs fixing + end + def test_template_option_suppress_sequence_update + expected = true + value = Mysql2psql::Config.template({ :suppress_sequence_update => expected }) + assert_match /supress_sequence_update: #{expected}/,value #NB: option spelling needs fixing + end + def test_template_option_force_truncate + expected = true + value = Mysql2psql::Config.template({ :force_truncate => expected }) + assert_match /force_truncate: #{expected}/,value + end + def test_template_option_use_timezones + expected = true + value = Mysql2psql::Config.template({ :use_timezones => expected }) + assert_match /use_timezones: #{expected}/,value + end end \ No newline at end of file From 3d569a515c1a14f4956d5fb7a49541b7c85dea07 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 13 Feb 2012 01:25:13 +0800 Subject: [PATCH 65/69] fix test_index_conversion(ConvertToDbTest) failure * update sql used to check for index presence * index migration to db was missing index name * added note on limitation concerning index names == table_name --- README.rdoc | 11 ++++++----- lib/mysql2psql/postgres_db_writer.rb | 22 +++++++++++----------- lib/mysql2psql/postgres_file_writer.rb | 2 +- test/fixtures/seed_integration_tests.sql | 2 +- test/integration/convert_to_db_test.rb | 9 +++++++-- test/integration/convert_to_file_test.rb | 1 - 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/README.rdoc b/README.rdoc index 127e2d0..8185c1a 100644 --- a/README.rdoc +++ b/README.rdoc @@ -112,14 +112,14 @@ e.g. == Notes, Limitations, Outstanding Issues.. -=== Todo -- more tests +* index migration doesn't handle cases where the index name is the same as a table name (this is ok on MYSQL by not PostgreSQL). Workaround: manual migration required. +* more test coverage, as always... == Contributors Project founded by Max Lapshin -Other contributors (in git log order): +Contributors (roughly git log order): - Anton Ageev - Samuel Tribehou - Marco Nenciarini @@ -131,5 +131,6 @@ Other contributors (in git log order): - Jacob Coby - Neszt Tibor - Miroslav Kratochvil -- Paul Gallagher -- James Coleman +- {Paul Gallagher}[https://github.com/tardate] +- {James Coleman}[https://github.com/jcoleman] +- {Aaron Peckham}[https://github.com/apeckham] diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index 132b2cd..45a54a3 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -8,8 +8,8 @@ class PostgresDbWriter < PostgresWriter attr_reader :conn, :hostname, :login, :password, :database, :schema, :port def initialize(options) - @hostname, @login, @password, @database, @port = - options.pghostname('localhost'), options.pgusername, + @hostname, @login, @password, @database, @port = + options.pghostname('localhost'), options.pgusername, options.pgpassword, options.pgdatabase, options.pgport(5432).to_s @database, @schema = database.split(":") open @@ -21,7 +21,7 @@ def open @conn.exec("SET client_encoding = 'UTF8'") @conn.exec("SET standard_conforming_strings = off") if @conn.server_version >= 80200 @conn.exec("SET check_function_bodies = false") - @conn.exec("SET client_min_messages = warning") + @conn.exec("SET client_min_messages = warning") end def close @@ -91,21 +91,21 @@ def write_table(table, options) puts "Created table #{table.name}" end - + def write_indexes(table) puts "Indexing table #{table.name}..." if primary_index = table.indexes.find {|index| index[:primary]} - @conn.exec("ALTER TABLE #{PGconn.quote_ident(table.name)} ADD CONSTRAINT \"#{table.name}_pkey\" PRIMARY KEY(#{primary_index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")})") + index_sql = "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD CONSTRAINT \"#{table.name}_pkey\" PRIMARY KEY(#{primary_index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")})" + @conn.exec(index_sql) end - + table.indexes.each do |index| next if index[:primary] unique = index[:unique] ? "UNIQUE " : nil - - @conn.exec("CREATE #{unique}INDEX ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});") + index_sql = "CREATE #{unique}INDEX #{PGconn.quote_ident(index[:name])} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});" + @conn.exec(index_sql) end - - + #@conn.exec("VACUUM FULL ANALYZE #{PGconn.quote_ident(table.name)}") puts "Indexed table #{table.name}" rescue Exception => e @@ -113,7 +113,7 @@ def write_indexes(table) puts e puts e.backtrace[0,3].join("\n") end - + def write_constraints(table) table.foreign_keys.each do |key| key_sql = "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD FOREIGN KEY (#{PGconn.quote_ident(key[:column])}) REFERENCES #{PGconn.quote_ident(key[:ref_table])}(#{PGconn.quote_ident(key[:ref_column])})" diff --git a/lib/mysql2psql/postgres_file_writer.rb b/lib/mysql2psql/postgres_file_writer.rb index 78e6ed3..3b66d48 100644 --- a/lib/mysql2psql/postgres_file_writer.rb +++ b/lib/mysql2psql/postgres_file_writer.rb @@ -68,7 +68,7 @@ def write_sequence_update(table, options) if !options.supress_sequence_update @f << <<-EOF -#{sqlfor_set_serial_sequence(table, serial_key_seq, max_value)} +#{sqlfor_set_serial_sequence(table, serial_key_seq, max_value)} EOF end end diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 63123f4..579e988 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -104,7 +104,7 @@ INSERT INTO test_datetime_conversion (column_a, column_f) VALUES ('0000-00-00 00 DROP TABLE IF EXISTS test_index_conversion; CREATE TABLE test_index_conversion (column_a VARCHAR(10)); -CREATE UNIQUE INDEX test_index_conversion ON test_index_conversion (column_a); +CREATE UNIQUE INDEX index_test_index_conversion_on_column_a ON test_index_conversion (column_a); DROP TABLE IF EXISTS test_foreign_keys_child; DROP TABLE IF EXISTS test_foreign_keys_parent; diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index bef6863..65c2719 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -101,8 +101,13 @@ def test_datetime_defaults end def test_index_conversion - result = exec_sql_on_psql('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = \'test_index_conversion\'::regclass').first - assert_equal "CREATE UNIQUE INDEX test_index_conversion_column_a_idx ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] + result = exec_sql_on_psql(%( + SELECT pg_get_indexdef(indexrelid) + FROM pg_index + join pg_class on pg_class.oid=pg_index.indrelid + where pg_class.relname='test_index_conversion' + )).first + assert_equal "CREATE UNIQUE INDEX index_test_index_conversion_on_column_a ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] end def test_foreign_keys diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index c317d18..e95e028 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -116,7 +116,6 @@ def test_should_not_copy_views_as_tables end def test_truncate - puts "@options: #{@options.inspect}" puts content.inspect assert_match /TRUNCATE/, content end From 533032dc72fd93eb14f09dee57b46c3aa64a952d Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 13 Feb 2012 01:50:01 +0800 Subject: [PATCH 66/69] clean up after 'jtippett/master' merge * green tests --- README.rdoc | 1 + lib/mysql2psql/config.rb | 6 +++--- test/fixtures/config_all_options.yml | 6 +++--- test/integration/convert_to_file_test.rb | 1 - test/units/config_test.rb | 11 ++++++++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.rdoc b/README.rdoc index 8185c1a..6206c24 100644 --- a/README.rdoc +++ b/README.rdoc @@ -134,3 +134,4 @@ Contributors (roughly git log order): - {Paul Gallagher}[https://github.com/tardate] - {James Coleman}[https://github.com/jcoleman] - {Aaron Peckham}[https://github.com/apeckham] +- {James Tippett}[https://github.com/jtippett] diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index 37f9ad0..124aaa3 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -103,10 +103,10 @@ def self.template(options={}) if !options[:suppress_sequence_update].nil? configtext += < expected }) - assert_match /supress_data: #{expected}/,value #NB: option spelling needs fixing + assert_match /suppress_data: #{expected}/,value #NB: option spelling needs fixing end def test_template_option_suppress_ddl expected = true value = Mysql2psql::Config.template({ :suppress_ddl => expected }) - assert_match /supress_ddl: #{expected}/,value #NB: option spelling needs fixing + assert_match /suppress_ddl: #{expected}/,value #NB: option spelling needs fixing end def test_template_option_suppress_sequence_update expected = true value = Mysql2psql::Config.template({ :suppress_sequence_update => expected }) - assert_match /supress_sequence_update: #{expected}/,value #NB: option spelling needs fixing + assert_match /suppress_sequence_update: #{expected}/,value #NB: option spelling needs fixing + end + def test_template_option_suppress_indexes + expected = true + value = Mysql2psql::Config.template({ :suppress_indexes => expected }) + assert_match /suppress_indexes: #{expected}/,value #NB: option spelling needs fixing end def test_template_option_force_truncate expected = true From 45739b598eb2610799003393ad404f899ae3e689 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 13 Feb 2012 02:11:05 +0800 Subject: [PATCH 67/69] step gem version to 0.2.0 * some major changes have been accumulated * including breaking change in config options --- .gitignore | 2 +- CHANGELOG | 8 ++++++ Gemfile | 8 ++++++ Gemfile.lock | 36 ++++++++++++++++++++++++++ Rakefile | 17 +++++++------ lib/mysql2psql/version.rb | 4 +-- mysql2psql.gemspec | 53 ++++++++++++++++++++++++++++++++++++--- 7 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 Gemfile create mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index 5262893..1046f4d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ pkg .rvmrc .bundle *.out - +rdoc diff --git a/CHANGELOG b/CHANGELOG index d969bba..f45f18d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,11 @@ +0.2.2 Feb 13, 2012 +=== + +* merge contributions from: James Coleman, Aaron Peckham, James Tippett +* breaking change: note that "supress*" configuration parameters now use correct spelling. Existing config files you may have will need housekeeping. +* new configuration options: suppress_sequence_update; suppress_indexes; use_timezones +* adds a Gemfile to help setup development environment + 0.1.1 May 3, 2011 === diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..0490c96 --- /dev/null +++ b/Gemfile @@ -0,0 +1,8 @@ +source 'http://rubygems.org' + +group :development do + gem "bundler", "~> 1.0.21" + gem "jeweler", "~> 1.5.2" +end + +gemspec \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..56caec2 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,36 @@ +PATH + remote: . + specs: + mysql2psql (0.2.0) + bundler (~> 1.0.21) + jeweler (~> 1.5.2) + mysql (= 2.8.1) + mysql2psql + pg (~> 0.11.0) + +GEM + remote: http://rubygems.org/ + specs: + git (1.2.5) + jeweler (1.5.2) + bundler (~> 1.0.0) + git (>= 1.2.5) + rake + json (1.6.5) + mysql (2.8.1) + pg (0.11.0) + rake (0.9.2.2) + rdoc (3.12) + json (~> 1.4) + test-unit (2.4.7) + +PLATFORMS + ruby + +DEPENDENCIES + bundler (~> 1.0.21) + jeweler (~> 1.5.2) + mysql2psql! + rake (~> 0.9.2.2) + rdoc (~> 3.12) + test-unit (>= 2.1.1) diff --git a/Rakefile b/Rakefile index da72929..8edfa0a 100644 --- a/Rakefile +++ b/Rakefile @@ -27,11 +27,16 @@ begin "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", - "Paul Gallagher " - ] + "Paul Gallagher ", + "James Coleman ", + "Aaron Peckham", + "James Tippett" + ] gem.add_dependency "mysql", "= 2.8.1" gem.add_dependency "pg", "~> 0.11.0" gem.add_development_dependency "test-unit", ">= 2.1.1" + gem.add_development_dependency "rake", "~> 0.9.2.2" + gem.add_development_dependency "rdoc", "~> 3.12" # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings end Jeweler::GemcutterTasks.new @@ -75,15 +80,13 @@ rescue LoadError end end - task :default => :test -require 'rake/rdoctask' -Rake::RDocTask.new do |rdoc| +require 'rdoc/task' +RDoc::Task.new do |rdoc| version = Mysql2psql::Version::STRING rdoc.rdoc_dir = 'rdoc' rdoc.title = "mysql2psql #{version}" - rdoc.rdoc_files.include('README*') - rdoc.rdoc_files.include('lib/**/*.rb') + rdoc.rdoc_files.include("README*", "lib/**/*.rb") end diff --git a/lib/mysql2psql/version.rb b/lib/mysql2psql/version.rb index b791a1a..efb916e 100644 --- a/lib/mysql2psql/version.rb +++ b/lib/mysql2psql/version.rb @@ -1,8 +1,8 @@ class Mysql2psql module Version MAJOR = 0 - MINOR = 1 - PATCH = 1 + MINOR = 2 + PATCH = 0 STRING = [MAJOR, MINOR, PATCH].compact.join('.') end diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index 0e41783..6b026e0 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -5,11 +5,12 @@ Gem::Specification.new do |s| s.name = %q{mysql2psql} - s.version = "0.1.1" + s.version = "0.2.0" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= - s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher "] - s.date = %q{2011-05-02} + s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher ", "James Coleman ", "Aaron Peckham", "James Tippett"] + s.date = %q{2012-02-13} + s.default_executable = %q{mysql2psql} s.description = %q{It can create postgresql dump from mysql database or directly load data from mysql to postgresql (at about 100 000 records per minute). Translates most data types and indexes.} s.email = %q{gallagher.paul@gmail.com} @@ -19,6 +20,8 @@ Gem::Specification.new do |s| ] s.files = [ "CHANGELOG", + "Gemfile", + "Gemfile.lock", "MIT-LICENSE", "README.rdoc", "Rakefile", @@ -51,7 +54,7 @@ Gem::Specification.new do |s| ] s.homepage = %q{https://github.com/tardate/mysql2postgres} s.require_paths = ["lib"] - s.rubygems_version = %q{1.7.2} + s.rubygems_version = %q{1.6.2} s.summary = %q{Tool for converting mysql database to postgresql} s.test_files = [ "test/integration/convert_to_db_test.rb", @@ -71,18 +74,60 @@ Gem::Specification.new do |s| s.specification_version = 3 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then + s.add_runtime_dependency(%q, [">= 0"]) + s.add_development_dependency(%q, ["~> 1.0.21"]) + s.add_development_dependency(%q, ["~> 1.5.2"]) + s.add_development_dependency(%q, [">= 2.1.1"]) + s.add_development_dependency(%q, ["~> 0.9.2.2"]) + s.add_development_dependency(%q, ["~> 3.12"]) + s.add_development_dependency(%q, [">= 2.1.1"]) + s.add_development_dependency(%q, ["~> 0.9.2.2"]) + s.add_development_dependency(%q, ["~> 3.12"]) + s.add_development_dependency(%q, [">= 2.1.1"]) + s.add_development_dependency(%q, ["~> 0.9.2.2"]) + s.add_development_dependency(%q, ["~> 3.12"]) s.add_runtime_dependency(%q, ["= 2.8.1"]) s.add_runtime_dependency(%q, ["~> 0.11.0"]) s.add_development_dependency(%q, [">= 2.1.1"]) + s.add_development_dependency(%q, ["~> 0.9.2.2"]) + s.add_development_dependency(%q, ["~> 3.12"]) else + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, ["~> 1.0.21"]) + s.add_dependency(%q, ["~> 1.5.2"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) s.add_dependency(%q, ["= 2.8.1"]) s.add_dependency(%q, ["~> 0.11.0"]) s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) end else + s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, ["~> 1.0.21"]) + s.add_dependency(%q, ["~> 1.5.2"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) s.add_dependency(%q, ["= 2.8.1"]) s.add_dependency(%q, ["~> 0.11.0"]) s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) end end From 62d0180c1cf7b5faa63ef5fe4a59484ec5249287 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 13 Feb 2012 03:15:39 +0800 Subject: [PATCH 68/69] update gemspec --- mysql2psql.gemspec | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/mysql2psql.gemspec b/mysql2psql.gemspec index 6b026e0..aab7d7f 100644 --- a/mysql2psql.gemspec +++ b/mysql2psql.gemspec @@ -8,7 +8,7 @@ Gem::Specification.new do |s| s.version = "0.2.0" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= - s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher ", "James Coleman ", "Aaron Peckham", "James Tippett"] + s.authors = ["Max Lapshin ", "Anton Ageev ", "Samuel Tribehou ", "Marco Nenciarini ", "James Nobis ", "quel ", "Holger Amann ", "Maxim Dobriakov ", "Michael Kimsal ", "Jacob Coby ", "Neszt Tibor ", "Miroslav Kratochvil ", "Paul Gallagher ", "James Coleman ", "Aaron Peckham", "James Tippett", "Tim Morgan", "dakhota"] s.date = %q{2012-02-13} s.default_executable = %q{mysql2psql} s.description = %q{It can create postgresql dump from mysql database or directly load data from mysql to @@ -77,6 +77,11 @@ Gem::Specification.new do |s| s.add_runtime_dependency(%q, [">= 0"]) s.add_development_dependency(%q, ["~> 1.0.21"]) s.add_development_dependency(%q, ["~> 1.5.2"]) + s.add_development_dependency(%q, ["~> 1.0.21"]) + s.add_development_dependency(%q, ["~> 1.5.2"]) + s.add_development_dependency(%q, [">= 2.1.1"]) + s.add_development_dependency(%q, ["~> 0.9.2.2"]) + s.add_development_dependency(%q, ["~> 3.12"]) s.add_development_dependency(%q, [">= 2.1.1"]) s.add_development_dependency(%q, ["~> 0.9.2.2"]) s.add_development_dependency(%q, ["~> 3.12"]) @@ -95,6 +100,11 @@ Gem::Specification.new do |s| s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, ["~> 1.0.21"]) s.add_dependency(%q, ["~> 1.5.2"]) + s.add_dependency(%q, ["~> 1.0.21"]) + s.add_dependency(%q, ["~> 1.5.2"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) s.add_dependency(%q, [">= 2.1.1"]) s.add_dependency(%q, ["~> 0.9.2.2"]) s.add_dependency(%q, ["~> 3.12"]) @@ -114,6 +124,11 @@ Gem::Specification.new do |s| s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, ["~> 1.0.21"]) s.add_dependency(%q, ["~> 1.5.2"]) + s.add_dependency(%q, ["~> 1.0.21"]) + s.add_dependency(%q, ["~> 1.5.2"]) + s.add_dependency(%q, [">= 2.1.1"]) + s.add_dependency(%q, ["~> 0.9.2.2"]) + s.add_dependency(%q, ["~> 3.12"]) s.add_dependency(%q, [">= 2.1.1"]) s.add_dependency(%q, ["~> 0.9.2.2"]) s.add_dependency(%q, ["~> 3.12"]) From b97fad149257d6a4cff34c914e599359bb4063a1 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 13 Feb 2012 03:55:10 +0800 Subject: [PATCH 69/69] Restore conflicting index name migration * merges intent of ef9951ed29b8916d28f3d212ffd68eef44818cce by Aaron Peckham with preious technique for duplicate detection for PostgreSQL < 9 --- README.rdoc | 1 - lib/mysql2psql/postgres_db_writer.rb | 21 ++++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/README.rdoc b/README.rdoc index b6f21d0..97ba3df 100644 --- a/README.rdoc +++ b/README.rdoc @@ -121,7 +121,6 @@ e.g. == Notes, Limitations, Outstanding Issues.. -* index migration doesn't handle cases where the index name is the same as a table name (this is ok on MYSQL by not PostgreSQL). Workaround: manual migration required. * more test coverage, as always... diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index 9f30f17..72bb15d 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -102,7 +102,26 @@ def write_indexes(table) table.indexes.each do |index| next if index[:primary] unique = index[:unique] ? "UNIQUE " : nil - index_sql = "CREATE #{unique}INDEX #{PGconn.quote_ident(index[:name])} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});" + + # MySQL allows an index name which could be equal to a table name, Postgres doesn't + indexname = index[:name] + indexname_quoted = '' + + if indexname.eql?(table.name) + indexname = (@conn.server_version < 90000) ? "#{indexname}_index" : nil + puts "WARNING: index \"#{index[:name]}\" equals table name. This is not allowed in PostgreSQL and will be renamed." + end + + if indexname + indexname_quoted = PGconn.quote_ident(indexname) + if @conn.server_version < 80200 + @conn.exec("DROP INDEX #{PGconn.quote_ident(indexname)} CASCADE;") if exists?(indexname) + else + @conn.exec("DROP INDEX IF EXISTS #{PGconn.quote_ident(indexname)} CASCADE;") + end + end + + index_sql = "CREATE #{unique}INDEX #{indexname_quoted} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});" @conn.exec(index_sql) end