Skip to content

Commit 91612ec

Browse files
committed
resolved merge conflicts
2 parents c8cec71 + 9258a5e commit 91612ec

File tree

11 files changed

+333
-222
lines changed

11 files changed

+333
-222
lines changed

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ Or, if you have moved <code>config</code> folder out from Nmig's directory:<br /
109109
<br /><b>Note:</b> "logs_directory" will be created during script execution.</p>
110110

111111
<h3>VERSION</h3>
112-
<p>Current version is 5.4.0<br />
112+
<p>Current version is 5.5.0<br />
113113

114114
<h3>LICENSE</h3>
115115
<p>NMIG is available under "GNU GENERAL PUBLIC LICENSE" (v. 3) <br />

config/index_types_map.json

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"README" : [
3+
"This JSON document represents a correct index-types map between MySQL and PostgreSQL.",
4+
"If you wish to customize (not recommended) this map - you can.",
5+
"Map explanation:",
6+
"Each key represents a MySQL index-type, and each value represents corresponding PostgreSQL index-type."
7+
],
8+
9+
"BTREE": "BTREE",
10+
11+
"HASH": "HASH",
12+
13+
"SPATIAL": "GIST",
14+
15+
"FULLTEXT": "GIN"
16+
}

package-lock.json

Lines changed: 218 additions & 105 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "nmig",
3-
"version": "5.4.0",
3+
"version": "5.5.0",
44
"description": "The database migration app",
55
"author": "Anatoly Khaytovich<[email protected]>",
66
"license": "GPL-3.0",
@@ -12,18 +12,18 @@
1212
"node": ">=10.0.0"
1313
},
1414
"dependencies": {
15-
"json2csv": "^5.0.1",
15+
"json2csv": "^5.0.3",
1616
"mysql": "^2.18.1",
17-
"pg": "^8.3.3",
18-
"pg-copy-streams": "^2.2.2",
17+
"pg": "^8.4.2",
18+
"pg-copy-streams": "^5.1.1",
1919
"@types/mysql": "^2.15.15",
20-
"@types/node": "^14.10.1",
21-
"@types/pg": "^7.14.4"
20+
"@types/node": "^14.14.5",
21+
"@types/pg": "^7.14.5"
2222
},
2323
"devDependencies": {
2424
"@types/tape": "^4.13.0",
2525
"tape": "^5.0.1",
26-
"typescript": "^4.0.2"
26+
"typescript": "^4.0.5"
2727
},
2828
"scripts": {
2929
"build": "tsc",

src/BootProcessor.ts

Lines changed: 26 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import DBVendors from './DBVendors';
2626
import IDBAccessQueryParams from './IDBAccessQueryParams';
2727
import IConfAndLogsPaths from './IConfAndLogsPaths';
2828
import { getStateLogsTableName } from './MigrationStateManager';
29+
import { generateError, log } from './FsOps';
2930

3031
/**
3132
* Checks correctness of connection details of both MySQL and PostgreSQL.
@@ -67,61 +68,36 @@ export const getLogo = (): string => {
6768
/**
6869
* Boots the migration.
6970
*/
70-
export const boot = (conversion: Conversion): Promise<Conversion> => {
71-
return new Promise<Conversion>(async resolve => {
72-
const connectionErrorMessage = await checkConnection(conversion);
73-
const logo: string = getLogo();
71+
export const boot = async (conversion: Conversion): Promise<Conversion> => {
72+
const connectionErrorMessage = await checkConnection(conversion);
73+
const logo: string = getLogo();
74+
const logTitle: string = 'BootProcessor::boot';
7475

75-
if (connectionErrorMessage) {
76-
console.log(`${ logo } \n ${ connectionErrorMessage }`);
77-
process.exit(1);
78-
}
76+
if (connectionErrorMessage) {
77+
await generateError(conversion, `\t--[${ logTitle }]\n ${ logo } \n ${ connectionErrorMessage }`);
78+
process.exit(1);
79+
}
7980

80-
const sql: string = `SELECT EXISTS(SELECT 1 FROM information_schema.tables`
81-
+ ` WHERE table_schema = '${ conversion._schema }' AND table_name = '${ getStateLogsTableName(conversion, true) }');`;
81+
const sql: string = `SELECT EXISTS(SELECT 1 FROM information_schema.tables`
82+
+ ` WHERE table_schema = '${ conversion._schema }' AND table_name = '${ getStateLogsTableName(conversion, true) }');`;
8283

83-
const params: IDBAccessQueryParams = {
84-
conversion: conversion,
85-
caller: 'BootProcessor::boot',
86-
sql: sql,
87-
vendor: DBVendors.PG,
88-
processExitOnError: true,
89-
shouldReturnClient: false
90-
};
91-
92-
const result: DBAccessQueryResult = await DBAccess.query(params);
93-
const isExists: boolean = !!result.data.rows[0].exists;
94-
const message: string = `${ (isExists
95-
? '\n\t--[boot] NMIG is ready to restart after some failure.\n\t--[boot] Consider checking log files at the end of migration.'
96-
: '\n\t--[boot] NMIG is ready to start.') } \n\t--[boot] Proceed? [Y/n]`;
97-
98-
console.log(logo + message);
99-
100-
const _getUserInput = (input: string): void => {
101-
const trimedInput: string = input.trim();
102-
103-
if (trimedInput === 'n' || trimedInput === 'N') {
104-
console.log('\t--[boot] Migration aborted.\n');
105-
process.exit(0);
106-
}
107-
108-
if (trimedInput === 'y' || trimedInput === 'Y') {
109-
process.stdin.removeListener('data', _getUserInput);
110-
conversion._timeBegin = new Date();
111-
return resolve(conversion);
112-
}
113-
114-
const hint: string = `\t--[boot] Unexpected input ${ trimedInput }\n`
115-
+ `\t--[boot] Expected input is upper case Y\n\t--[boot] or lower case n\n${message}`;
84+
const params: IDBAccessQueryParams = {
85+
conversion: conversion,
86+
caller: 'BootProcessor::boot',
87+
sql: sql,
88+
vendor: DBVendors.PG,
89+
processExitOnError: true,
90+
shouldReturnClient: false
91+
};
11692

117-
console.log(hint);
118-
};
93+
const result: DBAccessQueryResult = await DBAccess.query(params);
94+
const isExists: boolean = !!result.data.rows[0].exists;
95+
const message: string = `${ (isExists
96+
? '\n\t--[boot] NMIG is restarting after some failure.\n\t--[boot] Consider checking log files at the end of migration.\n'
97+
: '\n\t--[boot] NMIG is starting.') } \n`;
11998

120-
process.stdin
121-
.resume()
122-
.setEncoding(conversion._encoding)
123-
.on('data', _getUserInput);
124-
});
99+
log(conversion, `\t--[${ logTitle }] ${ logo }${ message }`);
100+
return conversion;
125101
};
126102

127103
/**

src/Conversion.ts

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@ export default class Conversion {
8585
*/
8686
public readonly _dataTypesMapAddr: string;
8787

88+
/**
89+
* A path to the index types map.
90+
*/
91+
public readonly _indexTypesMapAddr: string;
92+
8893
/**
8994
* A path to the "errors-only.log" file.
9095
*/
@@ -108,7 +113,7 @@ export default class Conversion {
108113
/**
109114
* The timestamp, at which the migration began.
110115
*/
111-
public _timeBegin: Date | null;
116+
public _timeBegin: Date;
112117

113118
/**
114119
* Current version of source (MySQL) db.
@@ -185,6 +190,11 @@ export default class Conversion {
185190
*/
186191
public _dataTypesMap: any;
187192

193+
/**
194+
* The index types map.
195+
*/
196+
public _indexTypesMap: any;
197+
188198
/**
189199
* Buffer level when stream.write() starts returning false.
190200
* This number is a number of JavaScript objects.
@@ -200,12 +210,13 @@ export default class Conversion {
200210
this._targetConString = this._config.target;
201211
this._logsDirPath = this._config.logsDirPath;
202212
this._dataTypesMapAddr = this._config.dataTypesMapAddr;
213+
this._indexTypesMapAddr = this._config.indexTypesMapAddr;
203214
this._allLogsPath = path.join(this._logsDirPath, 'all.log');
204215
this._errorLogsPath = path.join(this._logsDirPath, 'errors-only.log');
205216
this._notCreatedViewsPath = path.join(this._logsDirPath, 'not_created_views');
206217
this._excludeTables = this._config.exclude_tables === undefined ? [] : this._config.exclude_tables;
207218
this._includeTables = this._config.include_tables === undefined ? [] : this._config.include_tables;
208-
this._timeBegin = null;
219+
this._timeBegin = new Date();
209220
this._encoding = this._config.encoding === undefined ? 'utf8' : this._config.encoding;
210221
this._0777 = '0777';
211222
this._mysqlVersion = '5.6.21'; // Simply a default value.

src/DataLoader.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,10 +165,10 @@ const getCopyStream = (
165165
const copyStream: any = client.query(from(sqlCopy));
166166

167167
copyStream
168-
.on('end', async () => {
168+
.on('finish', async () => {
169169
// COPY FROM STDIN does not return the number of rows inserted.
170170
// But the transactional behavior still applies, meaning no records inserted if at least one failed.
171-
// That is why in case of 'on end' the rowsCnt value is actually the number of records inserted.
171+
// That is why in case of 'on finish' the rowsCnt value is actually the number of records inserted.
172172
processSend(new MessageToMaster(tableName, rowsCnt));
173173
await deleteChunk(conv, dataPoolId, client);
174174
})

src/FsOps.ts

Lines changed: 35 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -84,48 +84,57 @@ export const log = (conversion: Conversion, log: string | NodeJS.ErrnoException,
8484
};
8585

8686
/**
87-
* Reads the configuration file.
87+
* Reads and parses JOSN file under given path.
8888
*/
89-
export const readConfig = (confPath: string, logsPath: string, configFileName: string = 'config.json'): Promise<any> => {
89+
const readAndParseJsonFile = (pathToFile: string): Promise<any> => {
9090
return new Promise<any>(resolve => {
91-
const pathToConfig = path.join(confPath, configFileName);
92-
93-
fs.readFile(pathToConfig, (error: ErrnoException | null, data: Buffer) => {
91+
fs.readFile(pathToFile, (error: ErrnoException | null, data: Buffer) => {
9492
if (error) {
95-
console.log(`\n\t--Cannot run migration\nCannot read configuration info from ${ pathToConfig }`);
93+
console.log(`\n\t--Cannot run migration\nCannot read configuration info from ${ pathToFile }`);
9694
process.exit(1);
9795
}
9896

9997
const config: any = JSON.parse(data.toString());
100-
config.logsDirPath = path.join(logsPath, 'logs_directory');
101-
config.dataTypesMapAddr = path.join(confPath, 'data_types_map.json');
10298
resolve(config);
10399
});
104100
});
105101
};
106102

107103
/**
108-
* Reads the extra configuration file, if necessary.
104+
* Reads the configuration file.
109105
*/
110-
export const readExtraConfig = (config: any, confPath: string, extraConfigFileName: string = 'extra_config.json'): Promise<any> => {
111-
return new Promise<any>(resolve => {
112-
if (config.enable_extra_config !== true) {
113-
config.extraConfig = null;
114-
return resolve(config);
115-
}
116-
117-
const pathToExtraConfig = path.join(confPath, extraConfigFileName);
106+
export const readConfig = async (confPath: string, logsPath: string, configFileName: string = 'config.json'): Promise<any> => {
107+
const pathToConfig = path.join(confPath, configFileName);
108+
const config: any = await readAndParseJsonFile(pathToConfig);
109+
config.logsDirPath = path.join(logsPath, 'logs_directory');
110+
config.dataTypesMapAddr = path.join(confPath, 'data_types_map.json');
111+
config.indexTypesMapAddr = path.join(confPath, 'index_types_map.json');
112+
return config;
113+
};
118114

119-
fs.readFile(pathToExtraConfig, (error: ErrnoException | null, data: Buffer) => {
120-
if (error) {
121-
console.log(`\n\t--Cannot run migration\nCannot read configuration info from ${ pathToExtraConfig }`);
122-
process.exit(1);
123-
}
115+
/**
116+
* Reads the extra configuration file, if necessary.
117+
*/
118+
export const readExtraConfig = async (config: any, confPath: string, extraConfigFileName: string = 'extra_config.json'): Promise<any> => {
119+
if (config.enable_extra_config !== true) {
120+
config.extraConfig = null;
121+
return config;
122+
}
123+
124+
const pathToExtraConfig = path.join(confPath, extraConfigFileName);
125+
config.extraConfig = await readAndParseJsonFile(pathToExtraConfig);
126+
return config;
127+
};
124128

125-
config.extraConfig = JSON.parse(data.toString());
126-
resolve(config);
127-
});
128-
});
129+
/**
130+
* Reads both "./config/data_types_map.json" and "./config/index_types_map.json" and converts its json content to js object.
131+
*/
132+
export const readDataAndIndexTypesMap = async (conversion: Conversion): Promise<Conversion> => {
133+
const logTitle: string = 'FsOps::readDataAndIndexTypesMap';
134+
conversion._dataTypesMap = await readAndParseJsonFile(conversion._dataTypesMapAddr);
135+
conversion._indexTypesMap = await readAndParseJsonFile(conversion._indexTypesMapAddr);
136+
log(conversion, `\t--[${ logTitle }] Data and Index Types Maps are loaded...`);
137+
return conversion;
129138
};
130139

131140
/**
@@ -166,23 +175,3 @@ const createDirectory = (conversion: Conversion, directoryPath: string, logTitle
166175
});
167176
});
168177
};
169-
170-
/**
171-
* Reads "./config/data_types_map.json" and converts its json content to js object.
172-
*/
173-
export const readDataTypesMap = (conversion: Conversion): Promise<Conversion> => {
174-
return new Promise<Conversion>(resolve => {
175-
fs.readFile(conversion._dataTypesMapAddr, (error: ErrnoException | null, data: Buffer) => {
176-
const logTitle: string = 'FsOps::readDataTypesMap';
177-
178-
if (error) {
179-
console.log(`\t--[${ logTitle }] Cannot read "DataTypesMap" from ${conversion._dataTypesMapAddr}`);
180-
process.exit(1);
181-
}
182-
183-
conversion._dataTypesMap = JSON.parse(data.toString());
184-
console.log(`\t--[${ logTitle }] Data Types Map is loaded...`);
185-
resolve(conversion);
186-
});
187-
});
188-
};

src/IndexAndKeyProcessor.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,13 @@ import DBAccessQueryResult from './DBAccessQueryResult';
2626
import IDBAccessQueryParams from './IDBAccessQueryParams';
2727
import * as extraConfigProcessor from './ExtraConfigProcessor';
2828

29+
/**
30+
* Returns PostgreSQL index type, that correlates to given MySQL index type.
31+
*/
32+
const getIndexType = (conversion: Conversion, indexType: string): string => {
33+
return indexType in conversion._indexTypesMap ? conversion._indexTypesMap[indexType] : 'BTREE';
34+
};
35+
2936
/**
3037
* Creates primary key and indices.
3138
*/
@@ -62,7 +69,7 @@ export default async (conversion: Conversion, tableName: string): Promise<void>
6269
objPgIndices[index.Key_name] = {
6370
is_unique: index.Non_unique === 0,
6471
column_name: [`"${ pgColumnName }"`],
65-
Index_type: ` USING ${ index.Index_type === 'SPATIAL' ? 'GIST' : index.Index_type }`
72+
index_type: ` USING ${ getIndexType(conversion, index.Index_type) }`,
6673
};
6774
});
6875

@@ -79,7 +86,7 @@ export default async (conversion: Conversion, tableName: string): Promise<void>
7986
indexType = 'index';
8087
sqlAddIndex = `CREATE ${ (objPgIndices[index].is_unique ? 'UNIQUE ' : '') }INDEX "${ conversion._schema }_${ tableName }_${ columnName }_idx"
8188
ON "${ conversion._schema }"."${ tableName }"
82-
${ objPgIndices[index].Index_type } (${ objPgIndices[index].column_name.join(',') });`;
89+
${ objPgIndices[index].index_type } (${ objPgIndices[index].column_name.join(',') });`;
8390
}
8491

8592
params.vendor = DBVendors.PG;

src/Main.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,16 @@ import { processConstraints } from './ConstraintsProcessor';
3030
import { getConfAndLogsPaths, boot } from './BootProcessor';
3131
import { createStateLogsTable, dropStateLogsTable } from './MigrationStateManager';
3232
import { createDataPoolTable, readDataPool } from './DataPoolManager';
33-
import { readConfig, readExtraConfig, createLogsDirectory, readDataTypesMap } from './FsOps';
33+
import { readConfig, readExtraConfig, createLogsDirectory, readDataAndIndexTypesMap } from './FsOps';
3434

3535
const { confPath, logsPath } = getConfAndLogsPaths();
3636

3737
readConfig(confPath, logsPath)
3838
.then(config => readExtraConfig(config, confPath))
3939
.then(Conversion.initializeConversion)
40-
.then(boot)
41-
.then(readDataTypesMap)
4240
.then(createLogsDirectory)
41+
.then(readDataAndIndexTypesMap)
42+
.then(boot)
4343
.then(createSchema)
4444
.then(createStateLogsTable)
4545
.then(createDataPoolTable)

0 commit comments

Comments
 (0)