Skip to content

Commit

Permalink
Fix autobackup defaults and zip level for performance (#6518)
Browse files Browse the repository at this point in the history 
* Fix autobackup defaults and zip level for performance

* Add zipcompression configuration option
  • Loading branch information
@PTR-inc
PTR-inc authored Nov 10, 2024
1 parent 7d59210 commit b71c69e
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 23 deletions.
28 changes: 10 additions & 18 deletions db.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3434,7 +3434,8 @@ module.exports.CreateDB = function (parent, func) {
obj.performBackup = function (func) {
parent.debug('db','Entering performBackup');
try {
if (obj.performingBackup) return 1;
if (obj.performingBackup) return 'Backup alreay in progress.';
if (parent.config.settings.autobackup.backupintervalhours == -1) { if (func) { func('Unable to create backup if backuppath is set to the data folder.'); return 'Backup aborted.' }};
obj.performingBackup = true;
let backupPath = parent.backuppath;
let dataPath = parent.datapath;
Expand All @@ -3446,11 +3447,10 @@ module.exports.CreateDB = function (parent, func) {
obj.newAutoBackupFile = path.join(backupPath, ((typeof parent.config.settings.autobackup.backupname == 'string') ? parent.config.settings.autobackup.backupname : 'meshcentral-autobackup-') + fileSuffix + '.zip');

if ((obj.databaseType == DB_MONGOJS) || (obj.databaseType == DB_MONGODB)) {
// Perform a MongoDump in the datadir
// Perform a MongoDump
const dbname = (parent.args.mongodbname) ? (parent.args.mongodbname) : 'meshcentral';
const dburl = parent.args.mongodb;

//const obj.newDBDumpFile = 'mongodump-' + fileSuffix;
obj.newDBDumpFile = path.join(backupPath, (dbname + '-mongodump-' + fileSuffix + '.archive'));

var cmd = buildMongoDumpCommand();
Expand Down Expand Up @@ -3490,16 +3490,6 @@ module.exports.CreateDB = function (parent, func) {
} else if (obj.databaseType == DB_SQLITE) {
//.db3 suffix to escape escape backupfile glob to exclude the sqlite db files
obj.newDBDumpFile = path.join(backupPath, databaseName + '-sqlitedump-' + fileSuffix + '.db3');
/*undocumented in node-sqlite3 API, check https://github.com/TryGhost/node-sqlite3/blob/593c9d498be2510d286349134537e3bf89401c4a/test/backup.test.js
var backup = obj.file.backup(obj.newDBDumpFile);
backup.step(-1, function (err) {
if (err) { console.log('SQLite start-backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP; obj.createBackupfile(func); };
backup.finish(function (err) {
if (err) { console.log('SQLite backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP;};
obj.createBackupfile(func);
});
});
*/
// do a VACUUM INTO in favor of the backup API to compress the export, see https://www.sqlite.org/backup.html
obj.file.exec('VACUUM INTO \'' + obj.newDBDumpFile + '\'', function (err) {
if (err) { console.log('SQLite start-backup error: ' + err); obj.backupStatus |=BACKUPFAIL_DBDUMP;};
Expand Down Expand Up @@ -3529,19 +3519,21 @@ module.exports.CreateDB = function (parent, func) {
obj.createBackupfile(func);
}
} catch (ex) { console.log(ex); };
return(0);
return 'Starting auto-backup...';
};

obj.createBackupfile = function(func) {
parent.debug('db', 'Entering createFileBackup');
let archiver = require('archiver');
let archive = null;
let zipLevel = Math.min(Math.max(Number(parent.config.settings.autobackup.zipcompression ? parent.config.settings.autobackup.zipcompression : 5),1),9);

//if password defined, create encrypted zip
if (parent.config.settings.autobackup && (typeof parent.config.settings.autobackup.zippassword == 'string')) {
try {
//Only register format once, otherwise it triggers an error
if (archiver.isRegisteredFormat('zip-encrypted') == false) { archiver.registerFormat('zip-encrypted', require('archiver-zip-encrypted')); }
archive = archiver.create('zip-encrypted', { zlib: { level: 9 }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword });
archive = archiver.create('zip-encrypted', { zlib: { level: zipLevel }, encryptionMethod: 'aes256', password: parent.config.settings.autobackup.zippassword });
if (func) { func('Creating encrypted ZIP'); }
} catch (ex) { // registering encryption failed, do not fall back to non-encrypted, fail backup and skip old backup removal as a precaution to not lose any backups
obj.backupStatus |= BACKUPFAIL_ZIPMODULE;
Expand All @@ -3550,7 +3542,7 @@ module.exports.CreateDB = function (parent, func) {
}
} else {
if (func) { func('Creating a NON-ENCRYPTED ZIP'); }
archive = archiver('zip', { zlib: { level: 9 } });
archive = archiver('zip', { zlib: { level: zipLevel } });
}

//original behavior, just a filebackup if dbdump fails : (obj.backupStatus == 0 || obj.backupStatus == BACKUPFAIL_DBDUMP)
Expand Down Expand Up @@ -3628,14 +3620,14 @@ module.exports.CreateDB = function (parent, func) {

let globIgnoreFiles;
//slice in case exclusion gets pushed
globIgnoreFiles = parent.config.settings.autobackup.backupignorefilesglob.slice();
globIgnoreFiles = parent.config.settings.autobackup.backupignorefilesglob ? parent.config.settings.autobackup.backupignorefilesglob.slice() : [];
if (parent.config.settings.sqlite3) { globIgnoreFiles.push (datapathFoldername + '/' + databaseName + '.sqlite*'); }; //skip sqlite database file, and temp files with ext -journal, -wal & -shm
//archiver.glob doesn't seem to use the third param, archivesubdir. Bug?
//workaround: go up a dir and add data dir explicitly to keep the zip tidy
archive.glob((datapathFoldername + '/**'), {
cwd: datapathParentPath,
ignore: globIgnoreFiles,
skip: parent.config.settings.autobackup.backupskipfoldersglob
skip: (parent.config.settings.autobackup.backupskipfoldersglob ? parent.config.settings.autobackup.backupskipfoldersglob : [])
});

if (parent.config.settings.autobackup.backupwebfolders) {
Expand Down
5 changes: 5 additions & 0 deletions meshcentral-config-schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -851,6 +851,11 @@
"default": 10,
"description": "How many days of backups should the autobackup keep? Default is 10 Days worth"
},
"zipCompression" : {
"type": "integer",
"default": "5",
"description": "Set the zip compression level, 1=fast/less small file to 9=slow/smallest file."
},
"zipPassword": {
"type": "string",
"default": "",
Expand Down
6 changes: 3 additions & 3 deletions meshcentral.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2089,7 +2089,7 @@ function CreateMeshCentralServer(config, args) {
obj.updateServerState('state', "running");

// Setup auto-backup defaults
if (obj.config.settings.autobackup == null || obj.config.settings.autobackup == false || obj.config.settings.autobackup == 'false') { delete obj.config.settings.autobackup; }
if (obj.config.settings.autobackup == null || obj.config.settings.autobackup == false || obj.config.settings.autobackup == 'false') { obj.config.settings.autobackup = {backupintervalhours: 0}; } //no schedule, but able to console autobackup
else {
if (obj.config.settings.autobackup === true) {obj.config.settings.autobackup = {backupintervalhours: 24, keeplastdaysbackup: 10}; };
if (typeof obj.config.settings.autobackup.backupintervalhours != 'number') { obj.config.settings.autobackup.backupintervalhours = 24; };
Expand All @@ -2104,7 +2104,7 @@ function CreateMeshCentralServer(config, args) {
// Check that autobackup path is not within the "meshcentral-data" folder.
if ((typeof obj.config.settings.autobackup == 'object') && (typeof obj.config.settings.autobackup.backuppath == 'string') && (obj.path.normalize(obj.config.settings.autobackup.backuppath).startsWith(obj.path.normalize(obj.datapath)))) {
addServerWarning("Backup path can't be set within meshcentral-data folder, backup settings ignored.", 21);
delete obj.config.settings.autobackup;
obj.config.settings.autobackup = {backupintervalhours: -1}; //block console autobackup
}

// Load Intel AMT passwords from the "amtactivation.log" file
Expand Down Expand Up @@ -2267,7 +2267,7 @@ function CreateMeshCentralServer(config, args) {

// Check if we need to perform an automatic backup
function checkAutobackup() {
if (obj.config.settings.autobackup && (typeof obj.config.settings.autobackup.backupintervalhours == 'number')) {
if (obj.config.settings.autobackup.backupintervalhours >= 1) {
obj.db.Get('LastAutoBackupTime', function (err, docs) {
if (err != null) return;
var lastBackup = 0;
Expand Down
3 changes: 1 addition & 2 deletions meshuser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7627,10 +7627,9 @@ module.exports.CreateMeshUser = function (parent, db, ws, req, args, domain, use
}

function serverUserCommandAutoBackup(cmdData) {
var backupResult = parent.db.performBackup(function (msg) {
cmdData.result = parent.db.performBackup(function (msg) {
try { ws.send(JSON.stringify({ action: 'serverconsole', value: msg, tag: cmdData.command.tag })); } catch (ex) { }
});
if (backupResult == 0) { cmdData.result = 'Starting auto-backup...'; } else { cmdData.result = 'Backup alreay in progress.'; }
}

function serverUserCommandBackupConfig(cmdData) {
Expand Down

0 comments on commit b71c69e

Please sign in to comment.