Merge branch 'QT-227' of github.com:Countly/countly-server into QT-227

CHANGELOG.md

@@ -1,3 +1,41 @@
+## Version 24.05.14
+Fixes:
+- [code] Added better handling for countly servers while deployed using subdirectory
+- [data-manager] Added missing "Create" button for event groups
+- [data-manager] Fixes for auto toggling data masking setting on redaction change
+- [populator] Fixed issues in case of wrongly structured data
+- [push] token shouldn't be revoked if it's already in db
+- [server-stats] Fixed a data point calculation bug
+
+Enterprise fixes:
+- [push_approver] Fixed issue with the plugin not handling being "soft disabled"
+- [surveys] fixed a bug with the Survey widget where the "submit" button could become non interactable
+- [surveys] fixed issues with dashboard editor in relation to the "terms and conditions"
+- [surveys] fixed issues with previewing, creating and editing nps widgets
+
+## Version 24.05.13
+Fixes:
+- [alerts] Fixed bugs related to NPS alerts
+- [crash] Reworked symbol files upload to support larger symbol files
+- [push] Fixed bug that would surface when sending Array or Object related payload
+
+Enterprise fixes:
+- [ab-testing] Slight improvements to the UI and UX
+- [remote-config] Slight improvements to the UI and UX
+
+Enterprise Features:
+- [ab-testing] Improved UI for selecting AB test expiration
+
+## Version 24.05.12
+Fixes:
+- [dashboards] Fixes for dashboards grid
+- [dasboards] UI fix for dashboard widget action menu
+- [push] Refactored fcm API related code
+- [reports] Use config for encryption key in reports
+
+Enterprise fixes:
+ - [retention] Fixes for byval retention query calculation
+
 ## Version 24.05.11
 Fixes:
 - [cache] Use a cursor without timeout
@@ -9,7 +47,7 @@ Fixes:
 - [logs]  Show collected problems on logger
 
 Enterprise fixes:
-- [data-manager] Fixes n UI to allow events starting with "/"
+- [data-manager] Fixes in UI to allow events starting with "/"
 - [flows] Fixes for flows step generation
 - [surveys] Other is not allowed as a valid answer for required questions
 

Gruntfile.js

@@ -241,7 +241,7 @@ module.exports = function(grunt) {
                         'frontend/express/public/javascripts/dom/drop/drop-theme-countly.min.css',
                         'frontend/express/public/javascripts/utils/tooltipster/tooltipster.bundle.min.css',
                         'frontend/express/public/stylesheets/bulma/bulma-custom.css',
-                        'frontend/express/public/stylesheets/styles/manifest.css',
+                        'frontend/express/public/stylesheets/styles/manifest2.css',
                         'frontend/express/public/stylesheets/vue/element-tiptap.css',
                     ]
                 }
@@ -324,7 +324,25 @@ module.exports = function(grunt) {
 
     grunt.registerTask('default', ['mochaTest']);
 
-    grunt.registerTask('dist', ['sass', 'concat', 'uglify', 'cssmin']);
+    grunt.registerTask('replace-paths', 'Replace image paths in prod CSS files', function() {
+        var cssFiles = [
+            {
+                filepath: 'frontend/express/public/stylesheets/styles/manifest.css',
+                lookup: '../../images',
+                replacement: '../images',
+                newPath: 'frontend/express/public/stylesheets/styles/manifest2.css'
+            }
+        ];
+
+        cssFiles.forEach(function(file) {
+            var content = grunt.file.read(file.filepath);
+            var newContent = content.replace(/\.\.\/\.\.\/images/g, file.replacement);
+            grunt.file.write(file.newPath, newContent);
+            grunt.log.writeln('Processed file: ' + file.filepath);
+        });
+    });
+
+    grunt.registerTask('dist', ['sass', 'concat', 'uglify', 'replace-paths', 'cssmin']);
 
     grunt.registerTask('plugins', 'Minify plugin JS / CSS files and copy images', function() {
         var js = [], css = [], img = [], fs = require('fs'), path = require('path');

api/api.js

@@ -370,10 +370,18 @@ plugins.connectToAllDatabases().then(function() {
                 }
 
                 const form = new formidable.IncomingForm(formidableOptions);
-                req.body = '';
-                req.on('data', (data) => {
-                    req.body += data;
-                });
+                if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) {
+                    req.body = [];
+                    req.on('data', (data) => {
+                        req.body.push(data);
+                    });
+                }
+                else {
+                    req.body = '';
+                    req.on('data', (data) => {
+                        req.body += data;
+                    });
+                }
 
                 let multiFormData = false;
                 // Check if we have 'multipart/form-data'
@@ -432,4 +440,4 @@ plugins.connectToAllDatabases().then(function() {
 
         plugins.loadConfigs(common.db);
     }
-});
+});

api/config.sample.js

@@ -99,6 +99,7 @@ var countlyConfig = {
     * @property {string} algorithm - name of the algorithm to use for encryption. The algorithm is dependent on OpenSSL, examples are 'aes192', etc. On recent OpenSSL releases, openssl list-cipher-algorithms will display the available cipher algorithms. Default value is aes-256-cbc
     * @property {string} input_encoding - how encryption input is encoded. Used as output for decrypting. Default utf-8.
     * @property {string} output_encoding - how encryption output is encoded. Used as input for decrypting. Default hex.
+    * @property {string} reports_key - key used for encryption of reports links
     */
     encryption: {},
 

api/parts/data/batcher.js

@@ -232,7 +232,7 @@ class WriteBatcher {
                         updateOne: {
                             filter: {_id: this.data[db][collection][key].id},
                             update: this.data[db][collection][key].value,
-                            upsert: true
+                            upsert: this.data[db][collection][key].upsert
                         }
                     });
                 }
@@ -340,18 +340,25 @@ class WriteBatcher {
      *  @param {string} id - id of the document
      *  @param {object} operation - operation
      *  @param {string} db - name of the database for which to write data
+     *  @param {object=} options - options for operation ((upsert: false) - if you don't want to upsert document)
      */
-    add(collection, id, operation, db = "countly") {
+    add(collection, id, operation, db = "countly", options) {
         if (!this.shared || cluster.isMaster) {
             if (!this.data[db][collection]) {
                 this.data[db][collection] = {};
             }
             if (!this.data[db][collection][id]) {
-                this.data[db][collection][id] = {id: id, value: operation};
+                this.data[db][collection][id] = {id: id, value: operation, upsert: true};
+                if (options && options.upsert === false) {
+                    this.data[db][collection][id].upsert = false;
+                }
                 batcherStats.update_queued++;
             }
             else {
                 this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation);
+                if (options && options.upsert === false) {
+                    this.data[db][collection][id].upsert = this.data[db][collection][id].upsert || false;
+                }
             }
             if (!this.process) {
                 this.flush(db, collection);

api/parts/data/cache.js

@@ -971,7 +971,7 @@ class StreamedCollection {
             let [col, last] = await createCollection(this.db, this.name, 1e7);
 
             this.col = col;
-            this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, noCursorTimeout: true, numberOfRetries: -1}).stream();
+            this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, numberOfRetries: -1}).stream();
 
             this.stream.on('data', doc => {
                 if (this.inserts.indexOf(doc._id.toString()) !== -1) {

api/parts/data/fetch.js

@@ -1614,7 +1614,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
         options = {};
     }
 
-    if (typeof options === "undefined") {
+    if (!options) {
         options = {};
     }
 
@@ -1704,7 +1704,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
 
         var zeroDocs = [zeroIdToFetch];
         var monthDocs = [monthIdToFetch];
-        if (!(options && options.dontBreak)) {
+        if (!options.dontBreak) {
             for (let i = 0; i < common.base64.length; i++) {
                 zeroDocs.push(zeroIdToFetch + "_" + common.base64[i]);
                 monthDocs.push(monthIdToFetch + "_" + common.base64[i]);

api/parts/jobs/job.js

@@ -1068,9 +1068,16 @@ class IPCFaçadeJob extends ResourcefulJob {
             this.ipcChannel.remove();
         }, (error) => {
             this.ipcChannel.remove();
-            log.e('[%s] Error in IPCFaçadeJob %s: %j / %j', this.job.channel, this.resourceFaçade._id, error, error.stack);
+            if (error) {
+                log.e('[%s] Error in IPCFaçadeJob %s: %j / %j', this.job.channel, this.resourceFaçade._id, error, error.stack);
+            }
+            else {
+                log.e('[%s] Error in IPCFaçadeJob %s: Unknown error', this.job.channel, this.resourceFaçade._id);
+            }
             this.job._finish(error || 'Aborted').catch(()=>{});
-            throw error;
+            if (error) {
+                throw error;
+            }
         });
     }
 

api/parts/mgmt/apps.js

@@ -613,6 +613,7 @@ appsApi.updateAppPlugins = function(params) {
 * @returns {boolean} true if operation successful
 **/
 appsApi.deleteApp = function(params) {
+    params = params || {};
     var argProps = {
             'app_id': {
                 'required': true,

api/parts/mgmt/cms.js

@@ -157,10 +157,21 @@ function syncCMSDataToDB(params) {
 }
 
 cmsApi.saveEntries = function(params) {
+
+    var entries = [];
+    try {
+        entries = JSON.parse(params.qstring.entries);
+    }
+    catch (ex) {
+        log.e(params.qstring.entries);
+        common.returnMessage(params, 400, 'Invalid entries parameter');
+        return;
+    }
+
     transformAndStoreData(
         Object.assign({dataTransformed: true}, params),
         null,
-        JSON.parse(params.qstring.entries),
+        entries,
         function(err1) {
             if (err1) {
                 log.e('An error occured while storing entries in DB: ' + err1);

api/parts/mgmt/mail.js

@@ -186,6 +186,9 @@ mail.sendToNewMember = function(member, memberPassword) {
 mail.sendToNewMemberLink = function(member, prid) {
     member.lang = member.lang || "en";
     mail.lookup(function(err, host) {
+        if (config.path) {
+            host = host + config.path;
+        }
         localize.getProperties(member.lang, function(err2, properties) {
             var message = localize.format(properties["mail.new-member-prid"], mail.escapedHTMLString(mail.getUserFirstName(member)), host, mail.escapedHTMLString(member.username), prid);
             mail.sendMessage(member.email, properties["mail.new-member-subject"], message);
@@ -217,6 +220,9 @@ mail.sendToUpdatedMember = function(member, memberPassword) {
 mail.sendPasswordResetInfo = function(member, prid) {
     member.lang = member.lang || "en";
     mail.lookup(function(err, host) {
+        if (config.path) {
+            host = host + config.path;
+        }
         localize.getProperties(member.lang, function(err2, properties) {
             var message = localize.format(properties["mail.password-reset"], mail.escapedHTMLString(mail.getUserFirstName(member)), host, prid);
             mail.sendMessage(member.email, properties["mail.password-reset-subject"], message);

api/utils/common.js

@@ -181,13 +181,14 @@ function getJSON(val) {
     }
     return ret;
 }
+
 /**
-* Logger object for creating module specific logging
-* @type {module:api/utils/log~Logger} 
-* @example
-* var log = common.log('myplugin:api');
-* log.i('myPlugin got a request: %j', params.qstring);
-*/
+ * Logger object for creating module-specific logging
+ * @type {function(string): Logger}
+ * @example
+ * const log = common.log('myplugin:api');
+ * log.i('myPlugin got a request: %j', params.qstring);
+ */
 common.log = logger;
 
 /**