PRO-6472: at startup, automatically supply a value for any new schema property that has a def or fallback def

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## UNRELEASED
+
+### Adds
+
+* Apostrophe now automatically adds the appropriate default values for new properties in the schema, even for existing documents in the database. This is done automatically during the migration phase of startup.
+
+### Fixes
+
+* Apostrophe's migration logic is no longer executed twice on every startup and three times in the migration task. It is executed exactly once, always at the same point in the startup process. This bug did not cause significant performance issues because migrations were only executed once, but there is a small performance improvement.
+
 ## 4.7.0 (2024-09-05)
 
 ### Adds

index.js

@@ -300,7 +300,7 @@ async function apostrophe(options, telemetry, rootSpan) {
     self.apos.schema.validateAllSchemas();
     self.apos.schema.registerAllSchemas();
     await self.apos.lock.withLock('@apostrophecms/migration:migrate', async () => {
-      await self.apos.migration.migrate(); // emits before and after events, inside the lock
+      await self.apos.migration.migrate(self.apos.argv);
       // Inserts the global doc in the default locale if it does not exist; same for other
       // singleton piece types registered by other modules
       for (const module of Object.values(self.modules)) {

lib/moog.js

@@ -170,7 +170,7 @@ module.exports = function(options) {
       }
       for (const key of Object.keys(step)) {
         if (!(validKeys.includes(key) || cascades.includes(key))) {
-          const message = upgradeHints[key] || `${key} is not a valid top level property for an Apostrophe 3.x module. Make sure you nest regular module options in the new "options" property.`;
+          const message = upgradeHints[key] || `${key} is not a valid top level property for an Apostrophe module. Make sure you nest regular module options in the "options" property.`;
           throw `${clarifyModuleName(step.__meta.name)}: ${message}`;
         }
       }

modules/@apostrophecms/migration/index.js

@@ -1,5 +1,6 @@
 const broadband = require('broadband');
 const _ = require('lodash');
+
 // Provide services for database migration. The `@apostrophecms/migration:migrate` task
 // carries out all migrations that have been registered with this module. Migrations
 // are used to make changes to the database at the time of a new code deployment,
@@ -38,15 +39,11 @@ module.exports = {
               manager.addSortifyMigration(field.name);
             });
           });
-        },
-        async executeMigrations() {
-          if ((process.env.NODE_ENV !== 'production') || self.apos.isNew) {
-            // Run migrations at dev startup (low friction).
-            // Also always run migrations at first startup, so even
-            // in prod with a brand new database the after event always fires
-            // and we get a chance to mark the migrations as skipped
-            await self.migrate(self.apos.argv);
-          }
+        }
+      },
+      before: {
+        async addMissingSchemaFields() {
+          await self.addMissingSchemaFields();
         }
       }
     };
@@ -245,8 +242,13 @@ module.exports = {
           // Just in case the db has no documents but did
           // start to run migrations on a previous attempt,
           // which causes an occasional unique key error if not
-          // corrected for here
-          await self.db.removeMany({});
+          // corrected for here.
+          //
+          // Other migration-related facts that are not migration
+          // names are stored with a leading *, leave them alone
+          await self.db.removeMany({
+            _id: /^[^*]/
+          });
           await self.db.insertMany(self.migrations.map(migration => ({
             _id: migration.name,
             at,
@@ -286,14 +288,18 @@ module.exports = {
             throw err;
           }
         }
-      }
+      },
+      ...require('./lib/addMissingSchemaFields.js')(self)
     };
   },
   tasks(self) {
     return {
       migrate: {
         usage: 'Apply any necessary migrations to the database.',
-        task: self.migrate
+        // Migrations actually run on every invocation
+        // and automatically detect whether any work
+        // must be done
+        task: () => {}
       }
     };
   }

modules/@apostrophecms/migration/lib/addMissingSchemaFields.js

@@ -0,0 +1,141 @@
+const _ = require('lodash');
+const { klona } = require('klona');
+
+module.exports = (self) => {
+  return {
+    async addMissingSchemaFields() {
+      let scans = 0; let updates = 0;
+      const lastPropLists = await self.getLastPropLists();
+      const propLists = self.getPropLists();
+      let changesToPropLists = false;
+      for (const name of Object.keys(propLists)) {
+        if (!_.isEqual(lastPropLists?.[name], propLists[name])) {
+          changesToPropLists = true;
+          scans++;
+          updates += await self.addMissingSchemaFieldsForDocType(name, propLists[name]);
+        }
+      }
+      if (changesToPropLists) {
+        await self.updateLastPropLists(propLists);
+      }
+      // Returned for tests
+      return {
+        scans,
+        updates
+      };
+    },
+    async addMissingSchemaFieldsForDocType(name) {
+      let updates = 0;
+      const schema = (self.apos.doc.managers[name] || {}).schema;
+      if (!schema) {
+        return;
+      }
+      await self.eachDoc({
+        type: name
+      }, async doc => {
+        const changes = {};
+        await self.addMissingSchemaFieldsFor(doc, schema, '', changes);
+        if (Object.keys(changes).length > 0) {
+          updates++;
+          return self.apos.doc.db.updateOne({
+            _id: doc._id
+          }, {
+            $set: changes
+          });
+        }
+      });
+      return updates;
+    },
+    // Adds changes to the object "changes" so that a single
+    // $set call can be made at the end. Use of a single
+    // object passed by reference also avoids creating many
+    // unnecessary objects in memory during a time-sensitive
+    // operation
+    addMissingSchemaFieldsFor(doc, schema, dotPath, changes) {
+      for (const field of schema) {
+        const newDotPath = dotPath ? `${dotPath}.${field.name}` : field.name;
+        // Supply the default
+        if (doc[field.name] === undefined) {
+          // Only undefined should fall back here
+          const def = klona((field.def === undefined) ? self.apos.schema.fieldTypes[field.type]?.def : field.def);
+          if (def !== undefined) {
+            if (!Object.hasOwn(changes, dotPath)) {
+              changes[newDotPath] = def;
+            }
+            // Also change it in memory so that if this is a subproperty of a
+            // new object, the change for that new object will have this
+            // subproperty too, plus we don't get crashes above when testing the
+            // subproperties' current values
+            doc[field.name] = def;
+          }
+        }
+        // Address defaults of subproperties
+        if (field.type === 'area') {
+          const basePath = `${newDotPath}.items`;
+          for (let i = 0; (i < (doc[field.name]?.items || []).length); i++) {
+            const widgetPath = `${basePath}.${i}`;
+            const widget = doc[field.name].items[i];
+            const widgetSchema = self.apos.area.getWidgetManager(widget.type)?.schema;
+            if (!widgetSchema) {
+              continue;
+            }
+            self.addMissingSchemaFieldsFor(widget, widgetSchema, widgetPath, changes);
+          }
+        } else if (field.type === 'object') {
+          self.addMissingSchemaFieldsFor(doc[field.name], field.schema, newDotPath, changes);
+        } else if (field.type === 'array') {
+          for (let i = 0; (i < (doc[field.name] || []).length); i++) {
+            const itemPath = `${newDotPath}.${i}`;
+            const item = doc[field.name][i];
+            self.addMissingSchemaFieldsFor(item, field.schema, itemPath, changes);
+          }
+        } else if (field.type === 'relationship') {
+          for (const [ key, item ] of Object.entries(doc[field.fieldsStorage] || {})) {
+            const itemPath = `${newDotPath}.${field.fieldsStorage}.${key}`;
+            self.addMissingSchemaFieldsFor(item, field.schema, itemPath, changes);
+          }
+        }
+      }
+      return changes;
+    },
+    getPropLists() {
+      const schema = {};
+      for (const [ name, module ] of Object.entries(self.apos.doc.managers)) {
+        if (!module.__meta.name) {
+          // Just a placeholder for a type present in the
+          // database but not found in the project code
+          continue;
+        }
+        schema[name] = [];
+        self.expandPropList(module.schema, schema[name], '');
+      }
+      return schema;
+    },
+    expandPropList(schema, propList, dotPath) {
+      for (const field of schema) {
+        const newDotPath = dotPath ? `${dotPath}.${field.name}` : field.name;
+        propList.push(newDotPath);
+        if (field.schema) {
+          self.expandPropList(field.schema, propList, newDotPath);
+        }
+      }
+    },
+    async getLastPropLists() {
+      const result = await self.db.findOne({
+        _id: '*lastPropLists'
+      });
+      return result?.propLists;
+    },
+    async updateLastPropLists(propLists) {
+      return self.db.updateOne({
+        _id: '*lastPropLists'
+      }, {
+        $set: {
+          propLists
+        }
+      }, {
+        upsert: true
+      });
+    }
+  };
+};