Add ability to prebundle tasks (#1602)

* add UI for task bundle id field and data submit

* add prebundle support for remote files during challenge creation

* store prebundle in lineByLineGeoJSON param as a blob

* add rebuild method for prebundling

* add support for lineByLine Geojson

* fix warnings

* add to gitignore

* fix overpass warning location

* remove console log and debugger

Author: jschwarz2030

.gitignore

@@ -38,4 +38,5 @@ yarn-error.log*
 #scala plugin
 /.bsp
 /project
-/target
+/target
+/.metals

src/components/AdminPane/HOCs/WithChallengeManagement/WithChallengeManagement.js

@@ -3,6 +3,8 @@ import _map from "lodash/map";
 import _get from "lodash/get";
 import _isObject from "lodash/isObject";
 import _compact from "lodash/compact";
+import bundleByTaskBundleId from "../../../../utils/bundleByTaskBundleId";
+import createBlob from "../../../../utils/createBlob";
 import {
   saveChallenge,
   uploadChallengeGeoJSON,
@@ -45,6 +47,95 @@ const WithChallengeManagement = (WrappedComponent) =>
     "deletingTasks"
   );
 
+/**
+ * Method used to direct remote or local data structures into line-by-line GeoJson format
+ * if a taskBundleIdProperty is provided. Any errors will be treated silently and return false
+ * to allow parent methods to continue other processes.
+ *
+ * @private
+ */
+async function rebuildPrebundle(challenge, localFile) {
+  try {
+    if (challenge.taskBundleIdProperty) {
+      if (localFile) {
+        const data = await new Response(localFile).text();
+
+        const bundled = bundleByTaskBundleId(
+          JSON.parse(data).features,
+          challenge.taskBundleIdProperty
+        );
+
+        return createBlob(bundled);
+      }
+    }
+
+    return false;
+  } catch (e) {
+    console.log(e);
+    return false;
+  }
+}
+
+/**
+ * Method used to direct remote or local data structures into line-by-line GeoJson format
+ * if a taskBundleIdProperty is provided. Any errors will be treated silently and return false
+ * to allow parent methods to continue other processes.
+ *
+ * @private
+ */
+async function convertAndBundleGeoJson(challenge) {
+  try {
+    let data = {};
+
+    if (challenge.taskBundleIdProperty) {
+      if (challenge.remoteGeoJson) {
+        data = await fetch(challenge.remoteGeoJson).then((response) =>
+          response.json()
+        );
+      } else if (challenge.lineByLineGeoJSON) {
+        const lineFile = AsLineReadableFile(challenge.lineByLineGeoJSON);
+        let allLinesRead = false;
+        let allLines = [];
+
+        while (!allLinesRead) {
+          let taskLine = await lineFile.readLines(1);
+          if (taskLine[0] === null) {
+            allLinesRead = true;
+          } else {
+            allLines.push(taskLine[0]);
+          }
+        }
+
+        data.features = [];
+
+        if (allLines?.length) {
+          allLines.forEach((taskLine) => {
+            JSON.parse(taskLine).features.forEach((feature) => {
+              data.features.push(feature);
+            });
+          });
+        }
+      } else if (challenge.localGeoJSON) {
+        data = JSON.parse(challenge.localGeoJSON);
+      }
+
+      if (data.features?.length) {
+        const bundled = bundleByTaskBundleId(
+          data.features,
+          challenge.taskBundleIdProperty
+        );
+
+        return bundled;
+      }
+    }
+
+    return false;
+  } catch (e) {
+    console.log(e);
+    return false;
+  }
+}
+
 /**
  * Upload a line-by-line GeoJSON file in chunks of 100 lines/tasks, updating
  * the task creation progress as it goes. Note that this does not signal completion
@@ -159,6 +250,17 @@ async function deleteIncompleteTasks(dispatch, ownProps, challenge) {
 
 const mapDispatchToProps = (dispatch, ownProps) => ({
   saveChallenge: async (challengeData) => {
+    const prebundled = await convertAndBundleGeoJson(challengeData);
+
+    //If the prebundler succeeds, mutate challenge data to fit line-by-line criteria
+    if (prebundled) {
+      challengeData.remoteGeoJson = undefined;
+      challengeData.localGeoJSON = undefined;
+      challengeData.overpassTargetType = null;
+      challengeData.source = "Local File";
+      challengeData.lineByLineGeoJSON = createBlob(prebundled);
+    }
+
     return dispatch(saveChallenge(challengeData))
       .then(async (challenge) => {
         // If we have line-by-line GeoJSON, we need to stream that separately
@@ -191,16 +293,20 @@ const mapDispatchToProps = (dispatch, ownProps) => ({
   rebuildChallenge: async (challenge, localFile, dataOriginDate) => {
     ownProps.updateCreatingTasksProgress(true);
 
+    const prebundle = await rebuildPrebundle(challenge, localFile);
+
     try {
       // For local files we need to figure out if it's line-by-line to
       // decide which service call to use
-      if (localFile) {
-        if (await AsValidatableGeoJSON(localFile).isLineByLine()) {
+      const fileData = prebundle || localFile;
+
+      if (fileData) {
+        if (await AsValidatableGeoJSON(fileData).isLineByLine()) {
           await uploadLineByLine(
             dispatch,
             ownProps,
             challenge,
-            localFile,
+            fileData,
             dataOriginDate
           );
         } else {

src/components/AdminPane/Manage/ManageChallenges/EditChallenge/Messages.js

@@ -587,6 +587,26 @@ will not be able to make sense of it.
       "first matching feature property from each task.",
   },
 
+  taskBundleIdPropertyLabel: {
+    id: "Admin.EditChallenge.form.taskBundlePropertyId.label",
+    defaultMessage: "Task Bundle Id Property",
+  },
+
+  taskBundleIdPropertyOverpassWarning: {
+    id: "Admin.EditChallenge.form.taskBundlePropertyId.overpassWarning",
+    defaultMessage:
+      "Currently not available for Overpass queries. Please select a different data location to use this feature.",
+  },
+
+  taskBundleIdPropertyHelp: {
+    id: "Admin.EditChallenge.form.taskBundlePropertyId.help",
+    defaultMessage:
+      "The name of the task feature property to treat as " +
+      "a bundle ID for related tasks. " +
+      "Tasks without this property will remain as isolated tasks. " +
+      "Please note that this feature currently does not work with Overpass queries.",
+  },
+
   osmIdPropertyLabel: {
     id: "Admin.EditChallenge.form.osmIdProperty.label",
     defaultMessage: "OSM/External Id Property",

src/components/AdminPane/Manage/ManageChallenges/EditChallenge/Schemas/PropertiesSchema.js

@@ -1,6 +1,6 @@
-import messages from '../Messages'
+import messages from "../Messages";
 
-const STEP_ID = "Properties"
+const STEP_ID = "Properties";
 
 /**
  * Generates a JSON Schema describing property fields of Edit Challenge
@@ -15,11 +15,21 @@ const STEP_ID = "Properties"
  *
  * @author [Neil Rotstan](https://github.com/nrotstan)
  */
-export const jsSchema = (intl, user, challengeData, extraErrors, options={}) => {
+export const jsSchema = (
+  intl,
+  user,
+  challengeData,
+  extraErrors,
+  options = {}
+) => {
   return {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    $schema: "http://json-schema.org/draft-07/schema#",
     type: "object",
     properties: {
+      taskBundleIdProperty: {
+        title: intl.formatMessage(messages.taskBundleIdPropertyLabel),
+        type: challengeData.source === "Overpass Query" ? null : "string",
+      },
       osmIdProperty: {
         title: intl.formatMessage(messages.osmIdPropertyLabel),
         type: "string",
@@ -34,8 +44,8 @@ export const jsSchema = (intl, user, challengeData, extraErrors, options={}) =>
         type: "string",
       },
     },
-  }
-}
+  };
+};
 
 /**
  * uiSchema configuration to assist react-jsonschema-form in determining
@@ -47,17 +57,38 @@ export const jsSchema = (intl, user, challengeData, extraErrors, options={}) =>
  * > the form configuration will help the RJSFFormFieldAdapter generate the
  * > proper markup
  */
-export const uiSchema = (intl, user, challengeData, extraErrors, options={}) => {
-  const isCollapsed = options.longForm && (options.collapsedGroups || []).indexOf(STEP_ID) !== -1
-  const toggleCollapsed = options.longForm && options.toggleCollapsed ? () => options.toggleCollapsed(STEP_ID) : undefined
+export const uiSchema = (
+  intl,
+  user,
+  challengeData,
+  extraErrors,
+  options = {}
+) => {
+  const isCollapsed =
+    options.longForm && (options.collapsedGroups || []).indexOf(STEP_ID) !== -1;
+  const toggleCollapsed =
+    options.longForm && options.toggleCollapsed
+      ? () => options.toggleCollapsed(STEP_ID)
+      : undefined;
 
   return {
+    taskBundleIdProperty: {
+      "ui:emptyValue": "",
+      "ui:help": intl.formatMessage(messages.taskBundleIdPropertyHelp),
+      "ui:collapsed": isCollapsed,
+      "ui:toggleCollapsed": toggleCollapsed,
+      "ui:groupHeader": options.longForm
+        ? intl.formatMessage(messages.propertiesStepHeader)
+        : undefined,
+      "ui:description":
+        challengeData.source === "Overpass Query"
+          ? intl.formatMessage(messages.taskBundleIdPropertyOverpassWarning)
+          : null,
+    },
     osmIdProperty: {
       "ui:emptyValue": "",
       "ui:help": intl.formatMessage(messages.osmIdPropertyDescription),
       "ui:collapsed": isCollapsed,
-      "ui:toggleCollapsed": toggleCollapsed,
-      "ui:groupHeader": options.longForm ? intl.formatMessage(messages.propertiesStepHeader) : undefined,
     },
     customTaskStyles: {
       "ui:field": "configureCustomTaskStyles",
@@ -69,5 +100,5 @@ export const uiSchema = (intl, user, challengeData, extraErrors, options={}) =>
       "ui:help": intl.formatMessage(messages.exportablePropertiesDescription),
       "ui:collapsed": isCollapsed,
     },
-  }
-}
+  };
+};

src/lang/en-US.json

@@ -219,6 +219,8 @@
   "Admin.EditChallenge.form.steps.yes.label": "Yes",
   "Admin.EditChallenge.form.steps.zoom.description": "Configure map zoom levels",
   "Admin.EditChallenge.form.steps.zoom.header": "Zoom Levels",
+  "Admin.EditChallenge.form.taskBundlePropertyId.description": "The name of the task feature property to treat as a bundle ID for related tasks. Tasks without this property will remain as isolated tasks. [Learn more](https://learn.maproulette.org/documentation/setting-task-bundle-identifiers/).",
+  "Admin.EditChallenge.form.taskBundlePropertyId.label": "Task Bundle Id Property",
   "Admin.EditChallenge.form.taskPropertyStyles.clear": "Clear",
   "Admin.EditChallenge.form.taskPropertyStyles.close": "Done",
   "Admin.EditChallenge.form.taskPropertyStyles.description": "Sets up task property style rules......",

src/utils/bundleByTaskBundleId.js

@@ -0,0 +1,67 @@
+const bundleByTaskBundleId = (tasks, externalId) => {
+  const bundled = [];
+  let leftoverTasks = tasks;
+
+  while (leftoverTasks.length) {
+    const task = leftoverTasks[0];
+    const id = task.properties[externalId];
+    const features = [];
+    let infiniteLoopCount = 0;
+    let prevLength = leftoverTasks.length;
+    let bundledTask;
+
+    if (id === undefined) {
+      bundledTask = {
+        type: "FeatureCollection",
+        features: [task],
+      };
+      bundled.push(JSON.stringify(bundledTask));
+      leftoverTasks.shift();
+    } else {
+      const matchingTasks = leftoverTasks.filter((t) => {
+        if (t.properties[externalId] === id) {
+          return true;
+        }
+
+        return false;
+      });
+
+      for (let j = 0; j < matchingTasks.length; j++) {
+        features.push(matchingTasks[j]);
+      }
+
+      bundledTask = {
+        type: "FeatureCollection",
+        features,
+      };
+
+      const nonMatchingTasks = leftoverTasks.filter((t) => {
+        if (t.properties[externalId] !== id) {
+          return true;
+        }
+
+        return false;
+      });
+
+      bundled.push(JSON.stringify(bundledTask));
+      leftoverTasks = nonMatchingTasks;
+    }
+
+    if (leftoverTasks.length === prevLength) {
+      infiniteLoopCount++;
+
+      if (infiniteLoopCount > 10) {
+        console.log(
+          "There was a problem with your data that caused an infinite loop.  Process stopped"
+        );
+        break;
+      }
+    } else {
+      infiniteLoopCount = 0;
+    }
+  }
+
+  return bundled.join("\n");
+};
+
+export default bundleByTaskBundleId;

src/utils/createBlob.js

@@ -0,0 +1,7 @@
+const createBlob = (jsonData) => {
+  const blob = new Blob([jsonData], { type: "application/json" });
+
+  return blob;
+};
+
+export default createBlob;