Merge branch 'master' into consent-management-private-beta

Makefile

@@ -56,7 +56,7 @@ serve: package
 	@docker run -p 4000:80 segment-docs:latest
 
 catalog:
-	@node scripts/catalog_papi.js
+	@node scripts/catalog/index.js
 
 # make the list of beta connections
 .PHONY: beta
@@ -163,7 +163,7 @@ docker-build:
 
 .PHONY: private-destination
 private_destination:
-	@node scripts/private-destination.js
+	@node scripts/catalog/addPrivateDestination.js
 #.PHONY: docs
 #docs: node_modules
 #	$(BIN)/webpack --mode=production

netlify.toml

@@ -14,7 +14,7 @@
 
 [context.branch-deploy]
   command = "yarn build"
-#  ignore = "./scripts/ignore.sh"
+  # ignore = "./scripts/ignore.sh"
 
 [context.develop]
   command = "yarn develop"

scripts/catalog/addPrivateDestination.js

@@ -0,0 +1,35 @@
+const path = require('path');
+const fs = require('fs');
+const { prompt } = require('enquirer');
+const {
+    getDestinationData,
+    checkExistingStatus
+  } = require('./utilities_private_destinations.js');
+
+const addPrivateDestination = async () => {
+  let ids = await checkExistingStatus();
+  ids.sort();
+
+  const DEST_ID = await prompt({
+    type: 'input',
+    name: 'id',
+    message: 'Enter the destination ID'
+  });
+
+
+    if (ids.includes(DEST_ID.id)) {
+      console.log("This destination is already captured.");
+      return;
+    } else {
+      ids.push(DEST_ID.id);
+      fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/destinations_private.yml`), '');
+    }
+    ids.sort();
+
+    for (const element in ids) {
+      let currentId = ids[element];
+      await getDestinationData(currentId);
+    }
+};
+
+addPrivateDestination();

scripts/catalog/catalog_papi.js

@@ -0,0 +1,95 @@
+const path = require('path');
+const fs = require('fs');
+const yaml = require('js-yaml');
+const {
+  slugify,
+  getCatalog,
+  getConnectionModes,
+  isCatalogItemHidden,
+  sanitize,
+  doesCatalogItemExist
+} = require('./utilities.js');
+
+require('dotenv').config();
+
+const PAPI_URL = "https://api.segmentapis.com";
+
+const regionalSupport = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/regional-support.yml`)));
+
+// This file keeps a list of known test sources that show up in the system. 
+// Because we don't have a status value for sources, they end up showing in our catalog.
+// We use this below to prevent them from being written to yaml.
+const testSources = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/catalog/test_sources.yml`)));
+
+
+
+const updateWarehouses = async () => {
+  let warehouses = [];
+  let nextPageToken = "MA==";
+  let warehousesUpdated = [];
+
+  while (nextPageToken !== undefined) {
+    const res = await getCatalog(`${PAPI_URL}/catalog/warehouses/`, nextPageToken);
+    warehouses = warehouses.concat(res.data.warehousesCatalog);
+    nextPageToken = res.data.pagination.next;
+  }
+
+  warehouses.sort((a, b) => {
+    if (a.name.toLowerCase() < b.name.toLowerCase()) {
+      return -1;
+    }
+    if (a.name.toLowerCase() > b.name.toLowerCase()) {
+      return 1;
+    }
+    return 0;
+  });
+
+  const regionalWarehouseEndpoints = regionalSupport.warehouses.endpoint;
+  const regionalWarehouseRegions = regionalSupport.warehouses.region;
+
+  warehouses.forEach(warehouse => {
+    let slug = slugify(warehouse.slug);
+    let endpoints = ['us'];
+    let regions = ['us'];
+    let url = `connections/storage/catalog/${slug}`;
+
+    if (regionalWarehouseEndpoints.includes(slug)) {
+      endpoints.push('eu');
+    }
+
+    if (regionalWarehouseRegions.includes(slug)) {
+      regions.push('eu');
+    }
+
+    let updatedWarehouse = {
+      id: warehouse.id,
+      display_name: warehouse.name,
+      url,
+      slug,
+      endpoints,
+      regions
+    };
+
+    warehousesUpdated.push(updatedWarehouse);
+  });
+
+  const options = {
+    noArrayIndent: true
+  };
+  const todayDate = new Date().toISOString().slice(0, 10);
+
+  // Create regional support YAML file
+  let output = "# AUTOGENERATED LIST OF CONNECTIONS THAT SUPPORT REGIONAL\n";
+  output += "# Last updated " + todayDate + " \n";
+  output += yaml.dump({
+    warehouses: warehousesUpdated
+  }, options);
+  fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/regional-supported.yml`), output);
+
+  console.log("warehouses done");
+};
+
+// Execute the update functions
+updateWarehouses();
+updateSources();
+updateDestinations();

scripts/catalog/index.js

@@ -0,0 +1,17 @@
+const {updateSources} = require('./updateSources.js');
+const {updateDestinations} = require('./updateDestinations.js');
+const {updateWarehouses} = require('./updateWarehouses.js');
+const {updatePrivateDestinations} = require('./updatePrivateDestinations.js');
+
+updateSources();
+updateWarehouses();
+
+// Wait for the main catalog to update before updating the private destinations
+async function destinations() {
+    await updateDestinations()
+    updatePrivateDestinations();
+}
+
+
+
+destinations();

scripts/catalog/updateDestinations.js

@@ -0,0 +1,195 @@
+const path = require('path');
+const fs = require('fs');
+const yaml = require('js-yaml');
+const {
+  slugify,
+  getCatalog,
+  getConnectionModes,
+  isCatalogItemHidden,
+  sanitize,
+  doesCatalogItemExist
+} = require('./utilities.js');
+
+require('dotenv').config();
+
+const PAPI_URL = "https://api.segmentapis.com";
+
+
+const updateDestinations = async () => {
+    let destinations = [];
+    let destinationsUpdated = [];
+    let destinationCategories = [];
+    let categories = new Set();
+    let nextPageToken = "MA==";
+
+    // Get all destinations from the Public API
+    while (nextPageToken !== undefined) {
+      const res = await getCatalog(`${PAPI_URL}/catalog/destinations/`, nextPageToken);
+      destinations = destinations.concat(res.data.destinationsCatalog);
+      nextPageToken = res.data.pagination.next;
+    }
+
+    // Sort the destinations alphabetically
+    destinations.sort((a, b) => {
+      if (a.name.toLowerCase() < b.name.toLowerCase()) {
+        return -1;
+      }
+      if (a.name.toLowerCase() > b.name.toLowerCase()) {
+        return 1;
+      }
+      return 0;
+    });
+
+    // Loop through all destinations and create a new object with the data we want
+    destinations.forEach(destination => {
+      let endpoints = [];
+      let regions = [];
+
+      let slug = slugify(destination.name, "destinations");
+
+      if (typeof destination.supportedRegions != "undefined") {
+        regions = destination.supportedRegions;
+      } else {
+        regions.push('us-west-2', 'eu-west-1');
+      }
+
+      if (typeof destination.regionEndpoints != "undefined") {
+        endpoints = destination.regionEndpoints;
+      } else {
+        endpoints.push('US');
+      }
+
+      let url = `connections/destinations/catalog/${slug}`;
+
+      let tempCategories = [destination.categories];
+      tempCategories = tempCategories.filter(category => category != '');
+      tempCategories = tempCategories.flat();
+
+      let connection_modes = getConnectionModes({
+        components: destination.components,
+        platforms: destination.supportedPlatforms,
+        browserUnbundling: destination.supportedFeatures.browserUnbundling,
+        browserUnbundlingPublic: destination.supportedFeatures.browserUnbundlingPublic,
+        methods: destination.supportedMethods
+      });
+
+      let settings = destination.options;
+
+      settings.sort((a, b) => {
+        if (a.name.toLowerCase() < b.name.toLowerCase()) {
+          return -1;
+        }
+        if (a.name.toLowerCase() > b.name.toLowerCase()) {
+          return 1;
+        }
+        return 0;
+      });
+
+      settings.forEach(setting => {
+        setting.description = sanitize(setting.description);
+      });
+
+      let actions = destination.actions;
+      let presets = destination.presets;
+
+      const clone = (obj) => Object.assign({}, obj);
+      const renameKey = (object, key, newKey) => {
+        const clonedObj = clone(object);
+        const targetKey = clonedObj[key];
+        delete clonedObj[key];
+
+        clonedObj[newKey] = targetKey;
+        return clonedObj;
+      };
+
+      // I honestly don't remember why I did this.
+      // I think someone wanted to mention support for the Screen method to whatever destination that is
+      destination.supportedMethods.screen = false;
+      if (destination.id == '63e42b47479274407b671071') {
+        destination.supportedMethods.screen = true;
+      }
+
+      // Pageview is renamed to Page
+      destination.supportedMethods = renameKey(destination.supportedMethods, 'pageview', 'page');
+
+      // All updated destination information gets added to this object
+      let updatedDestination = {
+        id: destination.id,
+        display_name: destination.name,
+        name: destination.name,
+        slug,
+        hidden: isCatalogItemHidden(url),
+        endpoints,
+        regions,
+        url,
+        previous_names: destination.previousNames,
+        website: destination.website,
+        status: destination.status,
+        categories: tempCategories,
+        logo: {
+          url: destination.logos.default
+        },
+        mark: {
+          url: destination.logos.mark
+        },
+        methods: destination.supportedMethods,
+        platforms: destination.supportedPlatforms,
+        components: destination.components,
+        browserUnbundlingSupported: destination.supportedFeatures.browserUnbundling,
+        browserUnbundlingPublic: destination.supportedFeatures.browserUnbundlingPublic,
+        replay: destination.supportedFeatures.replay,
+        connection_modes,
+        settings,
+        actions,
+        presets
+      };
+
+      // Add the updated destination to the destinationsUpdated array
+      destinationsUpdated.push(updatedDestination);
+      doesCatalogItemExist(updatedDestination);
+      tempCategories.reduce((s, e) => s.add(e), categories);
+    });
+
+    const destinationArray = Array.from(categories);
+    destinationArray.forEach(category => {
+      destinationCategories.push({
+        display_name: category,
+        slug: slugify(category)
+      });
+      destinationCategories.sort((a, b) => {
+        if (a.display_name.toLowerCase() < b.display_name.toLowerCase()) {
+          return -1;
+        }
+        if (a.display_name.toLowerCase() > b.display_name.toLowerCase()) {
+          return 1;
+        }
+        return 0;
+      });
+    });
+
+    const options = {
+      noArrayIndent: true
+    };
+    const todayDate = new Date().toISOString().slice(0, 10);
+
+    // Create destination catalog YAML file
+    let output = "# AUTOGENERATED FROM PUBLIC API. DO NOT EDIT\n";
+    output += "# destination data last updated " + todayDate + " \n";
+    output += yaml.dump({
+      items: destinationsUpdated
+    }, options);
+    fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/destinations.yml`), output);
+
+    // Create destination-category mapping YAML file
+    output = "# AUTOGENERATED FROM PUBLIC API. DO NOT EDIT\n";
+    output += "# destination categories last updated " + todayDate + " \n";
+    output += yaml.dump({
+      items: destinationCategories
+    }, options);
+    fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/destination_categories.yml`), output);
+
+    console.log("destinations done");
+  };
+
+
+  exports.updateDestinations = updateDestinations;

scripts/catalog/updatePrivateDestinations.js

@@ -0,0 +1,17 @@
+const {
+  getDestinationData,
+  checkExistingStatus
+} = require('./utilities_private_destinations.js');
+
+const updatePrivateDestinations = async () => {
+  let ids = await checkExistingStatus();
+  ids.sort();
+
+  for (const element in ids) {
+    let currentId = ids[element];
+    await getDestinationData(currentId);
+  }
+  console.log("private destinations done");
+};
+
+exports.updatePrivateDestinations = updatePrivateDestinations;