Merge branch 'develop' into feat.sources-v2-spec-controller

.github/workflows/build-push-docker-image.yml

@@ -98,7 +98,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       - name: Login to DockerHub
         uses: docker/[email protected]
@@ -107,7 +107,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -124,7 +124,7 @@ jobs:
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -151,7 +151,7 @@ jobs:
           fetch-depth: 1
 
       - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       - name: Login to DockerHub
         uses: docker/[email protected]
@@ -160,7 +160,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -177,7 +177,7 @@ jobs:
           docker run ${{ inputs.build_tag }} npm run test:ts:ci
 
       - name: Build and Push Multi-platform Images
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -199,7 +199,7 @@ jobs:
 
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       - name: Login to DockerHub
         uses: docker/[email protected]

.github/workflows/dt-test-and-report-code-coverage.yml

@@ -58,14 +58,14 @@ jobs:
           npm run lint:fix
 
       - name: Upload Coverage Reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.6.0
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           directory: ./reports/coverage
 
       - name: Upload TS Coverage Reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.6.0
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:

CHANGELOG.md

@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.83.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.82.2...v1.83.0) (2024-10-25)
+
+
+### Features
+
+* add support for identity stitching for shopify pixel flow ([#3818](https://github.com/rudderlabs/rudder-transformer/issues/3818)) ([3a09181](https://github.com/rudderlabs/rudder-transformer/commit/3a091810bff12ae893c16b514c07d17e6374062a))
+* onboard tune destination ([#3795](https://github.com/rudderlabs/rudder-transformer/issues/3795)) ([741f0c6](https://github.com/rudderlabs/rudder-transformer/commit/741f0c6d6714cf760ce98cc9354b61f7b5ce4684))
+* snowpipe streaming ([#3740](https://github.com/rudderlabs/rudder-transformer/issues/3740)) ([21b1039](https://github.com/rudderlabs/rudder-transformer/commit/21b1039c26067b4896a15adf8b10a044e69cb495))
+* support for multiple zap urls ([#3825](https://github.com/rudderlabs/rudder-transformer/issues/3825)) ([f79dfe7](https://github.com/rudderlabs/rudder-transformer/commit/f79dfe74b447b8ccd80287e143ff771688c510be))
+
+
+### Bug Fixes
+
+* add missing field for pinterest_tag single product events ([f781a84](https://github.com/rudderlabs/rudder-transformer/commit/f781a84ade98649d68cebf4da13c2ceff8df2df2))
+* add missing field for pinterest_tag single product events ([#3826](https://github.com/rudderlabs/rudder-transformer/issues/3826)) ([4a63277](https://github.com/rudderlabs/rudder-transformer/commit/4a63277efd0b4357d8321618640d1a0ba2a47d71))
+* heap userId extraction ([#3801](https://github.com/rudderlabs/rudder-transformer/issues/3801)) ([e578413](https://github.com/rudderlabs/rudder-transformer/commit/e57841396ad666d716e195fbd4e9b74a63bf5191))
+* not allowing empty string or null values for mandatory fields in zoho ([#3800](https://github.com/rudderlabs/rudder-transformer/issues/3800)) ([fcd8d99](https://github.com/rudderlabs/rudder-transformer/commit/fcd8d997fe815d61d21ffff235b0799e69b7ded9))
+* populate source destination info env set properly ([#3806](https://github.com/rudderlabs/rudder-transformer/issues/3806)) ([d730daf](https://github.com/rudderlabs/rudder-transformer/commit/d730dafbbd2de30b67c35db8ca05396a98a8d2e0))
+* str replace is not a function error ([#3799](https://github.com/rudderlabs/rudder-transformer/issues/3799)) ([8f18e1a](https://github.com/rudderlabs/rudder-transformer/commit/8f18e1aca70ab68e3f157a4632d63ae7cec0e87b))
+* update order_id in checkout events, messageId in pixel events ([#3794](https://github.com/rudderlabs/rudder-transformer/issues/3794)) ([427be71](https://github.com/rudderlabs/rudder-transformer/commit/427be71a91df8495f81b42d2b58aa490db439b23))
+
 ### [1.82.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.82.1...v1.82.2) (2024-10-18)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.82.2",
+  "version": "1.83.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml

@@ -167,7 +167,11 @@ steps:
               "content_ids": (props.product_id ?? props.sku ?? props.id)[],
               "contents": {
                 "quantity": Number(props.quantity) || 1,
-                "item_price": String(props.price)
+                "item_price": String(props.price),
+                "item_name": String(props.name),
+                "id": props.product_id ?? props.sku,
+                "item_category": props.category,
+                "item_brand": props.brand
               }[]
             };
       - name: combineAllEcomFields

src/cdk/v2/destinations/zapier/procWorkflow.yaml

@@ -1,10 +1,9 @@
 bindings:
   - name: EventType
     path: ../../../../constants
-  - name: getHashFromArray
+  - name: getHashFromArrayWithDuplicate
     path: ../../../../v0/util/index
-  - name: defaultRequestConfig
-    path: ../../../../v0/util
+  - path: ./utils
 
 steps:
   - name: validateInput
@@ -15,18 +14,18 @@ steps:
   - name: prepareContext
     template: |
       $.context.messageType = .message.type.toLowerCase();
-      $.context.endpoint = .destination.Config.zapUrl;
+      $.context.endpoint = [.destination.Config.zapUrl];
   - name: trackEndpoint
     condition: $.context.messageType === {{$.EventType.TRACK}}
     template: |
-      const trackEventsMap = $.getHashFromArray(.destination.Config.trackEventsToZap);
+      const trackEventsMap = $.getHashFromArrayWithDuplicate(.destination.Config.trackEventsToZap);
       const eventName = .message.event.toLowerCase();
       (eventName && trackEventsMap[eventName]) ? 
         ($.context.endpoint = trackEventsMap[eventName])
     else:
       name: endpointForOthers
       template: |
-        const pageScreenEventsMap = $.getHashFromArray(.destination.Config.pageScreenEventsToZap);
+        const pageScreenEventsMap = $.getHashFromArrayWithDuplicate(.destination.Config.pageScreenEventsToZap);
         const pageName = .message.name.toLowerCase();
         (pageName && pageScreenEventsMap[pageName]) ? 
           ($.context.endpoint = pageScreenEventsMap[pageName])
@@ -37,10 +36,5 @@ steps:
     else:
       name: buildResponseForProcessTransformation
       template: |
-        const response = $.defaultRequestConfig();
-        response.body.JSON = .message;
-        response.endpoint = $.context.endpoint;
-        response.headers = {
-          "content-type": "application/json"
-        };
-        response
+        const responseList = $.buildResponseList(.message, $.context.endpoint)
+        responseList

src/cdk/v2/destinations/zapier/utils.js

@@ -0,0 +1,15 @@
+const { defaultRequestConfig } = require('../../../../v0/util');
+
+const buildResponseList = (payload, endpointList) => {
+  const responseList = [];
+  endpointList.forEach((endpoint) => {
+    const response = defaultRequestConfig();
+    response.body.JSON = payload;
+    response.endpoint = endpoint;
+    response.headers = { 'content-type': 'application/json' };
+    responseList.push(response);
+  });
+  return responseList;
+};
+
+module.exports = { buildResponseList };

src/util/prometheus.js

@@ -439,6 +439,24 @@ class Prometheus {
         type: 'counter',
         labelNames: ['writeKey', 'source', 'shopifyTopic'],
       },
+      {
+        name: 'shopify_pixel_cart_token_not_found',
+        help: 'shopify_pixel_cart_token_not_found',
+        type: 'counter',
+        labelNames: ['event', 'writeKey'],
+      },
+      {
+        name: 'shopify_pixel_cart_token_set',
+        help: 'shopify_pixel_cart_token_set',
+        type: 'counter',
+        labelNames: ['event', 'writeKey'],
+      },
+      {
+        name: 'shopify_pixel_cart_token_redis_error',
+        help: 'shopify_pixel_cart_token_redis_error',
+        type: 'counter',
+        labelNames: ['event', 'writeKey'],
+      },
       {
         name: 'outgoing_request_count',
         help: 'Outgoing HTTP requests count',

src/v0/destinations/ga4_v2/networkHandler.js

@@ -0,0 +1 @@
+export { networkHandler } from '../ga4/networkHandler';

src/v0/destinations/google_adwords_enhanced_conversions/transform.js

@@ -1,8 +1,9 @@
 /* eslint-disable no-param-reassign */
 
 const get = require('get-value');
-const { cloneDeep } = require('lodash');
+const { cloneDeep, isNumber } = require('lodash');
 const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib');
+const isString = require('lodash/isString');
 const {
   constructPayload,
   defaultRequestConfig,
@@ -35,7 +36,18 @@ const updateMappingJson = (mapping) => {
 const responseBuilder = async (metadata, message, { Config }, payload) => {
   const response = defaultRequestConfig();
   const { event } = message;
-  const filteredCustomerId = removeHyphens(Config.customerId);
+  const { subAccount } = Config;
+  let { customerId, loginCustomerId } = Config;
+  if (isNumber(customerId)) {
+    customerId = customerId.toString();
+  }
+  if (isNumber(loginCustomerId)) {
+    loginCustomerId = loginCustomerId.toString();
+  }
+  if (!isString(customerId) || !isString(loginCustomerId)) {
+    throw new InstrumentationError('customerId and loginCustomerId should be a string or number');
+  }
+  const filteredCustomerId = removeHyphens(customerId);
   response.endpoint = `${BASE_ENDPOINT}/${filteredCustomerId}:uploadConversionAdjustments`;
   response.body.JSON = payload;
   const accessToken = getAccessToken(metadata, 'access_token');
@@ -45,9 +57,9 @@ const responseBuilder = async (metadata, message, { Config }, payload) => {
     'developer-token': getValueFromMessage(metadata, 'secret.developer_token'),
   };
   response.params = { event, customerId: filteredCustomerId };
-  if (Config.subAccount)
-    if (Config.loginCustomerId) {
-      const filteredLoginCustomerId = removeHyphens(Config.loginCustomerId);
+  if (subAccount)
+    if (loginCustomerId) {
+      const filteredLoginCustomerId = removeHyphens(loginCustomerId);
       response.headers['login-customer-id'] = filteredLoginCustomerId;
     } else throw new ConfigurationError(`LoginCustomerId is required as subAccount is true.`);
 

src/v0/destinations/snowpipe_streaming/transform.js

@@ -0,0 +1 @@
+module.exports = require('../snowflake/transform');

src/v0/destinations/tune/transform.js

@@ -0,0 +1,94 @@
+const get = require('get-value');
+const { InstrumentationError } = require('@rudderstack/integrations-lib');
+const {
+  defaultRequestConfig,
+  simpleProcessRouterDest,
+  getHashFromArray,
+  isDefinedAndNotNull,
+  isNotEmpty,
+} = require('../../util');
+
+const getTuneEndpoint = (subdomain) => `https://${subdomain}.go2cloud.org/aff_l`;
+
+const mapPropertiesWithNestedSupport = (msg, properties, mappings) => {
+  const mappedObj = {}; // Create a new object for parameters
+  Object.entries(mappings).forEach(([key, value]) => {
+    const keyStr = `${key}`;
+    const args = { object: properties, key: keyStr };
+    if (args.key.split('.').length > 1) {
+      // Handle nested keys
+      args.object = msg; // This line modifies the object property of args
+    }
+    const data = get(args.object, args.key);
+    if (isDefinedAndNotNull(data) && isNotEmpty(data)) {
+      mappedObj[value] = data; // Map to the corresponding destination key
+    }
+  });
+  return mappedObj; // Return the new params object
+};
+
+const responseBuilder = (message, { Config }) => {
+  const { tuneEvents, subdomain } = Config; // Extract tuneEvents from config
+  const { properties, event: messageEvent } = message; // Destructure properties and event from message
+
+  // Find the relevant tune event based on the message's event name
+  const tuneEvent = tuneEvents.find((event) => event.eventName === messageEvent);
+
+  if (tuneEvent) {
+    const standardHashMap = getHashFromArray(tuneEvent.standardMapping, 'from', 'to', false);
+    const advSubIdHashMap = getHashFromArray(tuneEvent.advSubIdMapping, 'from', 'to', false);
+    const advUniqueIdHashMap = getHashFromArray(tuneEvent.advUniqueIdMapping, 'from', 'to', false);
+
+    const params = {
+      ...mapPropertiesWithNestedSupport(message, properties, standardHashMap),
+      ...mapPropertiesWithNestedSupport(message, properties, advSubIdHashMap),
+      ...mapPropertiesWithNestedSupport(message, properties, advUniqueIdHashMap),
+    };
+
+    const endpoint = getTuneEndpoint(subdomain);
+
+    // Prepare the response
+    const response = defaultRequestConfig();
+    response.params = params; // Set only the mapped params
+    response.endpoint = endpoint;
+
+    return response;
+  }
+
+  throw new InstrumentationError('No matching tune event found for the provided event.', 400);
+};
+
+const processEvent = (message, destination) => {
+  // Validate message type
+  if (!isDefinedAndNotNull(message.type) || typeof message.type !== 'string') {
+    throw new InstrumentationError(
+      'Message Type is not present or is not a string. Aborting message.',
+      400,
+    );
+  }
+  const messageType = message.type.toLowerCase();
+
+  // Initialize response variable
+  let response;
+
+  // Process 'track' messages using the responseBuilder
+  if (messageType === 'track') {
+    response = responseBuilder(message, destination);
+  } else {
+    throw new InstrumentationError('Message type not supported. Only "track" is allowed.', 400);
+  }
+
+  return response;
+};
+
+const process = (event) => processEvent(event.message, event.destination);
+
+const processRouterDest = async (inputs, reqMetadata) => {
+  const respList = await simpleProcessRouterDest(inputs, process, reqMetadata);
+  return respList;
+};
+
+module.exports = {
+  process,
+  processRouterDest,
+};

src/v0/sources/shopify/util.js

@@ -124,11 +124,11 @@ const extractEmailFromPayload = (event) => {
 };
 
 const getCartToken = (message) => {
-  const { event, properties } = message;
+  const { event, properties, context } = message;
   if (event === SHOPIFY_TRACK_MAP.carts_update) {
     return properties?.id || properties?.token;
   }
-  return properties?.cart_token || null;
+  return properties?.cart_token || context?.cart_token || null;
 };
 
 /**