From a5c8728227bb2bfc96f4eaefa5042eb7a6aeb6ba Mon Sep 17 00:00:00 2001 From: David Hochbaum Date: Fri, 25 Oct 2024 17:19:53 -0400 Subject: [PATCH 1/6] Initial test send works, need to transform data, and fix template issues in gmail. --- server/package.json | 1 + server/src/subscriber/subscriber.module.ts | 3 +- server/src/subscriber/subscriber.service.ts | 41 ++++++++++++++++++++- server/yarn.lock | 17 ++++++++- 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/server/package.json b/server/package.json index 65e4a1a9..f080c4e0 100644 --- a/server/package.json +++ b/server/package.json @@ -38,6 +38,7 @@ "@sentry/nestjs": "^8.35.0", "@sentry/profiling-node": "^8.35.0", "@sentry/wizard": "^3.34.2", + "@sendgrid/mail": "^8.1.4", "@turf/bbox": "^6.0.1", "@turf/buffer": "^5.1.5", "@turf/helpers": "^6.1.4", diff --git a/server/src/subscriber/subscriber.module.ts b/server/src/subscriber/subscriber.module.ts index 4f9b1956..3b8e69e2 100644 --- a/server/src/subscriber/subscriber.module.ts +++ b/server/src/subscriber/subscriber.module.ts @@ -3,10 +3,11 @@ import { SubscriberController } from "./subscriber.controller"; import { SubscriberService } from "./subscriber.service"; import { ConfigModule } from "../config/config.module"; import { Client } from "@sendgrid/client"; +import { MailService } from "@sendgrid/mail"; @Module({ imports: [ConfigModule], - providers: [SubscriberService, Client], + providers: [SubscriberService, Client, MailService], exports: [SubscriberService], controllers: [SubscriberController] }) diff --git a/server/src/subscriber/subscriber.service.ts b/server/src/subscriber/subscriber.service.ts index 5ab0584f..d1aed569 100644 --- a/server/src/subscriber/subscriber.service.ts +++ b/server/src/subscriber/subscriber.service.ts @@ -1,6 +1,7 @@ import { Injectable, Res } from "@nestjs/common"; import { ConfigService } from "../config/config.service"; import { Client } from "@sendgrid/client"; +import { MailService } from "@sendgrid/mail"; import crypto from 'crypto'; import * as Sentry from "@sentry/nestjs"; @@ -26,10 +27,12 @@ export class SubscriberService { sendgridEnvironmentIdVariable = ""; constructor( private readonly config: ConfigService, - private client: Client + private client: Client, + private mailer: MailService ) { this.client.setApiKey(this.config.get("SENDGRID_API_KEY")); this.sendgridEnvironmentIdVariable = `zap_${this.config.get("SENDGRID_ENVIRONMENT")}_id`; + this.mailer.setApiKey(this.config.get("SENDGRID_API_KEY")); } /** @@ -79,6 +82,42 @@ export class SubscriberService { }] } } +// https://github.com/sendgrid/sendgrid-nodejs/blob/main/docs/use-cases/transactional-templates.md + const msg = { + to: email, // Change to your recipient + from: 'do-not-reply@planning.nyc.gov', // Change to your verified sender + templateId: 'd-3684647ef2b242d8947b65b20497baa0', + dynamicTemplateData: { + "subscriptions": { + "citywide": true, + "boroughs": [ + { + "name": "Brooklyn", + "communityBoards": [1, 2, 3] + }, + { + "name": "Queens", + "communityBoards": [4] + }, + { + "name": "Manhattan", + "communityBoards": [3, 11] + } + ] + } + } + // subject: 'Sending with SendGrid is Fun', + // text: 'and easy to do anywhere, even with Node.js', + // html: 'and easy to do anywhere, even with Node.js', + } + this.mailer.send(msg) + .then((response) => { + console.log(response[0].statusCode) + console.log(response[0].headers) + }) + .catch((error) => { + console.error(error) + }) // If successful, this will add the request to the queue and return a 202 // https://www.twilio.com/docs/sendgrid/api-reference/contacts/add-or-update-a-contact diff --git a/server/yarn.lock b/server/yarn.lock index e05d889e..b9bd68bb 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -1265,6 +1265,14 @@ "@sendgrid/helpers" "^8.0.0" axios "^1.6.8" +"@sendgrid/client@^8.1.4": + version "8.1.4" + resolved "https://registry.yarnpkg.com/@sendgrid/client/-/client-8.1.4.tgz#4db39e49d8ed732169d73b5d5c94d2b11907970d" + integrity sha512-VxZoQ82MpxmjSXLR3ZAE2OWxvQIW2k2G24UeRPr/SYX8HqWLV/8UBN15T2WmjjnEb5XSmFImTJOKDzzSeKr9YQ== + dependencies: + "@sendgrid/helpers" "^8.0.0" + axios "^1.7.4" + "@sendgrid/helpers@^8.0.0": version "8.0.0" resolved "https://registry.yarnpkg.com/@sendgrid/helpers/-/helpers-8.0.0.tgz#f74bf9743bacafe4c8573be46166130c604c0fc1" @@ -1449,6 +1457,13 @@ xcode "3.0.1" xml-js "^1.6.11" yargs "^16.2.0" +"@sendgrid/mail@^8.1.4": + version "8.1.4" + resolved "https://registry.yarnpkg.com/@sendgrid/mail/-/mail-8.1.4.tgz#0ba72906685eae1a1ef990cca31e962f1ece6928" + integrity sha512-MUpIZykD9ARie8LElYCqbcBhGGMaA/E6I7fEcG7Hc2An26QJyLtwOaKQ3taGp8xO8BICPJrSKuYV4bDeAJKFGQ== + dependencies: + "@sendgrid/client" "^8.1.4" + "@sendgrid/helpers" "^8.0.0" "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -3666,7 +3681,7 @@ axios@^0.21.1: dependencies: follow-redirects "^1.10.0" -axios@^1.6.8: +axios@^1.6.8, axios@^1.7.4: version "1.7.7" resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== From d5e599aceac7152d4365dbdb9f234a78173e551d Mon Sep 17 00:00:00 2001 From: David Hochbaum Date: Tue, 29 Oct 2024 17:49:53 -0400 Subject: [PATCH 2/6] Sending email after signup success --- .../src/subscriber/subscriber.controller.ts | 7 +- server/src/subscriber/subscriber.service.ts | 72 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/server/src/subscriber/subscriber.controller.ts b/server/src/subscriber/subscriber.controller.ts index bdfa3e05..2b3cf088 100644 --- a/server/src/subscriber/subscriber.controller.ts +++ b/server/src/subscriber/subscriber.controller.ts @@ -4,6 +4,7 @@ import { SubscriberService } from "./subscriber.service"; import { Request } from "express"; import validateEmail from "../_utils/validate-email"; import * as Sentry from "@sentry/nestjs"; +import crypto from 'crypto'; const PAUSE_BETWEEN_CHECKS = 30000; const CHECKS_BEFORE_FAIL = 10; @@ -59,7 +60,8 @@ export class SubscriberController { } // If we have reached this point, the user either doesn't exist or isn't signed up for the list - const addToQueue = await this.subscriberService.create(request.body.email, this.list, this.sendgridEnvironment, request.body.subscriptions, response) + const id = crypto.randomUUID(); + const addToQueue = await this.subscriberService.create(request.body.email, this.list, this.sendgridEnvironment, request.body.subscriptions, id, response) if(addToQueue.isError) { response.status(addToQueue.code).send({errors: addToQueue.response.body.errors}) @@ -78,6 +80,9 @@ export class SubscriberController { // Now we keep checking to make sure the import was successful const importConfirmation = await this.subscriberService.checkCreate(addToQueue.result[1]["job_id"], response, 0, CHECKS_BEFORE_FAIL, PAUSE_BETWEEN_CHECKS, errorInfo); + + // Send the confirmation email + await this.subscriberService.sendConfirmationEmail(request.body.email, this.sendgridEnvironment, request.body.subscriptions, id) return; diff --git a/server/src/subscriber/subscriber.service.ts b/server/src/subscriber/subscriber.service.ts index d1aed569..96277959 100644 --- a/server/src/subscriber/subscriber.service.ts +++ b/server/src/subscriber/subscriber.service.ts @@ -64,10 +64,10 @@ export class SubscriberService { * @param {string} list - The email list to which we will add the user * @param {string} environment - Staging or production * @param {object} subscriptions - The CDs the user is subscribing to + * @param {string} id - The id needed for confirmation * @returns {object} */ - async create(email: string, list: string, environment: string, subscriptions: object, @Res() response) { - const id = crypto.randomUUID(); + async create(email: string, list: string, environment: string, subscriptions: object, id: string, @Res() response) { var custom_fields = Object.entries(subscriptions).reduce((acc, curr) => ({...acc, [`zap_${environment}_${curr[0]}`]: curr[1]}), {[`zap_${environment}_confirmed`]: 0}) custom_fields[this.sendgridEnvironmentIdVariable] = id; @@ -82,42 +82,6 @@ export class SubscriberService { }] } } -// https://github.com/sendgrid/sendgrid-nodejs/blob/main/docs/use-cases/transactional-templates.md - const msg = { - to: email, // Change to your recipient - from: 'do-not-reply@planning.nyc.gov', // Change to your verified sender - templateId: 'd-3684647ef2b242d8947b65b20497baa0', - dynamicTemplateData: { - "subscriptions": { - "citywide": true, - "boroughs": [ - { - "name": "Brooklyn", - "communityBoards": [1, 2, 3] - }, - { - "name": "Queens", - "communityBoards": [4] - }, - { - "name": "Manhattan", - "communityBoards": [3, 11] - } - ] - } - } - // subject: 'Sending with SendGrid is Fun', - // text: 'and easy to do anywhere, even with Node.js', - // html: 'and easy to do anywhere, even with Node.js', - } - this.mailer.send(msg) - .then((response) => { - console.log(response[0].statusCode) - console.log(response[0].headers) - }) - .catch((error) => { - console.error(error) - }) // If successful, this will add the request to the queue and return a 202 // https://www.twilio.com/docs/sendgrid/api-reference/contacts/add-or-update-a-contact @@ -184,6 +148,37 @@ export class SubscriberService { } } + /** + * Send the user an email requesting signup confirmation. + * @param {string} email - The user's email address + * @param {string} environment - Staging or production + * @param {object} subscriptions - The CDs the user is subscribing to + * @param {string} id - The id needed for confirmation + * @returns {object} + */ + async sendConfirmationEmail(email: string, environment: string, subscriptions: object, id: string) { + // https://github.com/sendgrid/sendgrid-nodejs/blob/main/docs/use-cases/transactional-templates.md + const msg = { + to: email, + from: 'do-not-reply@planning.nyc.gov', // Your verified sender + templateId: 'd-3684647ef2b242d8947b65b20497baa0', + dynamicTemplateData: { + "id": id, + "subscriptions": this.convertSubscriptionsToHandlebars(subscriptions) + } + } + this.mailer.send(msg) + .then((response) => { + // console.log(response[0].statusCode) + // console.log(response[0].headers) + return {isError: false, statusCode: response[0].statusCode} + }) + .catch((error) => { + console.error(error) + return {isError: true, ...error} + }) + } + /** * Validate a list of subscriptions. * @param {object} subscriptions - The subscriptions to validate. @@ -222,5 +217,4 @@ export class SubscriberService { return validCustomFieldValues.includes(value as CustomFieldValue); } - } From fcbb68a16f0c15d3718bf28b306d34c5a1fdff9b Mon Sep 17 00:00:00 2001 From: David Hochbaum Date: Wed, 30 Oct 2024 16:40:21 -0400 Subject: [PATCH 3/6] Updated domain in email link --- server/src/subscriber/subscriber.service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/subscriber/subscriber.service.ts b/server/src/subscriber/subscriber.service.ts index 96277959..cef4304f 100644 --- a/server/src/subscriber/subscriber.service.ts +++ b/server/src/subscriber/subscriber.service.ts @@ -164,13 +164,12 @@ export class SubscriberService { templateId: 'd-3684647ef2b242d8947b65b20497baa0', dynamicTemplateData: { "id": id, + "domain": environment === "production" ? "zap.planning.nyc.gov" : "zap-staging.planninglabs.nyc", "subscriptions": this.convertSubscriptionsToHandlebars(subscriptions) } } this.mailer.send(msg) .then((response) => { - // console.log(response[0].statusCode) - // console.log(response[0].headers) return {isError: false, statusCode: response[0].statusCode} }) .catch((error) => { From ab766759abf0bb86512d78408371c92b3a2a1931 Mon Sep 17 00:00:00 2001 From: David Hochbaum Date: Thu, 31 Oct 2024 10:55:57 -0400 Subject: [PATCH 4/6] Dealing with issues from merge. --- .../src/subscriber/subscriber.controller.ts | 6 ++-- server/src/subscriber/subscriber.service.ts | 35 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/server/src/subscriber/subscriber.controller.ts b/server/src/subscriber/subscriber.controller.ts index 2b3cf088..edf5a713 100644 --- a/server/src/subscriber/subscriber.controller.ts +++ b/server/src/subscriber/subscriber.controller.ts @@ -82,8 +82,10 @@ export class SubscriberController { const importConfirmation = await this.subscriberService.checkCreate(addToQueue.result[1]["job_id"], response, 0, CHECKS_BEFORE_FAIL, PAUSE_BETWEEN_CHECKS, errorInfo); // Send the confirmation email - await this.subscriberService.sendConfirmationEmail(request.body.email, this.sendgridEnvironment, request.body.subscriptions, id) - + if (!importConfirmation.isError) { + await this.subscriberService.sendConfirmationEmail(request.body.email, this.sendgridEnvironment, request.body.subscriptions, id); + } + return; } diff --git a/server/src/subscriber/subscriber.service.ts b/server/src/subscriber/subscriber.service.ts index cef4304f..c4959c7c 100644 --- a/server/src/subscriber/subscriber.service.ts +++ b/server/src/subscriber/subscriber.service.ts @@ -126,7 +126,6 @@ export class SubscriberService { const confirmationRequest = { url: `/v3/marketing/contacts/imports/${importId}`, - // method: 'GET', method: 'GET', } @@ -216,4 +215,38 @@ export class SubscriberService { return validCustomFieldValues.includes(value as CustomFieldValue); } + /** + * Convert the uploaded subscriptions object into a format for Handlebars to use in the confirmation email + * @param {object} subscriptions - The set of CDs the user is subscribing to + * @returns {boolean} + */ + private convertSubscriptionsToHandlebars(subscriptions: object) { + var handlebars = { "citywide": false, "boroughs": [] } + const boros = { + "K": "Brooklyn", + "X": "Bronx", + "M": "Manhattan", + "Q": "Queens", + "R": "Staten Island" + } + for (const [key, value] of Object.entries(subscriptions)) { + if (value === 1) { + if (key === "CW") { + handlebars.citywide = true; + } else if (boros[key[0]]) { + const i = handlebars.boroughs.findIndex((boro) => boro.name === boros[key[0]]); + if (i === -1) { + handlebars.boroughs.push({ + "name": boros[key[0]], + "communityBoards": [parseInt(key.slice(-2))] + }) + } else { + handlebars.boroughs[i]["communityBoards"].push(parseInt(key.slice(-2))) + } + } + } + } + return handlebars; + } + } From 27f8a8d4e1dc5ef75e450abdff57c3e263a56c06 Mon Sep 17 00:00:00 2001 From: David Hochbaum Date: Wed, 6 Nov 2024 15:17:55 -0500 Subject: [PATCH 5/6] Try to make a type for this subscriptions argument that is stricter than just object. --- server/src/subscriber/subscriber.service.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/subscriber/subscriber.service.ts b/server/src/subscriber/subscriber.service.ts index c4959c7c..17f928fe 100644 --- a/server/src/subscriber/subscriber.service.ts +++ b/server/src/subscriber/subscriber.service.ts @@ -13,6 +13,8 @@ const validCustomFieldValues = [1] as const; export type CustomFieldValueTuple = typeof validCustomFieldValues; type CustomFieldValue = CustomFieldValueTuple[number]; +type ValidSubscriptionSet = {"K01": CustomFieldValue; "K02": CustomFieldValue; "K03": CustomFieldValue; "K04": CustomFieldValue; "K05": CustomFieldValue; "K06": CustomFieldValue; "K07": CustomFieldValue; "K08": CustomFieldValue; "K09": CustomFieldValue; "K10": CustomFieldValue; "K11": CustomFieldValue; "K12": CustomFieldValue; "K13": CustomFieldValue; "K14": CustomFieldValue; "K15": CustomFieldValue; "K16": CustomFieldValue; "K17": CustomFieldValue; "K18": CustomFieldValue; "X01": CustomFieldValue; "X02": CustomFieldValue; "X03": CustomFieldValue; "X04": CustomFieldValue; "X05": CustomFieldValue; "X06": CustomFieldValue; "X07": CustomFieldValue; "X08": CustomFieldValue; "X09": CustomFieldValue; "X10": CustomFieldValue; "X11": CustomFieldValue; "X12": CustomFieldValue; "M01": CustomFieldValue; "M02": CustomFieldValue; "M03": CustomFieldValue; "M04": CustomFieldValue; "M05": CustomFieldValue; "M06": CustomFieldValue; "M07": CustomFieldValue; "M08": CustomFieldValue; "M09": CustomFieldValue; "M10": CustomFieldValue; "M11": CustomFieldValue; "M12": CustomFieldValue; "Q01": CustomFieldValue; "Q02": CustomFieldValue; "Q03": CustomFieldValue; "Q04": CustomFieldValue; "Q05": CustomFieldValue; "Q06": CustomFieldValue; "Q07": CustomFieldValue; "Q08": CustomFieldValue; "Q09": CustomFieldValue; "Q10": CustomFieldValue; "Q11": CustomFieldValue; "Q12": CustomFieldValue; "Q13": CustomFieldValue; "Q14": CustomFieldValue; "R01": CustomFieldValue; "R02": CustomFieldValue; "R03": CustomFieldValue; "CW": CustomFieldValue} + type HttpMethod = 'get'|'GET'|'post'|'POST'|'put'|'PUT'|'patch'|'PATCH'|'delete'|'DELETE'; @@ -155,7 +157,7 @@ export class SubscriberService { * @param {string} id - The id needed for confirmation * @returns {object} */ - async sendConfirmationEmail(email: string, environment: string, subscriptions: object, id: string) { + async sendConfirmationEmail(email: string, environment: string, subscriptions: ValidSubscriptionSet, id: string) { // https://github.com/sendgrid/sendgrid-nodejs/blob/main/docs/use-cases/transactional-templates.md const msg = { to: email, @@ -182,7 +184,7 @@ export class SubscriberService { * @param {object} subscriptions - The subscriptions to validate. * @returns {boolean} */ - validateSubscriptions(subscriptions: object) { + validateSubscriptions(subscriptions: ValidSubscriptionSet) { if (!subscriptions) return false; @@ -220,7 +222,7 @@ export class SubscriberService { * @param {object} subscriptions - The set of CDs the user is subscribing to * @returns {boolean} */ - private convertSubscriptionsToHandlebars(subscriptions: object) { + private convertSubscriptionsToHandlebars(subscriptions: ValidSubscriptionSet) { var handlebars = { "citywide": false, "boroughs": [] } const boros = { "K": "Brooklyn", From a4c8da6f1ea5dfcd3003b18a7e4081e8c7aabfd6 Mon Sep 17 00:00:00 2001 From: David Hochbaum Date: Thu, 7 Nov 2024 14:39:50 -0500 Subject: [PATCH 6/6] Update type --- server/src/subscriber/subscriber.service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/subscriber/subscriber.service.ts b/server/src/subscriber/subscriber.service.ts index 17f928fe..f1b394c3 100644 --- a/server/src/subscriber/subscriber.service.ts +++ b/server/src/subscriber/subscriber.service.ts @@ -13,8 +13,7 @@ const validCustomFieldValues = [1] as const; export type CustomFieldValueTuple = typeof validCustomFieldValues; type CustomFieldValue = CustomFieldValueTuple[number]; -type ValidSubscriptionSet = {"K01": CustomFieldValue; "K02": CustomFieldValue; "K03": CustomFieldValue; "K04": CustomFieldValue; "K05": CustomFieldValue; "K06": CustomFieldValue; "K07": CustomFieldValue; "K08": CustomFieldValue; "K09": CustomFieldValue; "K10": CustomFieldValue; "K11": CustomFieldValue; "K12": CustomFieldValue; "K13": CustomFieldValue; "K14": CustomFieldValue; "K15": CustomFieldValue; "K16": CustomFieldValue; "K17": CustomFieldValue; "K18": CustomFieldValue; "X01": CustomFieldValue; "X02": CustomFieldValue; "X03": CustomFieldValue; "X04": CustomFieldValue; "X05": CustomFieldValue; "X06": CustomFieldValue; "X07": CustomFieldValue; "X08": CustomFieldValue; "X09": CustomFieldValue; "X10": CustomFieldValue; "X11": CustomFieldValue; "X12": CustomFieldValue; "M01": CustomFieldValue; "M02": CustomFieldValue; "M03": CustomFieldValue; "M04": CustomFieldValue; "M05": CustomFieldValue; "M06": CustomFieldValue; "M07": CustomFieldValue; "M08": CustomFieldValue; "M09": CustomFieldValue; "M10": CustomFieldValue; "M11": CustomFieldValue; "M12": CustomFieldValue; "Q01": CustomFieldValue; "Q02": CustomFieldValue; "Q03": CustomFieldValue; "Q04": CustomFieldValue; "Q05": CustomFieldValue; "Q06": CustomFieldValue; "Q07": CustomFieldValue; "Q08": CustomFieldValue; "Q09": CustomFieldValue; "Q10": CustomFieldValue; "Q11": CustomFieldValue; "Q12": CustomFieldValue; "Q13": CustomFieldValue; "Q14": CustomFieldValue; "R01": CustomFieldValue; "R02": CustomFieldValue; "R03": CustomFieldValue; "CW": CustomFieldValue} - +type ValidSubscriptionSet = Record; type HttpMethod = 'get'|'GET'|'post'|'POST'|'put'|'PUT'|'patch'|'PATCH'|'delete'|'DELETE';