feat: release v1

Author: rehanvdm

.projen/tasks.json

@@ -2,6 +2,7 @@ import { awscdk, javascript } from 'projen';
 import { ArrowParens, NpmAccess, TrailingComma } from 'projen/lib/javascript';
 
 const project = new awscdk.AwsCdkConstructLibrary({
+  majorVersion: 1,
   projenrcTs: true,
   name: 'serverless-website-analytics',
   author: 'rehanvdm',

.projenrc.ts

@@ -239,6 +239,51 @@ the date will be stored after about 1min ± 1min.
 Location data is obtained by looking the IP address up in the [MaxMind GeoLite2](https://dev.maxmind.com/geoip/geoip2/geolite2/) database.
 We don't store any Personally Identifiable Information (PII) in the logs or S3, the IP address is never stored.
 
+# Upgrading
+
+## From V0 to V1
+
+This upgrade brings two breaking changes:
+1. Daily partitions, querying is not backwards compatible. The data is still there, it is just in
+a different location so the dashboard will look empty after migrating.
+2. A change of Route53 record construct IDs that need manual intervention (only if you specified the `domains` property)
+
+Install the new version:
+```
+npm install npm install serverless-website-analytics@~1
+```
+
+### Data "loss" because of S3 path changes to accommodate daily partitions
+
+Data will seem lost after upgrading to V1 because of the S3 path changes to accommodate daily partitions. The data is
+still there, it is just in a different location. The backend won't know about the old location and only use the new location
+so your dashboard will look empty after migrating. You can possibly run an Athena CTAS query to migrate the data to the new
+location, but it would need to be crafted carefully. If this is really important for you, please create a ticket and I can
+see if I can help.
+
+### Recreate the old Route53 records (only if you specified the `domains' property)
+
+This is because we needed to change the CDK construct IDs of the Route53 records and Route53 can not create duplicate
+record names. See issue: https://github.com/rehanvdm/serverless-website-analytics/issues/26
+
+There will be some downtime, it should be less than 10 minutes. If downtime is not acceptable then use CDK escape
+hatches to hardcode the Route53 record IDs of your existing constructs.
+
+**IMPORTANT: Take note of the names and values of these DNS records as we need to recreate them manually after deleting them.**
+
+Order of operation:
+1. Delete DNS records with AWS CLI/Console
+   1.1 Delete the A record pointing to your CloudFront as defined by the `domain.name` property.
+   1.2 Optional, if using `auth.cognito` delete the Cognito login A record as well, which is defined as: `{auth.cognito.loginSubDomain}.{domain.name}`
+2. CDK deploy
+3. Recreate the DNS records with AWS CLI/Console that you deleted in step 1.
+
+If you do not delete them before upgrading, you will get one of these errors in CloudFormation and it will roll back.
+```
+[Tried to create resource record set [name='analytics.rehanvdm.com.', type='A'] but it already exists]
+[Tried to create resource record set [name='login.analytics.rehanvdm.com.', type='A'] but it already exists]
+```
+
 ## Sponsors
 
 Proudly sponsored by:

README.md

@@ -239,6 +239,51 @@ the date will be stored after about 1min ± 1min.
 Location data is obtained by looking the IP address up in the [MaxMind GeoLite2](https://dev.maxmind.com/geoip/geoip2/geolite2/) database.
 We don't store any Personally Identifiable Information (PII) in the logs or S3, the IP address is never stored.
 
+# Upgrading
+
+## From V0 to V1
+
+This upgrade brings two breaking changes:
+1. Daily partitions, querying is not backwards compatible. The data is still there, it is just in
+a different location so the dashboard will look empty after migrating.
+2. A change of Route53 record construct IDs that need manual intervention (only if you specified the `domains` property)
+
+Install the new version:
+```
+npm install npm install serverless-website-analytics@~1
+```
+
+### Data "loss" because of S3 path changes to accommodate daily partitions
+
+Data will seem lost after upgrading to V1 because of the S3 path changes to accommodate daily partitions. The data is
+still there, it is just in a different location. The backend won't know about the old location and only use the new location
+so your dashboard will look empty after migrating. You can possibly run an Athena CTAS query to migrate the data to the new
+location, but it would need to be crafted carefully. If this is really important for you, please create a ticket and I can
+see if I can help.
+
+### Recreate the old Route53 records (only if you specified the `domains' property)
+
+This is because we needed to change the CDK construct IDs of the Route53 records and Route53 can not create duplicate
+record names. See issue: https://github.com/rehanvdm/serverless-website-analytics/issues/26
+
+There will be some downtime, it should be less than 10 minutes. If downtime is not acceptable then use CDK escape
+hatches to hardcode the Route53 record IDs of your existing constructs.
+
+**IMPORTANT: Take note of the names and values of these DNS records as we need to recreate them manually after deleting them.**
+
+Order of operation:
+1. Delete DNS records with AWS CLI/Console
+   1.1 Delete the A record pointing to your CloudFront as defined by the `domain.name` property.
+   1.2 Optional, if using `auth.cognito` delete the Cognito login A record as well, which is defined as: `{auth.cognito.loginSubDomain}.{domain.name}`
+2. CDK deploy
+3. Recreate the DNS records with AWS CLI/Console that you deleted in step 1.
+
+If you do not delete them before upgrading, you will get one of these errors in CloudFormation and it will roll back.
+```
+[Tried to create resource record set [name='analytics.rehanvdm.com.', type='A'] but it already exists]
+[Tried to create resource record set [name='login.analytics.rehanvdm.com.', type='A'] but it already exists]
+```
+
 ## Sponsors
 
 Proudly sponsored by:

docs/API.md

@@ -108,12 +108,8 @@ export function backendAnalytics(scope: Construct, name: (name: string) => strin
           type: 'string',
         },
         {
-          name: 'year',
-          type: 'int',
-        },
-        {
-          name: 'month',
-          type: 'int',
+          name: 'page_opened_at_date',
+          type: 'string',
         },
       ],
       storageDescriptor: {
@@ -128,7 +124,9 @@ export function backendAnalytics(scope: Construct, name: (name: string) => strin
         },
         parameters: {
           'storage.location.template':
-            's3://' + analyticsBucket.bucketName + '/page_views/site=${site}/year=${year}/month=${month}',
+            's3://' +
+            analyticsBucket.bucketName +
+            '/page_views/site=${site}/page_opened_at_date=${page_opened_at_date}',
         },
         columns: [
           {
@@ -207,12 +205,11 @@ export function backendAnalytics(scope: Construct, name: (name: string) => strin
       },
       parameters: {
         'projection.enabled': 'true',
-        'projection.year.type': 'integer',
-        'projection.year.range': '2023,3023',
-        'projection.year.interval': '1',
-        'projection.month.type': 'integer',
-        'projection.month.range': '1,12',
-        'projection.month.interval': '1',
+        'projection.page_opened_at_date.type': 'date',
+        'projection.page_opened_at_date.format': 'yyyy-MM-dd',
+        'projection.page_opened_at_date.interval': '1',
+        'projection.page_opened_at_date.interval.unit': 'DAYS',
+        'projection.page_opened_at_date.range': '2023-01-01,NOW',
         'projection.site.type': 'enum',
         'projection.site.values': props.sites.join(','),
       },
@@ -231,7 +228,7 @@ export function backendAnalytics(scope: Construct, name: (name: string) => strin
       bucketArn: analyticsBucket.bucketArn,
       roleArn: firehoseDeliveryRole.roleArn,
       prefix:
-        'page_views/site=!{partitionKeyFromQuery:site}/year=!{partitionKeyFromQuery:year}/month=!{partitionKeyFromQuery:month}/',
+        'page_views/site=!{partitionKeyFromQuery:site}/page_opened_at_date=!{partitionKeyFromQuery:page_opened_at_date}/',
       errorOutputPrefix: 'error/!{firehose:error-output-type}/',
       bufferingHints: {
         intervalInSeconds: props.firehoseBufferInterval ?? defaultFirehoseBufferInterval,
@@ -250,7 +247,7 @@ export function backendAnalytics(scope: Construct, name: (name: string) => strin
             parameters: [
               {
                 parameterName: 'MetadataExtractionQuery',
-                parameterValue: '{site: .site,' + ' year: .year,' + ' month: .month}',
+                parameterValue: '{site: .site, page_opened_at_date: .page_opened_at_date}',
               },
               //Required as property it seems
               {

src/backendAnalytics.ts

@@ -187,7 +187,7 @@ export function frontend(
     /* First CloudFront A record if we can */
     let cloudFrontRecord: route53.ARecord | undefined = undefined;
     if (props.domain.hostedZone) {
-      cloudFrontRecord = new route53.ARecord(scope, 'cloudfront-record', {
+      cloudFrontRecord = new route53.ARecord(scope, name('cloudfront-record'), {
         recordName: props.domain.name,
         target: route53.RecordTarget.fromAlias(new route53Targets.CloudFrontTarget(frontendDist)),
         zone: props.domain.hostedZone,
@@ -218,7 +218,7 @@ export function frontend(
 
       /* Add the Cognito A record if we can */
       if (props.domain.hostedZone) {
-        new route53.ARecord(scope, 'custom-domain-dns-record', {
+        new route53.ARecord(scope, name('custom-domain-dns-record'), {
           recordName: props.auth.cognito.loginSubDomain,
           target: route53.RecordTarget.fromAlias({
             bind: () => ({

src/frontend.ts

@@ -47,16 +47,16 @@ export function pageView(trpcInstance: TrpcInstance) {
       );
       const referrer = getCleanedUpReferrer(input.site, input.referrer);
       const pageOpenedAt = DateUtils.parseIso(input.page_opened_at);
+      const pageOpenedAtDate = DateUtils.stringifyFormat(pageOpenedAt, 'yyyy-MM-dd');
+
       const data: Page = {
         site: input.site,
-        year: pageOpenedAt.getFullYear(),
-        month: pageOpenedAt.getMonth() + 1,
-
         user_id: input.user_id,
         session_id: input.session_id,
         page_id: input.page_id,
         page_url: input.page_url,
         page_opened_at: input.page_opened_at,
+        page_opened_at_date: pageOpenedAtDate,
         time_on_page: input.time_on_page,
         country_iso: countryIso,
         country_name: countryName,

src/src/backend/api-ingest/v1/page/view/index.ts

@@ -25,14 +25,8 @@ export class AthenaPageViews extends AthenaBase {
    * @param sites
    */
   cteFilteredDataQuery(columns: string[], fromDate: Date, toDate: Date, sites: string[]) {
-    const months = DateUtils.getMonthsBetweenDates(fromDate, toDate);
-    const cteWhereClauseDates: string = months
-      // +1 to the month because months are 0-indexed in JS but Athena/Firehose is 1-indexed
-      .map((month) => `(year = ${month.getFullYear()} AND month = ${month.getMonth() + 1})`)
-      .join(' OR ');
     const cteWhereClauseSites = sites.map((site) => `site = '${site}'`).join(' OR ');
-
-    const cteWhereClause = `(${cteWhereClauseSites}) AND (${cteWhereClauseDates})`;
+    const cteWhereClause = `(${cteWhereClauseSites})`;
     const exactTimeFrom = DateUtils.stringifyFormat(fromDate, 'yyyy-MM-dd HH:mm:ss.SSS');
     const exactTimeTo = DateUtils.stringifyFormat(toDate, 'yyyy-MM-dd HH:mm:ss.SSS');
 

src/src/backend/lib/dal/athena/page_views.ts

@@ -2,13 +2,12 @@ import { z } from 'zod';
 
 export const SchemaPage = z.object({
   site: z.string(),
-  year: z.number(),
-  month: z.number(),
   user_id: z.string(),
   session_id: z.string(),
   page_id: z.string(),
   page_url: z.string(),
   page_opened_at: z.string(),
+  page_opened_at_date: z.string().optional(),
   time_on_page: z.number(),
   country_iso: z.string().optional(),
   country_name: z.string().optional(),

src/src/backend/lib/models/page.ts

@@ -196,9 +196,9 @@ describe('API Frontend', function () {
         // from: new Date(2023, 3, 1, 0, 0, 0).toISOString(),
         // to: new Date(2023, 3, 30, 23, 59, 59,999).toISOString(),
 
-        // Month 4 - April - Past so data fixed, not changing still
-        from: new Date(2023, 3, 1, 0, 0, 0).toISOString(),
-        to: new Date(2023, 3, 22, 23, 59, 59, 999).toISOString(),
+        // Month 5 - June
+        from: new Date(2023, 5, 1, 0, 0, 0).toISOString(),
+        to: new Date(2023, 5, 22, 23, 59, 59, 999).toISOString(),
       },
       authHeaders
     );