Merge pull request #2334 from firebase/next

cabljac · web-flow · commit 51b002471da9 · 2025-03-14T16:18:07.000Z
Release @firebaseextensions/firestore-bigquery-change-tracker
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json
@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.39",
+  "version": "1.1.40",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts
@@ -4,6 +4,7 @@ import {
   FirestoreBigQueryEventHistoryTracker,
   FirestoreDocumentChangeEvent,
 } from "../..";
+import { LogLevel } from "../../logger";
 
 export const changeTracker = ({
   datasetId = "",
@@ -22,6 +23,7 @@ export const changeTracker = ({
   useIncrementalMaterializedView = false,
   maxStaleness = undefined,
   refreshIntervalMinutes = undefined,
+  logLevel = LogLevel.INFO,
 }): FirestoreBigQueryEventHistoryTracker => {
   return new FirestoreBigQueryEventHistoryTracker({
     datasetId,
@@ -40,6 +42,7 @@ export const changeTracker = ({
     useIncrementalMaterializedView,
     maxStaleness,
     refreshIntervalMinutes,
+    logLevel,
   });
 };
 
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/logger.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/logger.test.ts
@@ -0,0 +1,66 @@
+import { Logger, LogLevel, logger as defaultLogger } from "../logger";
+import { logger as funcsLogger } from "firebase-functions";
+
+// Mock firebase-functions logger functions
+jest.mock("firebase-functions", () => ({
+  logger: {
+    debug: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+  },
+}));
+
+describe("Logger", () => {
+  let log: Logger;
+
+  beforeEach(() => {
+    // Create a new instance before each test
+    log = new Logger(LogLevel.DEBUG);
+    jest.clearAllMocks();
+  });
+
+  describe("log level methods", () => {
+    test("debug should call funcsLogger.debug when level is DEBUG", () => {
+      log.setLogLevel(LogLevel.DEBUG);
+      log.debug("debug message");
+      expect(funcsLogger.debug).toHaveBeenCalledWith("debug message");
+    });
+
+    test("debug should not call funcsLogger.debug when level is INFO", () => {
+      log.setLogLevel(LogLevel.INFO);
+      log.debug("debug message");
+      expect(funcsLogger.debug).not.toHaveBeenCalled();
+    });
+
+    test("info should call funcsLogger.info when level is INFO or lower", () => {
+      log.setLogLevel(LogLevel.INFO);
+      log.info("info message");
+      expect(funcsLogger.info).toHaveBeenCalledWith("info message");
+    });
+
+    test("warn should call funcsLogger.warn when level is WARN or lower", () => {
+      log.setLogLevel(LogLevel.WARN);
+      log.warn("warn message");
+      expect(funcsLogger.warn).toHaveBeenCalledWith("warn message");
+    });
+
+    test("error should call funcsLogger.error when level is ERROR or lower", () => {
+      log.setLogLevel(LogLevel.ERROR);
+      log.error("error message");
+      expect(funcsLogger.error).toHaveBeenCalledWith("error message");
+    });
+
+    test("no logging should occur when log level is SILENT", () => {
+      log.setLogLevel(LogLevel.SILENT);
+      log.debug("debug message");
+      log.info("info message");
+      log.warn("warn message");
+      log.error("error message");
+      expect(funcsLogger.debug).not.toHaveBeenCalled();
+      expect(funcsLogger.info).not.toHaveBeenCalled();
+      expect(funcsLogger.warn).not.toHaveBeenCalled();
+      expect(funcsLogger.error).not.toHaveBeenCalled();
+    });
+  });
+});
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts
@@ -44,6 +44,7 @@ import { Clustering } from "./clustering";
 import { tableRequiresUpdate } from "./checkUpdates";
 import { parseErrorMessage, waitForInitialization } from "./utils";
 import { initializeLatestView } from "./initializeLatestView";
+import { logger, LogLevel } from "../logger";
 
 export { RawChangelogSchema, RawChangelogViewSchema } from "./schema";
 
@@ -68,6 +69,7 @@ export interface FirestoreBigQueryEventHistoryTrackerConfig {
   useIncrementalMaterializedView?: boolean;
   maxStaleness?: string;
   refreshIntervalMinutes?: number;
+  logLevel?: LogLevel | string;
 }
 
 /**
@@ -94,6 +96,8 @@ export class FirestoreBigQueryEventHistoryTracker
     if (!this.config.datasetLocation) {
       this.config.datasetLocation = "us";
     }
+
+    logger.setLogLevel(this.config.logLevel || LogLevel.INFO);
   }
 
   async record(events: FirestoreDocumentChangeEvent[]) {
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts
@@ -24,3 +24,4 @@ export {
   FirestoreDocumentChangeEvent,
   FirestoreEventHistoryTracker,
 } from "./tracker";
+export { LogLevel, Logger } from "./logger";
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logger.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logger.ts
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { logger as funcsLogger } from "firebase-functions";
+
+export enum LogLevel {
+  DEBUG = "debug", // Will log everything
+  INFO = "info", // Will log info, warnings, and errors
+  WARN = "warn", // Will log warnings and errors
+  ERROR = "error", // Will log errors only
+  SILENT = "silent", // Won't log anything
+}
+
+const levels = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+  silent: 4,
+};
+
+export class Logger {
+  private logLevel: number;
+
+  constructor(logLevel: LogLevel | string = LogLevel.INFO) {
+    this.setLogLevel(logLevel);
+  }
+
+  setLogLevel(logLevel: LogLevel | string): void {
+    if (typeof logLevel === "string") {
+      this.logLevel = levels[logLevel as keyof typeof levels] ?? levels.info;
+    } else {
+      this.logLevel = levels[logLevel];
+    }
+  }
+
+  debug(...args: any[]): void {
+    this.runIfLogLevel(levels.debug, funcsLogger.debug, ...args);
+  }
+
+  info(...args: any[]): void {
+    this.runIfLogLevel(levels.info, funcsLogger.info, ...args);
+  }
+
+  warn(...args: any[]): void {
+    this.runIfLogLevel(levels.warn, funcsLogger.warn, ...args);
+  }
+
+  error(...args: any[]): void {
+    this.runIfLogLevel(levels.error, funcsLogger.error, ...args);
+  }
+
+  log(...args: any[]): void {
+    this.info(...args);
+  }
+
+  private runIfLogLevel(level: number, func: Function, ...args: any[]): void {
+    if (this.logLevel <= level) {
+      func(...args);
+    }
+  }
+}
+
+export const logger = new Logger();
diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts
@@ -16,135 +16,135 @@
 
 import { Table } from "@google-cloud/bigquery";
 import { firestore } from "firebase-admin";
-import { logger } from "firebase-functions";
+import { logger } from "./logger";
 
 export const arrayFieldInvalid = (fieldName: string) => {
   logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
 };
 
 export const bigQueryDatasetCreated = (datasetId: string) => {
-  logger.log(`Created BigQuery dataset: ${datasetId}`);
+  logger.info(`Created BigQuery dataset: ${datasetId}`);
 };
 
 export const bigQueryDatasetCreating = (datasetId: string) => {
-  logger.log(`Creating BigQuery dataset: ${datasetId}`);
+  logger.debug(`Creating BigQuery dataset: ${datasetId}`);
 };
 
 export const bigQueryDatasetExists = (datasetId: string) => {
-  logger.log(`BigQuery dataset already exists: ${datasetId}`);
+  logger.info(`BigQuery dataset already exists: ${datasetId}`);
 };
 
 export const bigQueryErrorRecordingDocumentChange = (e: Error) => {
   logger.error(`Error recording document changes.`, e);
 };
 
 export const bigQueryLatestSnapshotViewQueryCreated = (query: string) => {
-  logger.log(`BigQuery latest snapshot view query:\n${query}`);
+  logger.debug(`BigQuery latest snapshot view query:\n${query}`);
 };
 
 export const bigQuerySchemaViewCreated = (name: string) => {
-  logger.log(`BigQuery created schema view ${name}\n`);
+  logger.debug(`BigQuery created schema view ${name}\n`);
 };
 
 export const bigQueryTableAlreadyExists = (
   tableName: string,
   datasetName: string
 ) => {
-  logger.log(
+  logger.debug(
     `BigQuery table with name ${tableName} already ` +
       `exists in dataset ${datasetName}!`
   );
 };
 
 export const bigQueryTableCreated = (tableName: string) => {
-  logger.log(`Created BigQuery table: ${tableName}`);
+  logger.info(`Created BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableCreating = (tableName: string) => {
-  logger.log(`Creating BigQuery table: ${tableName}`);
+  logger.debug(`Creating BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableUpdated = (tableName: string) => {
-  logger.log(`Updated existing BigQuery table: ${tableName}`);
+  logger.info(`Updated existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableUpdating = (tableName: string) => {
-  logger.log(`Updating existing BigQuery table: ${tableName}`);
+  logger.debug(`Updating existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableUpToDate = (tableName: string) => {
-  logger.log(`BigQuery table: ${tableName} is up to date`);
+  logger.info(`BigQuery table: ${tableName} is up to date`);
 };
 
 export const bigQueryTableValidated = (tableName: string) => {
-  logger.log(`Validated existing BigQuery table: ${tableName}`);
+  logger.info(`Validated existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableValidating = (tableName: string) => {
-  logger.log(`Validating existing BigQuery table: ${tableName}`);
+  logger.debug(`Validating existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryUserDefinedFunctionCreating = (functionName: string) => {
-  logger.log(`Creating BigQuery user-defined function ${functionName}`);
+  logger.debug(`Creating BigQuery user-defined function ${functionName}`);
 };
 
 export const bigQueryUserDefinedFunctionCreated = (functionName: string) => {
-  logger.log(`Created BigQuery user-defined function ${functionName}`);
+  logger.info(`Created BigQuery user-defined function ${functionName}`);
 };
 
 export const bigQueryViewCreated = (viewName: string) => {
-  logger.log(`Created BigQuery view: ${viewName}`);
+  logger.info(`Created BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewCreating = (viewName: string, query: string) => {
-  logger.log(`Creating BigQuery view: ${viewName}\nQuery:\n${query}`);
+  logger.debug(`Creating BigQuery view: ${viewName}\nQuery:\n${query}`);
 };
 
 export const bigQueryViewAlreadyExists = (
   viewName: string,
   datasetName: string
 ) => {
-  logger.log(
+  logger.info(
     `View with id ${viewName} already exists in dataset ${datasetName}.`
   );
 };
 
 export const bigQueryViewUpdated = (viewName: string) => {
-  logger.log(`Updated existing BigQuery view: ${viewName}`);
+  logger.info(`Updated existing BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewUpdating = (viewName: string) => {
-  logger.log(`Updating existing BigQuery view: ${viewName}`);
+  logger.debug(`Updating existing BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewUpToDate = (viewName: string) => {
-  logger.log(`BigQuery view: ${viewName} is up to date`);
+  logger.info(`BigQuery view: ${viewName} is up to date`);
 };
 
 export const bigQueryViewValidated = (viewName: string) => {
-  logger.log(`Validated existing BigQuery view: ${viewName}`);
+  logger.info(`Validated existing BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewValidating = (viewName: string) => {
-  logger.log(`Validating existing BigQuery view: ${viewName}`);
+  logger.debug(`Validating existing BigQuery view: ${viewName}`);
 };
 
 export const complete = () => {
-  logger.log("Completed mod execution");
+  logger.info("Completed mod execution");
 };
 
 export const dataInserted = (rowCount: number) => {
-  logger.log(`Inserted ${rowCount} row(s) of data into BigQuery`);
+  logger.debug(`Inserted ${rowCount} row(s) of data into BigQuery`);
 };
 
 export const dataInsertRetried = (rowCount: number) => {
-  logger.log(
+  logger.debug(
     `Retried to insert ${rowCount} row(s) of data into BigQuery (ignoring unknown columns)`
   );
 };
 
 export const dataInserting = (rowCount: number) => {
-  logger.log(`Inserting ${rowCount} row(s) of data into BigQuery`);
+  logger.debug(`Inserting ${rowCount} row(s) of data into BigQuery`);
 };
 
 export const dataTypeInvalid = (
@@ -168,11 +168,11 @@ export const timestampMissingValue = (fieldName: string) => {
 };
 
 export const addNewColumn = (table: string, field: string) => {
-  logger.log(`Updated '${table}' table with a '${field}' column`);
+  logger.info(`Updated '${table}' table with a '${field}' column`);
 };
 
 export const addPartitionFieldColumn = (table, field) => {
-  logger.log(
+  logger.info(
     `Updated '${table}' table with a partition field '${field}' column`
   );
 };

Original file line number	Diff line number	Diff line change
`@@ -44,6 +44,7 @@ import { Clustering } from "./clustering";`
`44`	`44`	`import { tableRequiresUpdate } from "./checkUpdates";`
`45`	`45`	`import { parseErrorMessage, waitForInitialization } from "./utils";`
`46`	`46`	`import { initializeLatestView } from "./initializeLatestView";`
	`47`	`+import { logger, LogLevel } from "../logger";`
`47`	`48`
`48`	`49`	`export { RawChangelogSchema, RawChangelogViewSchema } from "./schema";`
`49`	`50`
`@@ -68,6 +69,7 @@ export interface FirestoreBigQueryEventHistoryTrackerConfig {`
`68`	`69`	`useIncrementalMaterializedView?: boolean;`
`69`	`70`	`maxStaleness?: string;`
`70`	`71`	`refreshIntervalMinutes?: number;`
	`72`	`+ logLevel?: LogLevel \| string;`
`71`	`73`	`}`
`72`	`74`
`73`	`75`	`/**`
`@@ -94,6 +96,8 @@ export class FirestoreBigQueryEventHistoryTracker`
`94`	`96`	`if (!this.config.datasetLocation) {`
`95`	`97`	`this.config.datasetLocation = "us";`
`96`	`98`	`}`
	`99`	`+`
	`100`	`+ logger.setLogLevel(this.config.logLevel \|\| LogLevel.INFO);`
`97`	`101`	`}`
`98`	`102`
`99`	`103`	`async record(events: FirestoreDocumentChangeEvent[]) {`