Merge pull request #55 from luusluus/53-suggest-adjust-date-type-display-format

53 suggest adjust date type display format

Author: luusluus

.vscode/settings.json

@@ -14,4 +14,6 @@
     "parquet-visualizer.backend": "duckdb",
     "parquet-visualizer.defaultQuery": "SELECT *\r\nFROM data\r\nLIMIT 1000;",
     "parquet-visualizer.RunQueryKeyBinding": "Ctrl-Enter",
+    "parquet-visualizer.dateTimeFormat": "RFC2822",
+    "parquet-visualizer.outputDateTimeFormatInUTC": true
 }

README.md

@@ -54,6 +54,9 @@ The following configuration options are available:
 |`parquet-visualizer.backend`|`duckdb`| Backend for reading the parquet file. Options: `duckdb`, `parquet-wasm`|
 |`parquet-visualizer.defaultPageSizes`|`[20, 50, 100, 500]`|Set the default page size for data and query tab.|
 |`parquet-visualizer.defaultQuery`|`SELECT *\r\nFROM data\r\nLIMIT 1000;`|Default SQL query for parquet file. The table `data` should remain the same.|
+|`parquet-visualizer.RunQueryKeyBinding`|`Ctrl-Enter`|Default Key Binding for running queries. If Ctrl is written, it will be translated to Command for mac and vica versa. E.g., Ctrl-E will be synonymous to Command-E.|
+|`parquet-visualizer.dateTimeFormat`|`ISO8601`|Set datetime format for columns of timestamp type. Defaults to ISO8601. You can set a custom format like `YYYY-MM-DD HH:mm:ss.SSS Z`. Find rules for formatting [here](https://www.npmjs.com/package/date-and-time#formatdateobj-arg-utc).|
+|`parquet-visualizer.outputDateTimeFormatInUTC`|`true`|Outputs the datetime format for timestamp columns in UTC or in local time.|
 
 
 ## Parquet backends

package-lock.json

@@ -79,6 +79,20 @@
             100,
             500
           ]
+        },
+        "parquet-visualizer.dateTimeFormat": {
+          "markdownDescription": "Set datetime format for columns of timestamp type. Defaults to ISO8601. You can set a custom format like `YYYY-MM-DD HH:mm:ss.SSS Z`. Find rules for formatting [here](https://www.npmjs.com/package/date-and-time#formatdateobj-arg-utc).",
+          "type": "string",
+          "enumDescriptions": [
+            "ISO8601",
+            "RFC2822"
+          ],
+          "default": "ISO8601"
+        },
+        "parquet-visualizer.outputDateTimeFormatInUTC": {
+          "type": "boolean",
+          "markdownDescription": "Outputs the datetime format for timestamp columns in UTC or in local time.",
+          "default": true
         }
       }
     },
@@ -127,6 +141,7 @@
   },
   "dependencies": {
     "apache-arrow": "^15.0.0",
+    "date-and-time": "^3.6.0",
     "duckdb-async": "^1.0.0",
     "parquet-wasm": "^0.6.0"
   },

package.json

@@ -1,18 +1,21 @@
 import { Schema, Field, Type } from "apache-arrow";
 
 import { convertBigIntToString, convertObjectsToJSONStrings } from './util';
-
+import { DateTimeFormatSettings } from './types';
 
 export abstract class Backend {
     public filePath: string;
     protected arrowSchema: Schema;
     protected metadata: any;
 
-    public constructor(filePath: string) {
+    private dateTimeFormat: DateTimeFormatSettings;
+
+    public constructor(filePath: string, dateTimeFormat: DateTimeFormatSettings) {
         this.filePath = filePath;
+        this.dateTimeFormat = dateTimeFormat;
     }
 
-    static createAsync(filePath: string): Promise<any> {
+    static createAsync(filePath: string, dateTimeFormat: DateTimeFormatSettings): Promise<any> {
         throw new Error("Method not implemented");
     };
 
@@ -107,7 +110,7 @@ export abstract class Backend {
                 key: 'footer_signing_key_metadata',
                 value: Number(this.metadata[0]["footer_signing_key_metadata"])
             },
-        ]
+        ];
     };
 
     abstract getMetaDataImpl(): Promise<any>;
@@ -119,7 +122,9 @@ export abstract class Backend {
 
         const queryResult = await this.queryImpl(query);
         const result = queryResult.map(v => {
-            return convertObjectsToJSONStrings(convertBigIntToString(v));
+            return convertObjectsToJSONStrings(
+                convertBigIntToString(v, this.dateTimeFormat)
+            );
         });
 
         const endTime = performance.now();

src/backend.ts

@@ -1,27 +1,27 @@
-import * as fs from 'fs';
-
 import * as duckdb from "duckdb-async";
-import { Backend } from "./backend";
 
 import { tableFromIPC, Schema } from "apache-arrow";
 
+import { Backend } from "./backend";
+import { DateTimeFormatSettings } from './types';
+
 export class DuckDBBackend extends Backend{
     private db: duckdb.Database;
     public arrowSchema: Schema<any>;
     public metadata: any;
 
-    constructor(filePath: string, db: duckdb.Database){
-      super(filePath);
+    constructor(filePath: string, dateTimeFormatSettings: DateTimeFormatSettings, db: duckdb.Database){
+      super(filePath, dateTimeFormatSettings);
       this.db = db;
     }
 
-    public static override async createAsync (path: string){
+    public static override async createAsync (path: string, dateTimeFormatSettings: DateTimeFormatSettings){
       const db = await duckdb.Database.create(":memory:");
       // const splitted = path.split('/');
       // const filename = splitted[splitted.length - 1];
       // const filePath = `${__dirname}/${filename}.duckdb`;
       // const db = await duckdb.Database.create(filePath);
-      return new DuckDBBackend(path, db);
+      return new DuckDBBackend(path, dateTimeFormatSettings, db);
     }
 
     dispose() {

src/duckdb-backend.ts

@@ -12,7 +12,8 @@ import { DuckDBBackend } from './duckdb-backend';
 import { DuckDBPaginator } from './duckdb-paginator';
 import { ParquetWasmBackend } from './parquet-wasm-backend';
 import { ParquetWasmPaginator } from './parquet-wasm-paginator';
-import { affectsDocument, defaultPageSizes, defaultQuery, defaultBackend, defaultRunQueryKeyBinding } from './settings';
+import { affectsDocument, defaultPageSizes, defaultQuery, defaultBackend, defaultRunQueryKeyBinding, dateTimeFormat, outputDateTimeFormatInUTC } from './settings';
+import { DateTimeFormatSettings } from './types';
 
 // import { getLogger } from './logger';
 
@@ -30,10 +31,14 @@ class CustomParquetDocument extends Disposable implements vscode.CustomDocument
     static async create(
       uri: vscode.Uri
     ): Promise<CustomParquetDocument | PromiseLike<CustomParquetDocument>> {
+        const dateTimeFormatSettings: DateTimeFormatSettings = {
+          format: dateTimeFormat(),
+          useUTC: outputDateTimeFormatInUTC()
+        };
         try{
             switch (defaultBackend()) {
               case 'duckdb': {
-                const backend = await DuckDBBackend.createAsync(uri.fsPath);
+                const backend = await DuckDBBackend.createAsync(uri.fsPath, dateTimeFormatSettings);
                 await backend.initialize();
                 const totalItems = backend.getRowCount();
                 const table = 'data';
@@ -42,7 +47,7 @@ class CustomParquetDocument extends Disposable implements vscode.CustomDocument
                 return new CustomParquetDocument(uri, backend, paginator);
               }
               case 'parquet-wasm': {
-                const backend = await ParquetWasmBackend.createAsync(uri.fsPath);
+                const backend = await ParquetWasmBackend.createAsync(uri.fsPath, dateTimeFormatSettings);
                 const paginator = new ParquetWasmPaginator(backend);
                 return new CustomParquetDocument(uri, backend, paginator);
               }
@@ -53,7 +58,7 @@ class CustomParquetDocument extends Disposable implements vscode.CustomDocument
         } catch (err: any){
             console.log(err);
 
-            const backend = await ParquetWasmBackend.createAsync(uri.fsPath);
+            const backend = await ParquetWasmBackend.createAsync(uri.fsPath, dateTimeFormatSettings);
             const paginator = new ParquetWasmPaginator(backend);
             return new CustomParquetDocument(uri, backend, paginator);
         }
@@ -73,28 +78,34 @@ class CustomParquetDocument extends Disposable implements vscode.CustomDocument
       if (this.backend instanceof DuckDBBackend) {
         this.isQueryAble = true;
 
+        const dateTimeFormatSettings: DateTimeFormatSettings = {
+          format: dateTimeFormat(),
+          useUTC: outputDateTimeFormatInUTC()
+        };
+
         this.worker = new Worker(__dirname + "/worker.js", {
-            workerData: {
-              pathParquetFile: this.uri.fsPath
-            }
-          });
-    
-          this.worker.on('message', (message) => {
-            if (message.type === 'query'){
-              this.emitQueryResult(message);
-            } else if (message.type === 'paginator') {
-              this.fireDataPaginatorEvent(
-                message.result, 
-                message.rowCount,
-                message.pageSize,
-                message.pageNumber,
-                message.pageCount,
-                requestSourceQueryTab
-              );
-            } else if (message.type === 'exportQueryResults') {
-              vscode.window.showInformationMessage(`Exported query result to ${message.path}`);
-            }
-          });
+          workerData: {
+            pathParquetFile: this.uri.fsPath,
+            dateTimeFormatSettings: dateTimeFormatSettings,
+          }
+        });
+  
+        this.worker.on('message', (message) => {
+          if (message.type === 'query'){
+            this.emitQueryResult(message);
+          } else if (message.type === 'paginator') {
+            this.fireDataPaginatorEvent(
+              message.result, 
+              message.rowCount,
+              message.pageSize,
+              message.pageNumber,
+              message.pageCount,
+              requestSourceQueryTab
+            );
+          } else if (message.type === 'exportQueryResults') {
+            vscode.window.showInformationMessage(`Exported query result to ${message.path}`);
+          }
+        });
       }
     }
 

src/parquet-editor.ts

@@ -11,23 +11,25 @@ import { ParquetFile } from 'parquet-wasm';
 import { Backend } from './backend';
 import { Table } from 'apache-arrow/table';
 import { tableFromIPC } from 'apache-arrow';
+import { DateTimeFormatSettings } from './types';
 
 export class ParquetWasmBackend extends Backend {
   public parquetFile: ParquetFile;
 
 
   private constructor(
     filePath: string,
+    dateTimeFormat: DateTimeFormatSettings,
     parquetFile: ParquetFile, 
     table: Table<any>
   ) {
-    super(filePath);
+    super(filePath, dateTimeFormat);
     this.parquetFile = parquetFile;
     this.arrowSchema = table.schema;
     this.metadata = this.getMetaDataImpl();
   }
 
-  public static override async createAsync (path: string) {
+  public static override async createAsync (path: string, dateTimeFormat: DateTimeFormatSettings) {
     const buffer = fs.readFileSync(path);
     const file = new File([buffer], "fileName", {
       type: "application/vnd.apache.parquet",
@@ -43,6 +45,7 @@ export class ParquetWasmBackend extends Backend {
 
     return new ParquetWasmBackend(
       path, 
+      dateTimeFormat,
       parquetFile, 
       table
     );

src/parquet-wasm-backend.ts

@@ -21,6 +21,14 @@ export function defaultBackend(): string {
 export function defaultRunQueryKeyBinding(): string {
     return settings().get('RunQueryKeyBinding');
 }
+
+export function dateTimeFormat(): string {
+    return settings().get('dateTimeFormat') as string;
+}
+
+export function outputDateTimeFormatInUTC(): boolean {
+    return settings().get<boolean>('outputDateTimeFormatInUTC') as boolean;
+}
 
 function settingsChanged(e: vscode.ConfigurationChangeEvent, sections: string[]): boolean {
     return sections.map(s => `${name}.${s}`).some(s => e.affectsConfiguration(s));

src/settings.ts

@@ -0,0 +1,5 @@
+
+export interface DateTimeFormatSettings {
+  format: string;
+  useUTC: boolean;
+}

src/types.ts

@@ -1,3 +1,7 @@
+import date from 'date-and-time';
+
+import { DateTimeFormatSettings } from './types';
+
 export function getNonce() {
 	let text = '';
 	const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
@@ -17,21 +21,33 @@ export function convertToTabulatorData(array: any[]) {
     });
   }
 
-export function convertBigIntToString(obj: any): any {
+export function convertBigIntToString(obj: any, dateTimeFormatSettings: DateTimeFormatSettings): any {
   if (Array.isArray(obj)) {
-    return obj.map(item => convertBigIntToString(item));
+    return obj.map(item => convertBigIntToString(item, dateTimeFormatSettings));
   } 
   else if (obj instanceof Uint8Array) {
     return Array.from(obj.values());
   }
   else if (obj !== null && typeof obj === 'object') {
     if (obj instanceof Date) {
-      return obj.toUTCString();
+      if (dateTimeFormatSettings.format === "ISO8601") {
+        return obj.toISOString();
+      }
+      else if (dateTimeFormatSettings.format === "RFC2822") {
+        return obj.toUTCString();
+      }
+      else {
+        return date.format(
+          obj, 
+          dateTimeFormatSettings.format, 
+          dateTimeFormatSettings.useUTC
+        );
+      }
     }
     const newObj: { [key: string]: any } = {};
     for (const key in obj) {
       if (obj.hasOwnProperty(key)) {
-          newObj[key] = convertBigIntToString(obj[key]);
+          newObj[key] = convertBigIntToString(obj[key], dateTimeFormatSettings);
       }
     }
     return newObj;
@@ -95,7 +111,7 @@ export function createHeadersFromData(data: any) {
 }
 
 export function replacePeriodWithUnderscoreInKey(data: any) {
-  return data.map(obj => {
+  return data.map((obj: { [x: string]: any; }) => {
     const newObj: { [key: string]: any } = {};
 
     Object.keys(obj).forEach(key => {

src/util.ts

@@ -12,7 +12,7 @@ import { Paginator } from './paginator';
 import { DuckDBBackend } from './duckdb-backend';
 import { DuckDBPaginator } from './duckdb-paginator';
 import { createHeadersFromData, replacePeriodWithUnderscoreInKey } from './util';
-
+import { DateTimeFormatSettings } from './types';
 
 class BackendWorker {
   paginator: Paginator;
@@ -23,8 +23,8 @@ class BackendWorker {
     this.backend = backend;
   }
 
-  static async create(path: string) {
-    const backend = await DuckDBBackend.createAsync(path);
+  static async create(path: string, dateTimeFormatSettings: DateTimeFormatSettings) {
+    const backend = await DuckDBBackend.createAsync(path, dateTimeFormatSettings);
     await backend.initialize();
     return new BackendWorker(backend);
   }
@@ -140,7 +140,10 @@ class BackendWorker {
 }
 
 (async () => {
-    const worker = await BackendWorker.create(workerData.pathParquetFile);
+    const worker = await BackendWorker.create(
+      workerData.pathParquetFile,
+      workerData.dateTimeFormatSettings
+    );
 
     parentPort.on('message', async (message: any) => {
         switch (message.source) {