✨ feat: Add configurable PDF processing method with Unstructured

src/config/knowledge.ts

@@ -1,19 +1,17 @@
 import { createEnv } from '@t3-oss/env-nextjs';
 import { z } from 'zod';
 
-export const getKnowledgeConfig = () => {
-  return createEnv({
-    runtimeEnv: {
-      DEFAULT_FILES_CONFIG: process.env.DEFAULT_FILES_CONFIG,
-      UNSTRUCTURED_API_KEY: process.env.UNSTRUCTURED_API_KEY,
-      UNSTRUCTURED_SERVER_URL: process.env.UNSTRUCTURED_SERVER_URL,
-    },
-    server: {
-      DEFAULT_FILES_CONFIG: z.string().optional(),
-      UNSTRUCTURED_API_KEY: z.string().optional(),
-      UNSTRUCTURED_SERVER_URL: z.string().optional(),
-    },
-  });
-};
-
-export const knowledgeEnv = getKnowledgeConfig();
+export const knowledgeEnv = createEnv({
+  runtimeEnv: {
+    DEFAULT_FILES_CONFIG: process.env.DEFAULT_FILES_CONFIG,
+    UNSTRUCTURED_API_KEY: process.env.UNSTRUCTURED_API_KEY,
+    UNSTRUCTURED_SERVER_URL: process.env.UNSTRUCTURED_SERVER_URL,
+    USE_UNSTRUCTURED_FOR_PDF: process.env.USE_UNSTRUCTURED_FOR_PDF,
+  },
+  server: {
+    DEFAULT_FILES_CONFIG: z.string().optional(),
+    UNSTRUCTURED_API_KEY: z.string().optional(),
+    UNSTRUCTURED_SERVER_URL: z.string().optional(),
+    USE_UNSTRUCTURED_FOR_PDF: z.string().optional(),
+  },
+});

src/server/modules/ContentChunk/index.ts

@@ -3,6 +3,7 @@ import { Strategy } from 'unstructured-client/sdk/models/shared';
 
 import { NewChunkItem, NewUnstructuredChunkItem } from '@/database/schemas';
 import { ChunkingStrategy, Unstructured } from '@/libs/unstructured';
+import { knowledgeEnv } from '@/config/knowledge';
 
 export interface ChunkContentParams {
   content: Uint8Array;
@@ -26,7 +27,10 @@ export class ContentChunk {
   }
 
   isUsingUnstructured(params: ChunkContentParams) {
-    return params.fileType === 'application/pdf' && params.mode === 'hi-res';
+    return params.fileType === 'application/pdf' &&
+           !!knowledgeEnv.USE_UNSTRUCTURED_FOR_PDF &&
+           !!knowledgeEnv.UNSTRUCTURED_API_KEY &&
+           !!knowledgeEnv.UNSTRUCTURED_SERVER_URL;
   }
 
   async chunkContent(params: ChunkContentParams): Promise<ChunkResult> {

src/server/utils/env.ts

@@ -0,0 +1,12 @@
+export const isDev = process.env.NODE_ENV === 'development';
+
+export const isOnServerSide = typeof window === 'undefined';
+/**
+ * Get environment variable value
+ * @param key - Environment variable key
+ * @returns Environment variable value or empty string if not found
+ */
+export const getEnvironment = (key: string): string => {
+  if (typeof process === 'undefined') return '';
+  return process.env[key] || '';
+};