Merge pull request #97 from nemozak1/opeyII_integration

Opey ii integration

Author: simonredfern

.env.example

@@ -8,6 +8,9 @@ VITE_OBP_API_PORTAL_HOST=https://apisandbox.openbankproject.com
 ####################################################################################
 
 VITE_OBP_API_VERSION=v5.1.0
+#The default version of the root page, it has the default value `OBP+VITE_OBP_API_VERSION`
+#The format must follow standard+Version, e.g., OBPv5.1.0, BGv1, or BGv1.3.
+#VITE_OBP_API_DEFAULT_RESOURCE_DOC_VERSION=OBPv5.1.0
 VITE_OBP_API_MANAGER_HOST=https://apimanagersandbox.openbankproject.com
 VITE_OBP_API_EXPLORER_HOST=http://localhost:5173
 VITE_OBP_CONSUMER_KEY=your_consumer_key

.gitignore

@@ -38,11 +38,14 @@ tsconfig.node.tsbuildinfo
 vite.config.d.ts
 vite.config.js
 vitest.config.d.ts
-vitest.config.js
 components.d.ts
 
 #keys
 *.pem
 private_key.pem
 public_key.pem
-./server/cert/*
+./server/cert/*
+
+
+.vite/deps
+__snapshots__/

Dockerfiles/frontend_build.env

@@ -1,5 +1,12 @@
 VITE_OBP_API_HOST=VITE_OBP_API_HOST
 VITE_OBP_API_PORTAL_HOST=VITE_OBP_API_PORTAL_HOST
 VITE_OBP_API_MANAGER_HOST=VITE_OBP_API_MANAGER_HOST
-VITE_OBP_API_VERSION=v5.1.0
-
+VITE_OBP_LOGO_URL=VITE_OBP_LOGO_URL
+VITE_OBP_API_VERSION=VITE_OBP_API_VERSION
+VITE_OBP_LINKS_COLOR=VITE_OBP_LINKS_COLOR
+VITE_OBP_HEADER_LINKS_COLOR=VITE_OBP_HEADER_LINKS_COLOR
+VITE_OBP_HEADER_LINKS_HOVER_COLOR=VITE_OBP_HEADER_LINKS_HOVER_COLOR
+VITE_OBP_HEADER_LINKS_BACKGROUND_COLOR=VITE_OBP_HEADER_LINKS_BACKGROUND_COLOR
+VITE_OBP_API_DEFAULT_RESOURCE_DOC_VERSION=VITE_OBP_API_DEFAULT_RESOURCE_DOC_VERSION
+VITE_CHATBOT_ENABLED=VITE_CHATBOT_ENABLED
+VITE_CHATBOT_URL=VITE_CHATBOT_URL

Dockerfiles/prestart.go

@@ -7,16 +7,35 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"fmt"
 )
 
+// As the frontend environment is read at build time, we need to reprocess the values
+// at container runtime.
+// This app will search and replace the values set at build time from this build environment: Dockerfiles/frontend_build.env
+// with values taken from the container environment.
+
 func main() {
-	// Define the host env variables to be replaced at build time
-	config := []string{"VITE_OBP_API_HOST", "VITE_OBP_API_MANAGER_HOST", "VITE_OBP_API_PORTAL_HOST"}
+	// Define the build env variables to be replaced at container run time
+	// url config variables are expected to be a valid URL in the container environment
+	url_config := []string{"VITE_OBP_API_HOST", "VITE_OBP_API_MANAGER_HOST", "VITE_OBP_API_PORTAL_HOST", "VITE_OBP_LOGO_URL"}
+	// DANGERZONE: The following strings will be replaced by container environment variables without any checking of whatever!!!
+	config := []string{"VITE_OBP_API_VERSION", "VITE_OBP_LINKS_COLOR", "VITE_OBP_HEADER_LINKS_COLOR", "VITE_OBP_HEADER_LINKS_HOVER_COLOR", "VITE_OBP_HEADER_LINKS_BACKGROUND_COLOR", "VITE_OBP_API_DEFAULT_RESOURCE_DOC_VERSION", "VITE_CHATBOT_ENABLED", "VITE_CHATBOT_URL"}
 	configMap := make(map[string]string)
 
 	for _, key := range config {
+	    value := os.Getenv(key)
+	    if value == "" {
+            fmt.Printf("Skipping: Environment variable %s is not set\n", key)
+            continue
+        }
+		configMap[key] = value
+	}
+
+	for _, key := range url_config {
 		rawURL := os.Getenv(key)
 		if rawURL == "" {
+		    fmt.Printf("Skipping: Environment variable %s is not set\n", key)
 			continue
 		}
 		cleanURL := checkURL(rawURL)

README.md

@@ -161,6 +161,13 @@ openssl rsa -in private_key.pem -pubout -out public_key.pem
 cp public_key.pem {path-to-your-opey-install}/
 ```
 
+# Building the frontend container
+
+As the frontend environment is read at build time, we need to reprocess the values 
+at container runtime. 
+This is done here: Dockerfiles/prestart.go
+overwriting the values set here: Dockerfiles/frontend_build.env
+Any newly introduced environment variables should be added to the prestart.go and frontend_build.env files accordingly.
 # LICENSE
 
 This project is licensed under the AGPL V3 (see NOTICE) and a commercial license from TESOBE.

components.d.ts

@@ -7,8 +7,9 @@ export {}
 /* prettier-ignore */
 declare module 'vue' {
   export interface GlobalComponents {
+    ChatMessage: typeof import('./src/components/ChatMessage.vue')['default']
     ChatWidget: typeof import('./src/components/ChatWidget.vue')['default']
-    ChatWidgetII: typeof import('./src/components/ChatWidgetII.vue')['default']
+    ChatWidgetOld: typeof import('./src/components/ChatWidgetOld.vue')['default']
     Collections: typeof import('./src/components/Collections.vue')['default']
     Content: typeof import('./src/components/Content.vue')['default']
     ElAlert: typeof import('element-plus/es')['ElAlert']
@@ -32,9 +33,8 @@ declare module 'vue' {
     ElIcon: typeof import('element-plus/es')['ElIcon']
     ElInput: typeof import('element-plus/es')['ElInput']
     ElMain: typeof import('element-plus/es')['ElMain']
-    ElMenu: typeof import('element-plus/es')['ElMenu']
-    ElMenuItem: typeof import('element-plus/es')['ElMenuItem']
     ElRow: typeof import('element-plus/es')['ElRow']
+    ElScrollbar: typeof import('element-plus/es')['ElScrollbar']
     ElTooltip: typeof import('element-plus/es')['ElTooltip']
     GlossarySearchNav: typeof import('./src/components/GlossarySearchNav.vue')['default']
     HeaderNav: typeof import('./src/components/HeaderNav.vue')['default']
@@ -45,7 +45,4 @@ declare module 'vue' {
     RouterView: typeof import('vue-router')['RouterView']
     SearchNav: typeof import('./src/components/SearchNav.vue')['default']
   }
-  export interface ComponentCustomProperties {
-    vLoading: typeof import('element-plus/es')['ElLoadingDirective']
-  }
 }

package.json

@@ -10,8 +10,7 @@
     "build": "run-p build-only",
     "build-server": "tsc --project tsconfig.server.json",
     "preview": "vite preview",
-    "test:unit": "vitest",
-    "test": "jest --silent=false",
+    "test": "vitest",
     "build-only": "vite build",
     "type-check": "vue-tsc --noEmit -p tsconfig.vitest.json --composite false",
     "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
@@ -35,9 +34,11 @@
     "highlight.js": "^11.8.0",
     "json-editor-vue": "^0.17.3",
     "jsonwebtoken": "^9.0.2",
+    "langchain": "^0.3.19",
     "markdown-it": "^14.1.0",
     "node-fetch": "^2.6.7",
     "oauth": "^0.10.0",
+    "obp-api-typescript": "^1.0.1",
     "obp-typescript": "^1.0.36",
     "pinia": "^2.0.37",
     "prismjs": "^1.29.0",
@@ -56,20 +57,24 @@
     "ws": "^8.18.0"
   },
   "devDependencies": {
+    "@ai-sdk/vue": "^1.1.18",
     "@rushstack/eslint-patch": "^1.4.0",
+    "@testing-library/vue": "^8.1.0",
     "@types/jest": "^29.5.14",
     "@types/jsdom": "^21.1.7",
     "@types/jsonwebtoken": "^9.0.6",
     "@types/markdown-it": "^14.1.1",
     "@types/node": "^20.5.7",
-    "@vitejs/plugin-vue": "^4.3.0",
+    "@types/oauth": "^0.9.6",
+    "@vitejs/plugin-vue": "^4.6.2",
     "@vitejs/plugin-vue-jsx": "^3.1.0",
     "@vue/eslint-config-prettier": "^9.0.0",
     "@vue/eslint-config-typescript": "^14.0.0",
     "@vue/test-utils": "^2.4.0",
     "@vue/tsconfig": "^0.1.3",
     "eslint": "^9.15.0",
     "eslint-plugin-vue": "^9.12.0",
+    "happy-dom": "^17.1.4",
     "jest": "^29.7.0",
     "jsdom": "^25.0.1",
     "node-mocks-http": "^1.16.2",
@@ -86,7 +91,10 @@
     "vite": "^4.4.0",
     "vite-plugin-node-polyfills": "^0.10.0",
     "vite-plugin-rewrite-all": "^1.0.2",
-    "vitest": "^0.34.0",
+    "vitest": "^0.34.6",
     "vue-tsc": "^2.0.0"
+  },
+  "overrides": {
+    "@langchain/core": "0.1.5"
   }
 }

server/controllers/OpeyIIController.ts

@@ -1,11 +1,13 @@
 import { Controller, Session, Req, Res, Post, Get } from 'routing-controllers'
 import { Request, Response } from 'express'
-import { pipeline } from "node:stream/promises"
+import { Transform, pipeline, Readable } from "node:stream"
+import { ReadableStream as WebReadableStream } from "stream/web"
 import { Service } from 'typedi'
 import OBPClientService from '../services/OBPClientService'
 import OpeyClientService from '../services/OpeyClientService'
 
 import { UserInput } from '../schema/OpeySchema'
+import { APIApi, Configuration, ConsentApi, ConsumerConsentrequestsBody, InlineResponse20151 } from 'obp-api-typescript'
 
 @Service()
 @Controller('/opey')
@@ -42,9 +44,11 @@ export class OpeyController {
         @Res() response: Response,
     ) {
 
+        // Read user input from request body
         let user_input: UserInput
         try {
-           user_input = {
+          console.log("Request body: ", request.body)
+          user_input = {
             "message": request.body.message,
             "thread_id": request.body.thread_id,
             "is_tool_call_approval": request.body.is_tool_call_approval
@@ -54,45 +58,73 @@ export class OpeyController {
           return response.status(500).json({ error: 'Internal Server Error' })
         }
 
-        
-        console.log("Calling OpeyClientService.stream")
 
-        // const streamMiddlewareTransform = new Transform({
-        //   transform(chunk, encoding, callback) {
-        //     console.log(`Logged Chunk: ${chunk}`)
-        //     this.push(chunk);
-        
-        //     callback();
-        //   }
-        // })
+        // Define a function to transform the response from Opey (which is a text stream) into a TS-Native langchain stream
+        const frontendTransformer = new TransformStream({
+          transform(chunk, controller) {
+            // Decode the chunk to a string
+            const decodedChunk = new TextDecoder().decode(chunk)
+          
+            console.log("Sending chunk", decodedChunk)
+            controller.enqueue(decodedChunk);
+          },
+          flush(controller) {
+            console.log('[flush]');
+            // Close ReadableStream when done
+            controller.terminate();
+          },
+        });
+
 
-        let stream: NodeJS.ReadableStream | null = null
+        let stream: ReadableStream | null = null
 
         try {
-          // Read stream from OpeyClientService
+          // Read web stream from OpeyClientService
+          console.log("Calling OpeyClientService.stream")
           stream = await this.opeyClientService.stream(user_input)
-          console.debug(`Stream received readable: ${stream?.readable}`)
 
         } catch (error) {
           console.error("Error reading stream: ", error)
           return response.status(500).json({ error: 'Internal Server Error' })
         }
 
-        if (!stream || !stream.readable) {
+        if (!stream) {
           console.error("Stream is not recieved or not readable")
           return response.status(500).json({ error: 'Internal Server Error' })
         }
 
+        
+        // Transform our stream if needed, right now this is just a passthrough
+        const frontendStream: ReadableStream = stream.pipeThrough(frontendTransformer)
+        
+        // If we need to split the stream into two, we can use the tee method as below 
+
+        // const streamTee = langchainStream.tee()
+        // if (!streamTee) {
+        //   console.error("Stream is not tee'd")
+        //   return response.status(500).json({ error: 'Internal Server Error' })
+        // }
+        // const [stream1, stream2] = streamTee
+
+        
+
+        const nodeStream = Readable.fromWeb(frontendStream as WebReadableStream<any>)
+
+        response.setHeader('x-vercel-ai-data-stream', 'v1')
+        response.setHeader('Content-Type', 'text/event-stream');
+        response.setHeader('Cache-Control', 'no-cache');
+        response.setHeader('Connection', 'keep-alive');
+        nodeStream.pipe(response);
+      
+
         return new Promise<Response>((resolve, reject) => {
-          stream.pipe(response)
-          stream.on('end', () => {
-            response.status(200)
-            resolve(response)
-          })
-          stream.on('error', (error) => {
-            console.error("Error piping stream: ", error)
-            reject(error)
-          })
+          nodeStream.on('end', () => {
+            resolve(response);
+          });
+          nodeStream.on('error', (error) => {
+            console.error('Stream error:', error);
+            reject(error);
+          });
 
         })
 
@@ -129,6 +161,67 @@ export class OpeyController {
         }
     }
 
+    @Post('/consent/request')
+    /**
+     * Retrieves a consent request from OBP
+     * 
+     */
+    async getConsentRequest(
+        @Session() session: any,
+        @Req() request: Request,
+        @Res() response: Response,
+    ): Promise<Response | any> {
+      try {
+
+        let obpToken: string
+
+        obpToken = await this.obpClientService.getDirectLoginToken()
+        console.log("Got token: ", obpToken)
+        const authHeader = `DirectLogin token="${obpToken}"`
+        console.log("Auth header: ", authHeader)
+
+        const obpOAuthHeaders = await this.obpClientService.getOAuthHeader('/consents', 'POST')
+        console.log("OBP OAuth Headers: ", obpOAuthHeaders)
+
+        const obpConfig: Configuration = {
+          apiKey: authHeader,
+          basePath: process.env.VITE_OBP_API_HOST,
+        }
+
+        console.log("OBP Config: ", obpConfig)
+
+        const consentAPI = new ConsentApi(obpConfig, process.env.VITE_OBP_API_HOST)
+        
+
+        // OBP sdk naming is a bit mad, can be rectified in the future
+        const consentRequestResponse = await consentAPI.oBPv500CreateConsentRequest({
+            accountAccess: [],
+            everything: false,
+            entitlements: [], 
+            consumerId: '',
+          } as unknown as ConsumerConsentrequestsBody,
+          {
+            headers: {
+              'Content-Type': 'application/json',
+            },
+          }
+        )
+
+        //console.log("Consent request response: ", consentRequestResponse)
+        
+        console.log({consentId: consentRequestResponse.data.consent_request_id})
+        session['obpConsentRequestId'] = consentRequestResponse.data.consent_request_id
+
+        return response.status(200).json(JSON.stringify({consentId: consentRequestResponse.data.consent_request_id}))
+        //console.log(await response.body.json())
+        
+
+      } catch (error) {
+        console.error("Error in consent/request endpoint: ", error);
+        return response.status(500).json({ error: 'Internal Server Error' });
+      }
+    }
+
     @Post('/consent')
     /**
     * Retrieves a consent from OBP for the current user

server/schema/OpeySchema.ts

@@ -22,4 +22,8 @@ export type OpeyConfig = {
 export type AuthConfig = {
     consentId: string,
     opeyJWT: string,
+}
+
+export interface ConsentRequestResponse {
+    consentId: string;
 }