From 4bd62c693ac8e414bd68146039e5e35b501c38e4 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 24 Apr 2025 18:12:09 +0200 Subject: [PATCH 01/47] core: add ancient ci, trunk-check and a test ci --- .github/workflows/ci.yml | 130 +++++++++++++++++++++++++++++ .github/workflows/test.yml | 29 +++++++ .github/workflows/trunk-check.yaml | 22 +++++ 3 files changed, 181 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/trunk-check.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..a253424c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,130 @@ +name: CI + +on: + push: + branches: [cairo-chatbot, main] + pull_request: + branches: [cairo-chatbot, main] + +permissions: + contents: read + pull-requests: write + +jobs: + build: + name: Build + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Setup pnpm + uses: pnpm/action-setup@v3 + with: + version: 9.10.0 + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Setup pnpm cache + uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + run: pnpm install + + - name: Install turbo + run: pnpm add turbo@latest -g + - name: Build + run: turbo build + + test: + name: Test + runs-on: ubuntu-latest + needs: build + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Setup pnpm + uses: pnpm/action-setup@v3 + with: + version: 9.10.0 + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Setup pnpm cache + uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + run: pnpm install + + - name: Install turbo + run: pnpm add turbo@latest -g + + - name: Create config file + run: | + mkdir -p packages/agents + cat > packages/agents/config.toml << 'EOL' + [API_KEYS] + OPENAI = "${{ secrets.OPENAI_API_KEY }}" + GROQ = "${{ secrets.GROQ_API_KEY }}" + ANTHROPIC = "${{ secrets.ANTHROPIC_API_KEY }}" + DEEPSEEK = "${{ secrets.DEEPSEEK_API_KEY }}" + GEMINI = "${{ secrets.GEMINI_API_KEY }}" + + [API_ENDPOINTS] + OLLAMA = "${{ secrets.OLLAMA_ENDPOINT }}" + + [VECTOR_DB] + MONGODB_URI = "${{ secrets.MONGODB_URI }}" + DB_NAME = "${{ secrets.DB_NAME }}" + COLLECTION_NAME = "${{ secrets.COLLECTION_NAME }}" + + [GENERAL] + PORT = 3001 + SIMILARITY_MEASURE = "cosine" + + [HOSTED_MODE] + DEFAULT_CHAT_PROVIDER = "${{ secrets.DEFAULT_CHAT_PROVIDER }}" + DEFAULT_CHAT_MODEL = "${{ secrets.DEFAULT_CHAT_MODEL }}" + DEFAULT_FAST_CHAT_PROVIDER = "${{ secrets.DEFAULT_FAST_CHAT_PROVIDER }}" + DEFAULT_FAST_CHAT_MODEL = "${{ secrets.DEFAULT_FAST_CHAT_MODEL }}" + DEFAULT_EMBEDDING_PROVIDER = "${{ secrets.DEFAULT_EMBEDDING_PROVIDER }}" + DEFAULT_EMBEDDING_MODEL = "${{ secrets.DEFAULT_EMBEDDING_MODEL }}" + + [VERSIONS] + STARKNET_FOUNDRY = "${{ secrets.STARKNET_FOUNDRY_VERSION }}" + SCARB = "${{ secrets.SCARB_VERSION }}" + EOL + + - name: Run tests + run: turbo test diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..68f4edf7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: test + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + + +permissions: + pull-requests: write + + +jobs: + my-job: + name: My Job + runs-on: ubuntu-latest + steps: + - name: Print a greeting + env: + MY_VAR: Hi there! My name is + FIRST_NAME: Mona + MIDDLE_NAME: The + LAST_NAME: Octocat + run: | + echo $MY_VAR $FIRST_NAME $MIDDLE_NAME $LAST_NAME. + + diff --git a/.github/workflows/trunk-check.yaml b/.github/workflows/trunk-check.yaml new file mode 100644 index 00000000..474e33a1 --- /dev/null +++ b/.github/workflows/trunk-check.yaml @@ -0,0 +1,22 @@ +name: Pull Request +on: [pull_request] +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: read-all + +jobs: + trunk_check: + name: Trunk Code Quality Runner + runs-on: ubuntu-latest + permissions: + checks: write # For trunk to post annotations + contents: read # For repo checkout + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Trunk Code Quality + uses: trunk-io/trunk-action@v1 From 0f196b4445515988b7b432dc7b75d6d5646d12da Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 24 Apr 2025 18:42:42 +0200 Subject: [PATCH 02/47] fix: test.yml workflow_dispatch trigger event --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68f4edf7..7636e770 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,11 +1,11 @@ name: test on: - push: - branches: [main] - pull_request: - branches: [main] - workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: permissions: From 75bf2e31123d412d969e413c8e016a0f75a3ea98 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 24 Apr 2025 20:44:55 +0200 Subject: [PATCH 03/47] core: update ci with new config files --- .github/workflows/ci.yml | 25 ++++++++++++------- .github/workflows/generate-embeddings.yml | 25 ++++++++++++------- .github/workflows/test.yml | 29 ----------------------- .github/workflows/trunk-check.yaml | 2 +- 4 files changed, 33 insertions(+), 48 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a253424c..42752bd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,18 +96,14 @@ jobs: cat > packages/agents/config.toml << 'EOL' [API_KEYS] OPENAI = "${{ secrets.OPENAI_API_KEY }}" - GROQ = "${{ secrets.GROQ_API_KEY }}" ANTHROPIC = "${{ secrets.ANTHROPIC_API_KEY }}" - DEEPSEEK = "${{ secrets.DEEPSEEK_API_KEY }}" - GEMINI = "${{ secrets.GEMINI_API_KEY }}" - - [API_ENDPOINTS] - OLLAMA = "${{ secrets.OLLAMA_ENDPOINT }}" [VECTOR_DB] - MONGODB_URI = "${{ secrets.MONGODB_URI }}" - DB_NAME = "${{ secrets.DB_NAME }}" - COLLECTION_NAME = "${{ secrets.COLLECTION_NAME }}" + POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" + POSTGRES_HOST = "postgres" + POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" + POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" + POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" [GENERAL] PORT = 3001 @@ -125,6 +121,17 @@ jobs: STARKNET_FOUNDRY = "${{ secrets.STARKNET_FOUNDRY_VERSION }}" SCARB = "${{ secrets.SCARB_VERSION }}" EOL + + - name: Create env file + run: | + mkdir -p packages/agents + cat > packages/agents/.env << 'EOL' + POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" + POSTGRES_HOST = "localhost" + POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" + POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" + POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" + EOL - name: Run tests run: turbo test diff --git a/.github/workflows/generate-embeddings.yml b/.github/workflows/generate-embeddings.yml index d6f66cd0..658ea97a 100644 --- a/.github/workflows/generate-embeddings.yml +++ b/.github/workflows/generate-embeddings.yml @@ -70,18 +70,14 @@ jobs: cat > packages/agents/config.toml << 'EOL' [API_KEYS] OPENAI = "${{ secrets.OPENAI_API_KEY }}" - GROQ = "${{ secrets.GROQ_API_KEY }}" ANTHROPIC = "${{ secrets.ANTHROPIC_API_KEY }}" - DEEPSEEK = "${{ secrets.DEEPSEEK_API_KEY }}" - GEMINI = "${{ secrets.GEMINI_API_KEY }}" - - [API_ENDPOINTS] - OLLAMA = "${{ secrets.OLLAMA_ENDPOINT }}" [VECTOR_DB] - MONGODB_URI = "${{ secrets.MONGODB_URI }}" - DB_NAME = "${{ secrets.DB_NAME }}" - COLLECTION_NAME = "${{ secrets.COLLECTION_NAME }}" + POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" + POSTGRES_HOST = "postgres" + POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" + POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" + POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" [GENERAL] PORT = 3001 @@ -100,5 +96,16 @@ jobs: SCARB = "${{ secrets.SCARB_VERSION }}" EOL + - name: Create env file + run: | + mkdir -p packages/agents + cat > packages/agents/.env << 'EOL' + POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" + POSTGRES_HOST = "localhost" + POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" + POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" + POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" + EOL + - name: Generate embeddings run: turbo generate-embeddings:yes diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 7636e770..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: test - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - workflow_dispatch: - - -permissions: - pull-requests: write - - -jobs: - my-job: - name: My Job - runs-on: ubuntu-latest - steps: - - name: Print a greeting - env: - MY_VAR: Hi there! My name is - FIRST_NAME: Mona - MIDDLE_NAME: The - LAST_NAME: Octocat - run: | - echo $MY_VAR $FIRST_NAME $MIDDLE_NAME $LAST_NAME. - - diff --git a/.github/workflows/trunk-check.yaml b/.github/workflows/trunk-check.yaml index 474e33a1..322ca59a 100644 --- a/.github/workflows/trunk-check.yaml +++ b/.github/workflows/trunk-check.yaml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Trunk Code Quality uses: trunk-io/trunk-action@v1 From 6270a7b1d33cbf5a12242ff28e68d0018875ffc8 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 24 Apr 2025 21:03:55 +0200 Subject: [PATCH 04/47] chore: clean up trunk files --- .trunk/configs/.hadolint.yaml | 4 ++++ .trunk/configs/.markdownlint.yaml | 2 ++ .trunk/configs/.yamllint.yaml | 5 ++++ .trunk/configs/svgo.config.mjs | 14 +++++++++++ .trunk/trunk.yaml | 40 +++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+) create mode 100644 .trunk/configs/.hadolint.yaml create mode 100644 .trunk/configs/.markdownlint.yaml create mode 100644 .trunk/configs/.yamllint.yaml create mode 100644 .trunk/configs/svgo.config.mjs create mode 100644 .trunk/trunk.yaml diff --git a/.trunk/configs/.hadolint.yaml b/.trunk/configs/.hadolint.yaml new file mode 100644 index 00000000..98bf0cd2 --- /dev/null +++ b/.trunk/configs/.hadolint.yaml @@ -0,0 +1,4 @@ +# Following source doesn't work in most setups +ignored: + - SC1090 + - SC1091 diff --git a/.trunk/configs/.markdownlint.yaml b/.trunk/configs/.markdownlint.yaml new file mode 100644 index 00000000..b40ee9d7 --- /dev/null +++ b/.trunk/configs/.markdownlint.yaml @@ -0,0 +1,2 @@ +# Prettier friendly markdownlint config (all formatting rules disabled) +extends: markdownlint/style/prettier diff --git a/.trunk/configs/.yamllint.yaml b/.trunk/configs/.yamllint.yaml new file mode 100644 index 00000000..841de67e --- /dev/null +++ b/.trunk/configs/.yamllint.yaml @@ -0,0 +1,5 @@ +rules: + quoted-strings: disable + key-duplicates: {} + octal-values: + forbid-implicit-octal: true diff --git a/.trunk/configs/svgo.config.mjs b/.trunk/configs/svgo.config.mjs new file mode 100644 index 00000000..b86ef082 --- /dev/null +++ b/.trunk/configs/svgo.config.mjs @@ -0,0 +1,14 @@ +export default { + plugins: [ + { + name: 'preset-default', + params: { + overrides: { + removeViewBox: false, // https://github.com/svg/svgo/issues/1128 + sortAttrs: true, + removeOffCanvasPaths: true, + }, + }, + }, + ], +}; diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml new file mode 100644 index 00000000..a9c87935 --- /dev/null +++ b/.trunk/trunk.yaml @@ -0,0 +1,40 @@ +# This file controls the behavior of Trunk: https://docs.trunk.io/cli +# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml +version: 0.1 +cli: + version: 1.22.10 +# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) +plugins: + sources: + - id: trunk + ref: v1.6.7 + uri: https://github.com/trunk-io/plugins +# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) +runtimes: + enabled: + - node@18.20.5 + - python@3.10.8 +# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) +lint: + ignore: + - linters: [ALL] + paths: + - '*.dockerfile' + enabled: + - checkov@3.2.370 + - git-diff-check + - hadolint@2.12.1-beta + - markdownlint@0.44.0 + - osv-scanner@1.9.2 + - oxipng@9.1.3 + - prettier@3.5.1 + - svgo@3.3.2 + - taplo@0.9.3 + - trufflehog@3.88.8 + - yamllint@1.35.1 +actions: + enabled: + - trunk-announce + - trunk-check-pre-push + - trunk-fmt-pre-commit + - trunk-upgrade-available From e02635913e2dad6460aef5425e644b7e70e6cdde Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 24 Apr 2025 21:26:00 +0200 Subject: [PATCH 05/47] fix: trunk format --- .github/workflows/ci.yml | 2 +- .github/workflows/generate-embeddings.yml | 2 +- .trunk/.gitignore | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 .trunk/.gitignore diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42752bd8..7a292040 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,7 @@ jobs: STARKNET_FOUNDRY = "${{ secrets.STARKNET_FOUNDRY_VERSION }}" SCARB = "${{ secrets.SCARB_VERSION }}" EOL - + - name: Create env file run: | mkdir -p packages/agents diff --git a/.github/workflows/generate-embeddings.yml b/.github/workflows/generate-embeddings.yml index 658ea97a..5b800a8a 100644 --- a/.github/workflows/generate-embeddings.yml +++ b/.github/workflows/generate-embeddings.yml @@ -106,6 +106,6 @@ jobs: POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" EOL - + - name: Generate embeddings run: turbo generate-embeddings:yes diff --git a/.trunk/.gitignore b/.trunk/.gitignore new file mode 100644 index 00000000..10fcf035 --- /dev/null +++ b/.trunk/.gitignore @@ -0,0 +1,12 @@ +*out +*logs +*actions +*notifications +*tools +plugins +user_trunk.yaml +user.yaml +tmp + +!.trunk/trunk.yaml +!.trunk/configs \ No newline at end of file From 819a8c26ecd58ba128438055700d1359c1a129ed Mon Sep 17 00:00:00 2001 From: alvinouille Date: Fri, 25 Apr 2025 12:06:02 +0200 Subject: [PATCH 06/47] fix: unit test --- .github/workflows/ci.yml | 4 ++-- packages/agents/__tests__/ragAgentFactory.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a292040..fecb62cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [cairo-chatbot, main] + branches: [main] pull_request: - branches: [cairo-chatbot, main] + branches: [main] permissions: contents: read diff --git a/packages/agents/__tests__/ragAgentFactory.test.ts b/packages/agents/__tests__/ragAgentFactory.test.ts index 60725fa8..0e6ecdff 100644 --- a/packages/agents/__tests__/ragAgentFactory.test.ts +++ b/packages/agents/__tests__/ragAgentFactory.test.ts @@ -3,7 +3,7 @@ import { RagPipeline } from '../src/core/pipeline/ragPipeline'; import { AvailableAgents, LLMConfig, DocumentSource } from '../src/types'; import { Embeddings } from '@langchain/core/embeddings'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import { VectorStore } from '../src/db/vectorStore'; +import { VectorStore } from '../src/db/postgresVectorStore'; import { mockDeep, MockProxy } from 'jest-mock-extended'; import { BaseMessage } from '@langchain/core/messages'; import EventEmitter from 'events'; @@ -103,7 +103,7 @@ describe('RagAgentFactory', () => { // Assert expect(RagPipeline).toHaveBeenCalledTimes(1); expect(emitter).toBeInstanceOf(EventEmitter); - + // Check streaming option is passed const executeSpy = (RagPipeline as jest.Mock).mock.results[0].value .execute; From 210cc5e6e28f54acc700c86a8cd6cd6fca38b74b Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 28 Apr 2025 22:33:12 +0200 Subject: [PATCH 07/47] testdocquality available and add basic integration test ci --- .github/workflows/{ci.yml => backend.yml} | 87 ++- .../{generate-embeddings.yml => ingester.yml} | 17 +- backend.dockerfile | 3 + docker-compose.yml | 10 +- packages/agents/package.json | 2 +- packages/agents/src/tests/testDocQuality.ts | 632 ++++++++---------- packages/agents/src/types/index.ts | 2 +- scripts/database-connection.sh | 30 + scripts/integration-tests.sh | 38 ++ 9 files changed, 419 insertions(+), 402 deletions(-) rename .github/workflows/{ci.yml => backend.yml} (60%) rename .github/workflows/{generate-embeddings.yml => ingester.yml} (83%) create mode 100755 scripts/database-connection.sh create mode 100755 scripts/integration-tests.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/backend.yml similarity index 60% rename from .github/workflows/ci.yml rename to .github/workflows/backend.yml index fecb62cf..622424bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/backend.yml @@ -48,48 +48,10 @@ jobs: - name: Install turbo run: pnpm add turbo@latest -g + - name: Build run: turbo build - test: - name: Test - runs-on: ubuntu-latest - needs: build - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Setup pnpm - uses: pnpm/action-setup@v3 - with: - version: 9.10.0 - - - name: Get pnpm store directory - id: pnpm-cache - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT - - - name: Setup pnpm cache - uses: actions/cache@v4 - with: - path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install dependencies - run: pnpm install - - - name: Install turbo - run: pnpm add turbo@latest -g - - name: Create config file run: | mkdir -p packages/agents @@ -97,6 +59,7 @@ jobs: [API_KEYS] OPENAI = "${{ secrets.OPENAI_API_KEY }}" ANTHROPIC = "${{ secrets.ANTHROPIC_API_KEY }}" + GEMINI = "${{ secrets.GEMINI_API_KEY }}" [VECTOR_DB] POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" @@ -110,16 +73,16 @@ jobs: SIMILARITY_MEASURE = "cosine" [HOSTED_MODE] - DEFAULT_CHAT_PROVIDER = "${{ secrets.DEFAULT_CHAT_PROVIDER }}" - DEFAULT_CHAT_MODEL = "${{ secrets.DEFAULT_CHAT_MODEL }}" - DEFAULT_FAST_CHAT_PROVIDER = "${{ secrets.DEFAULT_FAST_CHAT_PROVIDER }}" - DEFAULT_FAST_CHAT_MODEL = "${{ secrets.DEFAULT_FAST_CHAT_MODEL }}" - DEFAULT_EMBEDDING_PROVIDER = "${{ secrets.DEFAULT_EMBEDDING_PROVIDER }}" - DEFAULT_EMBEDDING_MODEL = "${{ secrets.DEFAULT_EMBEDDING_MODEL }}" + DEFAULT_CHAT_PROVIDER = "anthropic" + DEFAULT_CHAT_MODEL = "Claude 3.5 Sonnet" + DEFAULT_FAST_CHAT_PROVIDER = "anthropic" + DEFAULT_FAST_CHAT_MODEL = "Claude 3.5 Sonnet" + DEFAULT_EMBEDDING_PROVIDER = "openai" + DEFAULT_EMBEDDING_MODEL = "Text embedding 3 large" [VERSIONS] - STARKNET_FOUNDRY = "${{ secrets.STARKNET_FOUNDRY_VERSION }}" - SCARB = "${{ secrets.SCARB_VERSION }}" + STARKNET_FOUNDRY = "0.37.0" + SCARB = "2.9.2" EOL - name: Create env file @@ -135,3 +98,33 @@ jobs: - name: Run tests run: turbo test + + - name: Build docker image + run: docker build -t cairo-coder-backend:${{ github.sha }} -f backend.dockerfile . + + - name: Integration docker connection test with Docker Compose + run: | + docker-compose up -d postgres backend + echo "Waiting for services to be ready..." + sleep 20 + + chmod +x ./scripts/integration-tests.sh + chmod +x ./scripts/test-chat-completions.sh + + echo -e "\n=== Running basic integration tests ===" + ./scripts/integration-tests.sh + INTEGRATION_RESULT=$? + + echo -e "\n=== Running database connection test via chat/completions endpoint ===" + ./scripts/test-chat-completions.sh + DB_CONNECTION_RESULT=$? + + if [ $INTEGRATION_RESULT -ne 0 ] || [ $DB_CONNECTION_RESULT -ne 0 ]; then + echo "❌ Integration tests failed!" + exit 1 + else + echo "✅ All integration tests passed!" + fi + + # - name: Push docker image + # run: docker push ${{ github.repository }}:${{ github.sha }} diff --git a/.github/workflows/generate-embeddings.yml b/.github/workflows/ingester.yml similarity index 83% rename from .github/workflows/generate-embeddings.yml rename to .github/workflows/ingester.yml index 5b800a8a..1edfeaa4 100644 --- a/.github/workflows/generate-embeddings.yml +++ b/.github/workflows/ingester.yml @@ -71,6 +71,7 @@ jobs: [API_KEYS] OPENAI = "${{ secrets.OPENAI_API_KEY }}" ANTHROPIC = "${{ secrets.ANTHROPIC_API_KEY }}" + GEMINI = "${{ secrets.GEMINI_API_KEY }}" [VECTOR_DB] POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" @@ -84,16 +85,16 @@ jobs: SIMILARITY_MEASURE = "cosine" [HOSTED_MODE] - DEFAULT_CHAT_PROVIDER = "${{ secrets.DEFAULT_CHAT_PROVIDER }}" - DEFAULT_CHAT_MODEL = "${{ secrets.DEFAULT_CHAT_MODEL }}" - DEFAULT_FAST_CHAT_PROVIDER = "${{ secrets.DEFAULT_FAST_CHAT_PROVIDER }}" - DEFAULT_FAST_CHAT_MODEL = "${{ secrets.DEFAULT_FAST_CHAT_MODEL }}" - DEFAULT_EMBEDDING_PROVIDER = "${{ secrets.DEFAULT_EMBEDDING_PROVIDER }}" - DEFAULT_EMBEDDING_MODEL = "${{ secrets.DEFAULT_EMBEDDING_MODEL }}" + DEFAULT_CHAT_PROVIDER = "anthropic" + DEFAULT_CHAT_MODEL = "Claude 3.5 Sonnet" + DEFAULT_FAST_CHAT_PROVIDER = "anthropic" + DEFAULT_FAST_CHAT_MODEL = "Claude 3.5 Sonnet" + DEFAULT_EMBEDDING_PROVIDER = "openai" + DEFAULT_EMBEDDING_MODEL = "Text embedding 3 large" [VERSIONS] - STARKNET_FOUNDRY = "${{ secrets.STARKNET_FOUNDRY_VERSION }}" - SCARB = "${{ secrets.SCARB_VERSION }}" + STARKNET_FOUNDRY = "0.37.0" + SCARB = "2.9.2" EOL - name: Create env file diff --git a/backend.dockerfile b/backend.dockerfile index 1f90a220..adc71b0f 100644 --- a/backend.dockerfile +++ b/backend.dockerfile @@ -2,6 +2,9 @@ FROM node:23.7-bullseye-slim WORKDIR /app +# Installation de ping pour tester la connectivité réseau +RUN apt-get update && apt-get install -y iputils-ping && rm -rf /var/lib/apt/lists/* + # Copy root workspace files COPY pnpm-workspace.yaml ./ COPY package.json ./ diff --git a/docker-compose.yml b/docker-compose.yml index 5db2513c..d23e6113 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,7 @@ -version: '3.8' - services: postgres: image: pgvector/pgvector:pg17 - container_name: "postgresql" + container_name: 'postgresql' shm_size: 1g env_file: - .env @@ -16,6 +14,7 @@ services: - cairo_coder backend: + container_name: 'backend' build: context: . dockerfile: backend.dockerfile @@ -29,12 +28,12 @@ services: restart: unless-stopped networks: - cairo_coder - + ingester: build: context: . dockerfile: ingest.dockerfile - profiles: ["ingester"] + profiles: ['ingester'] depends_on: postgres: condition: service_started @@ -46,4 +45,3 @@ networks: volumes: postgres_data: - diff --git a/packages/agents/package.json b/packages/agents/package.json index 156ec76f..076c2685 100644 --- a/packages/agents/package.json +++ b/packages/agents/package.json @@ -5,7 +5,7 @@ "scripts": { "build": "tsc -p tsconfig.json", "test": "jest", - "test-doc-quality": "ts-node src/testDocQuality.ts" + "test-doc-quality": "ts-node src/tests/testDocQuality.ts" }, "dependencies": { "@iarna/toml": "^2.2.5", diff --git a/packages/agents/src/tests/testDocQuality.ts b/packages/agents/src/tests/testDocQuality.ts index 0e08d93e..2048f3b6 100644 --- a/packages/agents/src/tests/testDocQuality.ts +++ b/packages/agents/src/tests/testDocQuality.ts @@ -1,339 +1,293 @@ -// import fs from 'fs'; -// import path from 'path'; -// import { Command } from 'commander'; -// import { BaseChatModel } from '@langchain/core/language_models/chat_models'; -// import { OpenAIEmbeddings } from '@langchain/openai'; -// import { DocumentSource, RagSearchConfig } from './core/types'; -// import { DocQualityTester } from './pipeline/docQualityTester'; -// import { -// AvailableAgents, -// getAgentConfig, -// LLMConfig, -// } from './config/agentConfigs'; -// import logger from './utils/logger'; -// import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; -// import { VectorStore } from './db/vectorStore'; -// import { getGeminiApiKey, getOpenaiApiKey, getVectorDbConfig } from './config'; -// import { Embeddings } from '@langchain/core/embeddings'; - -// const program = new Command(); - -// // Initialize the program -// program -// .name('test-doc-quality') -// .description('Test documentation quality using the Starknet Agent') -// .version('1.0.0'); - -// program -// .command('test') -// .description('Run documentation quality tests') -// .requiredOption( -// '-s, --source ', -// 'Documentation source to test (e.g., starknet_docs)', -// ) -// .requiredOption('-t, --test-file ', 'Path to test file (JSON)') -// .option('-o, --output ', 'Path to output file (JSON)') -// .option( -// '-m, --model ', -// 'LLM model to use for testing', -// 'Claude 3.5 Sonnet', -// ) -// .option( -// '-e, --eval-model ', -// 'LLM model to use for evaluation (defaults to same as model)', -// ) -// .option( -// '--no-detailed-output', -// 'Disable detailed test output with PASS/FAIL status', -// ) -// .option( -// '--thresholds ', -// 'Custom thresholds for determining pass/fail status (JSON string)', -// ) -// .action(async (options) => { -// try { -// // Validate source -// const source = options.source as DocumentSource; -// const focus = source as string; - -// // Load test file -// const testFilePath = path.resolve(process.cwd(), options.testFile); -// if (!fs.existsSync(testFilePath)) { -// logger.error(`Test file not found: ${testFilePath}`); -// process.exit(1); -// } - -// const testFileContent = fs.readFileSync(testFilePath, 'utf-8'); -// const testSet = JSON.parse(testFileContent); - -// const geminiApiKey = getGeminiApiKey(); -// const openaiApiKey = getOpenaiApiKey(); - -// // Initialize models and embeddings -// const defaultLLM = new ChatGoogleGenerativeAI({ -// temperature: 0.7, -// apiKey: geminiApiKey, -// modelName: 'gemini-2.0-flash', -// }); - -// const llmConfig = { -// defaultLLM: defaultLLM as unknown as BaseChatModel, -// fastLLM: defaultLLM as unknown as BaseChatModel, -// evaluationLLM: defaultLLM as unknown as BaseChatModel, -// }; - -// const embeddings = new OpenAIEmbeddings({ -// openAIApiKey: openaiApiKey, -// modelName: 'text-embedding-3-large', -// dimensions: 2048, -// }) as unknown as Embeddings; - -// // Initialize vector store -// const dbConfig = getVectorDbConfig(); -// const vectorStore = await VectorStore.getInstance(dbConfig, embeddings); - -// const source_to_agent_name: Record< -// DocumentSource | 'starknet_ecosystem', -// AvailableAgents -// > = { -// [DocumentSource.CAIRO_BOOK]: 'cairoBook', -// [DocumentSource.STARKNET_DOCS]: 'starknetDocs', -// starknet_ecosystem: 'starknetEcosystem', -// [DocumentSource.STARKNET_FOUNDRY]: 'starknetFoundry', -// [DocumentSource.CAIRO_BY_EXAMPLE]: 'cairoByExample', -// [DocumentSource.OPENZEPPELIN_DOCS]: 'openZeppelinDocs', -// }; -// // Get agent configuration -// const agentConfig = getAgentConfig( -// source_to_agent_name[source], -// vectorStore, -// ); -// if (!agentConfig) { -// logger.error(`Agent configuration not found for source: ${source}`); -// process.exit(1); -// } - -// // Create RAG config -// const ragConfig: RagSearchConfig = { -// ...agentConfig, -// vectorStore, -// sources: agentConfig.sources, -// }; - -// // Initialize DocQualityTester -// const tester = new DocQualityTester(llmConfig, embeddings, ragConfig); - -// // Parse thresholds if provided -// let thresholds = undefined; -// if (options.thresholds) { -// try { -// thresholds = JSON.parse(options.thresholds); -// } catch (e) { -// logger.error(`Error parsing thresholds JSON: ${e}`); -// process.exit(1); -// } -// } - -// // Run tests with detailed output options -// logger.info(`Starting documentation quality tests for focus ${focus}`); -// const results = await tester.testDocQuality(testSet, focus, { -// showDetailedOutput: options.detailedOutput, -// thresholds, -// }); - -// // Generate report -// const report = await tester.generateReport(results); - -// // Output results -// if (options.output) { -// const outputPath = path.resolve(process.cwd(), options.output); -// fs.writeFileSync(outputPath, JSON.stringify(report, null, 2)); -// logger.info(`Report saved to ${outputPath}`); -// } else { -// console.log('\nDocumentation Quality Report'); -// console.log('===========================\n'); -// console.log(`Focus: ${report.results.focus}`); -// console.log(`Version: ${report.results.version}`); -// console.log(`Test Cases: ${report.results.caseResults.length}`); -// console.log('\nSummary:'); -// console.log(report.summary); - -// console.log('\nKey Metrics:'); -// console.log( -// `- Relevance Score: ${report.results.metrics.overall.percentAnswered.toFixed(2)}`, -// ); -// console.log( -// `- Coverage Score: ${report.results.metrics.overall.avgClarityScore.toFixed(2)}`, -// ); -// console.log( -// `- Answer Completeness: ${report.results.metrics.overall.avgSourceAlignment.toFixed(2)}`, -// ); - -// console.log('\nTop Recommendations:'); -// const highPriorityRecs = report.recommendations.filter( -// (r) => r.priority === 'high', -// ); -// highPriorityRecs.forEach((rec, i) => { -// console.log(`${i + 1}. ${rec.description}`); -// }); - -// console.log( -// '\nFor full report, use the --output option to save to file.', -// ); -// } -// } catch (error) { -// logger.error('Error running documentation quality tests:', error); -// process.exit(1); -// } -// }); - -// program -// .command('compare') -// .description('Compare documentation quality between versions') -// .requiredOption('-s, --source ', 'Documentation source to test') -// .requiredOption( -// '-b, --baseline ', -// 'Path to baseline results file (JSON)', -// ) -// .requiredOption('-c, --current ', 'Path to current results file (JSON)') -// .option('-o, --output ', 'Path to output file (JSON)') -// .option( -// '-m, --model ', -// 'LLM model to use for comparison', -// 'Claude 3.5 Sonnet', -// ) -// .action(async (options) => { -// try { -// // Validate source -// const focus = options.source as DocumentSource; - -// // Load result files -// const baselinePath = path.resolve(process.cwd(), options.baseline); -// const currentPath = path.resolve(process.cwd(), options.current); - -// if (!fs.existsSync(baselinePath)) { -// logger.error(`Baseline file not found: ${baselinePath}`); -// process.exit(1); -// } - -// if (!fs.existsSync(currentPath)) { -// logger.error(`Current file not found: ${currentPath}`); -// process.exit(1); -// } - -// const baselineContent = fs.readFileSync(baselinePath, 'utf-8'); -// const currentContent = fs.readFileSync(currentPath, 'utf-8'); - -// const baseline = JSON.parse(baselineContent); -// const current = JSON.parse(currentContent); - -// const geminiApiKey = getGeminiApiKey(); -// const openaiApiKey = getOpenaiApiKey(); - -// // Initialize models and embeddings -// const defaultLLM = new ChatGoogleGenerativeAI({ -// temperature: 0.7, -// apiKey: geminiApiKey, -// modelName: 'gemini-2.0-flash', -// }); - -// const llmConfig = { -// defaultLLM: defaultLLM as unknown as BaseChatModel, -// fastLLM: defaultLLM as unknown as BaseChatModel, -// evaluationLLM: defaultLLM as unknown as BaseChatModel, -// }; - -// const embeddings = new OpenAIEmbeddings({ -// openAIApiKey: openaiApiKey, -// modelName: 'text-embedding-3-large', -// dimensions: 2048, -// }) as unknown as Embeddings; - -// // Initialize vector store -// const dbConfig = getVectorDbConfig(); -// const vectorStore = await VectorStore.getInstance(dbConfig, embeddings); - -// const focus_to_agent_name: Record< -// DocumentSource | 'starknet_ecosystem', -// AvailableAgents -// > = { -// [DocumentSource.CAIRO_BOOK]: 'cairoBook', -// [DocumentSource.STARKNET_DOCS]: 'starknetDocs', -// starknet_ecosystem: 'starknetEcosystem', -// [DocumentSource.STARKNET_FOUNDRY]: 'starknetFoundry', -// [DocumentSource.CAIRO_BY_EXAMPLE]: 'cairoByExample', -// [DocumentSource.OPENZEPPELIN_DOCS]: 'openZeppelinDocs', -// }; -// // Get agent configuration -// const agentConfig = getAgentConfig( -// focus_to_agent_name[focus], -// vectorStore, -// ); -// if (!agentConfig) { -// logger.error(`Agent configuration not found for focus: ${focus}`); -// process.exit(1); -// } - -// // Initialize DocQualityTester -// const tester = new DocQualityTester(llmConfig, embeddings, { -// ...agentConfig, -// vectorStore: {} as any, // Not needed for comparison -// sources: agentConfig.sources, -// }); - -// // Compare results -// logger.info(`Comparing documentation quality for ${focus}`); -// const report = await tester.compareResults( -// baseline.results, -// current.results, -// ); - -// // Output comparison -// if (options.output) { -// const outputPath = path.resolve(process.cwd(), options.output); -// fs.writeFileSync(outputPath, JSON.stringify(report, null, 2)); -// logger.info(`Comparison report saved to ${outputPath}`); -// } else { -// console.log('\nDocumentation Quality Comparison Report'); -// console.log('=====================================\n'); -// console.log(`Focus: ${report.results.focus}`); -// console.log(`Baseline Version: ${baseline.results.version}`); -// console.log(`Current Version: ${current.results.version}`); - -// console.log('\nSummary:'); -// console.log(report.summary); - -// console.log('\nMetric Changes:'); -// const baselineMetrics = baseline.results.metrics.overall; -// const currentMetrics = report.results.metrics.overall; - -// console.log( -// `- Relevance Score: ${baselineMetrics.percentAnswered.toFixed(2)} → ${currentMetrics.percentAnswered.toFixed(2)} (${baselineMetrics.percentAnswered > currentMetrics.percentAnswered ? '↓' : '↑'})`, -// ); -// console.log( -// `- Coverage Score: ${baselineMetrics.avgClarityScore.toFixed(2)} → ${currentMetrics.avgClarityScore.toFixed(2)} (${baselineMetrics.avgClarityScore > currentMetrics.avgClarityScore ? '↓' : '↑'})`, -// ); -// console.log( -// `- Answer Completeness: ${baselineMetrics.avgSourceAlignment.toFixed(2)} → ${currentMetrics.avgSourceAlignment.toFixed(2)} (${baselineMetrics.avgSourceAlignment > currentMetrics.avgSourceAlignment ? '↓' : '↑'})`, -// ); - -// if (report.recommendations.length > 0) { -// console.log('\nRecommendations:'); -// report.recommendations.forEach((rec, i) => { -// console.log( -// `${i + 1}. [${rec.priority.toUpperCase()}] ${rec.description}`, -// ); -// }); -// } - -// console.log( -// '\nFor full comparison report, use the --output option to save to file.', -// ); -// } -// } catch (error) { -// logger.error('Error comparing documentation quality results:', error); -// process.exit(1); -// } -// }); - -// program.parse(); +import fs from 'fs'; +import path from 'path'; +import { Command } from 'commander'; +import { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import { OpenAIEmbeddings } from '@langchain/openai'; +import { DocumentSource, RagSearchConfig } from '../types'; +import { DocQualityTester } from '../core/pipeline/docQualityTester'; +import { getAgentConfig } from '../config/agent'; +import { logger } from '../utils'; +import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; +import { VectorStore } from '../db/postgresVectorStore'; +import { + getGeminiApiKey, + getOpenaiApiKey, + getVectorDbConfig, +} from '../config/settings'; +import { Embeddings } from '@langchain/core/embeddings'; +import { LLMConfig } from '../types'; + +const program = new Command(); + +// Initialize the program +program + .name('test-doc-quality') + .description('Test documentation quality using the Starknet Agent') + .version('1.0.0'); + +program + .command('test') + .description('Run documentation quality tests') + .requiredOption( + '-s, --source ', + 'Documentation source to test (e.g., starknet_docs)', + ) + .requiredOption('-t, --test-file ', 'Path to test file (JSON)') + .option('-o, --output ', 'Path to output file (JSON)') + .option( + '-m, --model ', + 'LLM model to use for testing', + 'Claude 3.5 Sonnet', + ) + .option( + '-e, --eval-model ', + 'LLM model to use for evaluation (defaults to same as model)', + ) + .option( + '--no-detailed-output', + 'Disable detailed test output with PASS/FAIL status', + ) + .option( + '--thresholds ', + 'Custom thresholds for determining pass/fail status (JSON string)', + ) + .action(async (options) => { + try { + // Validate source + const source = options.source as DocumentSource; + const focus = source as string; + + // Load test file + const testFilePath = path.resolve(process.cwd(), options.testFile); + if (!fs.existsSync(testFilePath)) { + logger.error(`Test file not found: ${testFilePath}`); + process.exit(1); + } + + const testFileContent = fs.readFileSync(testFilePath, 'utf-8'); + const testSet = JSON.parse(testFileContent); + + const geminiApiKey = getGeminiApiKey(); + const openaiApiKey = getOpenaiApiKey(); + + // Initialize models and embeddings + const defaultLLM = new ChatGoogleGenerativeAI({ + temperature: 0.7, + apiKey: geminiApiKey, + modelName: 'gemini-2.0-flash', + }); + + const llmConfig: LLMConfig = { + defaultLLM: defaultLLM as unknown as BaseChatModel, + fastLLM: defaultLLM as unknown as BaseChatModel, + }; + + const embeddings = new OpenAIEmbeddings({ + openAIApiKey: openaiApiKey, + modelName: 'text-embedding-3-large', + dimensions: 1536, + }) as unknown as Embeddings; + + // Initialize vector store + const dbConfig = getVectorDbConfig(); + const vectorStore = await VectorStore.getInstance(dbConfig, embeddings); + + // Get agent configuration + const agentConfig = getAgentConfig(vectorStore); + if (!agentConfig) { + logger.error(`Agent configuration not found for source: ${source}`); + process.exit(1); + } + + // Create RAG config + const ragConfig: RagSearchConfig = { + ...agentConfig, + vectorStore, + sources: agentConfig.sources, + }; + + // Initialize DocQualityTester + const tester = new DocQualityTester(llmConfig, embeddings, ragConfig); + + // Parse thresholds if provided + let thresholds = undefined; + if (options.thresholds) { + try { + thresholds = JSON.parse(options.thresholds); + } catch (e) { + logger.error(`Error parsing thresholds JSON: ${e}`); + process.exit(1); + } + } + + // Run tests with detailed output options + logger.info(`Starting documentation quality tests for focus ${focus}`); + const results = await tester.testDocQuality(testSet, focus, { + showDetailedOutput: options.detailedOutput, + thresholds, + }); + + // Generate report + const report = await tester.generateReport(results); + + // Output results + if (options.output) { + const outputPath = path.resolve(process.cwd(), options.output); + fs.writeFileSync(outputPath, JSON.stringify(report, null, 2)); + logger.info(`Report saved to ${outputPath}`); + } else { + console.log('\nDocumentation Quality Report'); + console.log('===========================\n'); + console.log(`Focus: ${report.results.focus}`); + console.log(`Version: ${report.results.version}`); + console.log(`Test Cases: ${report.results.caseResults.length}`); + console.log('\nSummary:'); + console.log(report.summary); + + console.log('\nKey Metrics:'); + console.log( + `- Relevance Score: ${report.results.metrics.overall.percentAnswered.toFixed(2)}`, + ); + console.log( + `- Coverage Score: ${report.results.metrics.overall.avgClarityScore.toFixed(2)}`, + ); + console.log( + `- Answer Completeness: ${report.results.metrics.overall.avgSourceAlignment.toFixed(2)}`, + ); + + console.log('\nTop Recommendations:'); + const highPriorityRecs = report.recommendations.filter( + (r) => r.priority === 'high', + ); + highPriorityRecs.forEach((rec, i) => { + console.log(`${i + 1}. ${rec.description}`); + }); + + console.log( + '\nFor full report, use the --output option to save to file.', + ); + } + } catch (error) { + logger.error('Error running documentation quality tests:', error); + process.exit(1); + } + }); + +program + .command('compare') + .description('Compare documentation quality between versions') + .requiredOption('-s, --source ', 'Documentation source to test') + .requiredOption( + '-b, --baseline ', + 'Path to baseline results file (JSON)', + ) + .requiredOption('-c, --current ', 'Path to current results file (JSON)') + .option('-o, --output ', 'Path to output file (JSON)') + .option( + '-m, --model ', + 'LLM model to use for comparison', + 'Claude 3.5 Sonnet', + ) + .action(async (options) => { + try { + // Validate source + const focus = options.source as DocumentSource; + + // Load result files + const baselinePath = path.resolve(process.cwd(), options.baseline); + const currentPath = path.resolve(process.cwd(), options.current); + + if (!fs.existsSync(baselinePath)) { + logger.error(`Baseline file not found: ${baselinePath}`); + process.exit(1); + } + + if (!fs.existsSync(currentPath)) { + logger.error(`Current file not found: ${currentPath}`); + process.exit(1); + } + + const baselineContent = fs.readFileSync(baselinePath, 'utf-8'); + const currentContent = fs.readFileSync(currentPath, 'utf-8'); + + const baseline = JSON.parse(baselineContent); + const current = JSON.parse(currentContent); + + const geminiApiKey = getGeminiApiKey(); + const openaiApiKey = getOpenaiApiKey(); + + // Initialize models and embeddings + const defaultLLM = new ChatGoogleGenerativeAI({ + temperature: 0.7, + apiKey: geminiApiKey, + modelName: 'gemini-2.0-flash', + }); + + const llmConfig: LLMConfig = { + defaultLLM: defaultLLM as unknown as BaseChatModel, + fastLLM: defaultLLM as unknown as BaseChatModel, + }; + + const embeddings = new OpenAIEmbeddings({ + openAIApiKey: openaiApiKey, + modelName: 'text-embedding-3-large', + dimensions: 1536, + }) as unknown as Embeddings; + + // Initialize vector store + const dbConfig = getVectorDbConfig(); + const vectorStore = await VectorStore.getInstance(dbConfig, embeddings); + + // Get agent configuration + const agentConfig = getAgentConfig(vectorStore); + if (!agentConfig) { + logger.error(`Agent configuration not found for source: ${focus}`); + process.exit(1); + } + + // Create RAG config + const ragConfig: RagSearchConfig = { + ...agentConfig, + vectorStore, + sources: agentConfig.sources, + }; + + // Initialize DocQualityTester + const tester = new DocQualityTester(llmConfig, embeddings, ragConfig); + + // Run comparison + const comparisonReport = await tester.compareResults(baseline, current); + + // Output results + if (options.output) { + const outputPath = path.resolve(process.cwd(), options.output); + fs.writeFileSync(outputPath, JSON.stringify(comparisonReport, null, 2)); + logger.info(`Comparison report saved to ${outputPath}`); + } else { + console.log('\nDocumentation Quality Comparison'); + console.log('==============================\n'); + console.log(`Focus: ${current.focus}`); + console.log(`Baseline Version: ${baseline.version}`); + console.log(`Current Version: ${current.version}`); + console.log('\nComparison Summary:'); + console.log(comparisonReport.summary); + + // Display recommendations + console.log('\nRecommendations:'); + comparisonReport.recommendations.forEach((rec, i) => { + console.log( + `${i + 1}. [${rec.priority.toUpperCase()}] ${rec.description}`, + ); + }); + + console.log( + '\nFor full report, use the --output option to save to file.', + ); + } + } catch (error) { + logger.error('Error comparing documentation quality:', error); + process.exit(1); + } + }); + +program.parse(process.argv); diff --git a/packages/agents/src/types/index.ts b/packages/agents/src/types/index.ts index cbfb429e..c2995da1 100644 --- a/packages/agents/src/types/index.ts +++ b/packages/agents/src/types/index.ts @@ -7,7 +7,7 @@ export type AvailableAgents = 'cairoCoder'; export interface LLMConfig { defaultLLM: BaseChatModel; - fastLLM?: BaseChatModel; + fastLLM: BaseChatModel; } export interface VectorStoreConfig { diff --git a/scripts/database-connection.sh b/scripts/database-connection.sh new file mode 100755 index 00000000..4f20b41f --- /dev/null +++ b/scripts/database-connection.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +echo "=== Testing database connection via /chat/completions endpoint ===" + +# Préparation des données pour l'appel API (requête simple) +REQUEST_DATA='{ + "model": "gemini-2.0-flash", + "messages": [ + { + "role": "user", + "content": "Hello" + } + ], + "temperature": 0.7 + }' + +RESPONSE=$(curl -s -X POST http://localhost:3001/chat/completions \ + -H "Content-Type: application/json" \ + -d "$REQUEST_DATA") + +# Vérification de la réponse pour détecter des erreurs de base de données +if echo "$RESPONSE" | jq -e '.error' >/dev/null 2>&1; then + echo "❌ Database connection error detected" + echo "Error details:" + echo "$RESPONSE" | jq '.error' + exit 1 +else + echo "✅ Successfully connected to database via /chat/completions endpoint" + exit 0 +fi \ No newline at end of file diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh new file mode 100755 index 00000000..7a6ebcc6 --- /dev/null +++ b/scripts/integration-tests.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Variable pour suivre les erreurs +ERROR_COUNT=0 + +echo "=== Checking containers status ===" +docker ps + +echo -e "\n=== Checking PostgreSQL ===" +docker exec postgresql pg_isready -U postgres -h localhost +if [ $? -eq 0 ]; then + echo "✅ PostgreSQL is ready!" +else + echo "❌ PostgreSQL is not ready" + ((ERROR_COUNT++)) +fi + +echo -e "\n=== Checking network between backend and PostgreSQL ===" +# Using ping since it's installed in your backend +docker exec backend ping -c 2 postgres +if [ $? -eq 0 ]; then + echo "✅ Network connectivity to PostgreSQL works!" +else + echo "❌ Network connectivity issue to PostgreSQL" + ((ERROR_COUNT++)) +fi + +echo -e "\n=== Testing backend API ===" +RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3001/ 2>/dev/null) +if [ "$RESPONSE" == "200" ]; then + echo "✅ Backend API is working correctly!" +else + echo "❌ Issue with backend API. HTTP code: $RESPONSE" + # Get more details about the error + echo "Error details:" + curl -v http://localhost:3001/ + ((ERROR_COUNT++)) +fi From 0b77663b0e3bd3deaad588fa036513a22f8cb6c7 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 28 Apr 2025 22:34:21 +0200 Subject: [PATCH 08/47] fix: name script test --- .github/workflows/backend.yml | 2 +- packages/agents/src/types/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 622424bc..63472257 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -109,7 +109,7 @@ jobs: sleep 20 chmod +x ./scripts/integration-tests.sh - chmod +x ./scripts/test-chat-completions.sh + chmod +x ./scripts/database-connection.sh echo -e "\n=== Running basic integration tests ===" ./scripts/integration-tests.sh diff --git a/packages/agents/src/types/index.ts b/packages/agents/src/types/index.ts index c2995da1..bd5f49ba 100644 --- a/packages/agents/src/types/index.ts +++ b/packages/agents/src/types/index.ts @@ -64,6 +64,7 @@ export interface AgentConfig { export interface ProcessedQuery { original: string; + s; transformed: string | string[]; // Single query or list of search terms isContractRelated?: boolean; isTestRelated?: boolean; From 5f2e5161f96ba2f11ec510e673161cddb467c02e Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 28 Apr 2025 22:35:10 +0200 Subject: [PATCH 09/47] fix: name script test --- .github/workflows/backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 63472257..ddd7773b 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -116,7 +116,7 @@ jobs: INTEGRATION_RESULT=$? echo -e "\n=== Running database connection test via chat/completions endpoint ===" - ./scripts/test-chat-completions.sh + ./scripts/database-connection.sh DB_CONNECTION_RESULT=$? if [ $INTEGRATION_RESULT -ne 0 ] || [ $DB_CONNECTION_RESULT -ne 0 ]; then From bd20a5822b56efe290cdb5bbd8cbe8d814c7c194 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 28 Apr 2025 23:16:45 +0200 Subject: [PATCH 10/47] fix: remove useless s added in types/index --- packages/agents/src/types/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/agents/src/types/index.ts b/packages/agents/src/types/index.ts index bd5f49ba..c2995da1 100644 --- a/packages/agents/src/types/index.ts +++ b/packages/agents/src/types/index.ts @@ -64,7 +64,6 @@ export interface AgentConfig { export interface ProcessedQuery { original: string; - s; transformed: string | string[]; // Single query or list of search terms isContractRelated?: boolean; isTestRelated?: boolean; From cb00a025dfb1263cee2cf2d8c1293b76678d186b Mon Sep 17 00:00:00 2001 From: alvinouille Date: Wed, 30 Apr 2025 18:00:30 +0200 Subject: [PATCH 11/47] test: ci --- packages/backend/src/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/app.ts b/packages/backend/src/app.ts index 41492e3e..6fcae07b 100644 --- a/packages/backend/src/app.ts +++ b/packages/backend/src/app.ts @@ -7,7 +7,7 @@ process.on('uncaughtException', (err, origin) => { }); process.on('unhandledRejection', (reason, promise) => { - logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`); + logger.error(`Unhandled Rejection at : ${promise}, reason: ${reason}`); }); // Start the application From e526770e5007dd9624f81bf4a2e2434ceeb20053 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Wed, 30 Apr 2025 18:04:27 +0200 Subject: [PATCH 12/47] fix: name of secrets ci --- .github/workflows/backend.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ddd7773b..73668537 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -1,4 +1,4 @@ -name: CI +name: Backend-CI on: push: @@ -57,9 +57,9 @@ jobs: mkdir -p packages/agents cat > packages/agents/config.toml << 'EOL' [API_KEYS] - OPENAI = "${{ secrets.OPENAI_API_KEY }}" - ANTHROPIC = "${{ secrets.ANTHROPIC_API_KEY }}" - GEMINI = "${{ secrets.GEMINI_API_KEY }}" + OPENAI = "${{ secrets.OPENAI }}" + ANTHROPIC = "${{ secrets.ANTHROPIC }}" + GEMINI = "${{ secrets.GEMINI }}" [VECTOR_DB] POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" From f3f8c0d5ea8edcc2126024f90af5246e7a2aa0dd Mon Sep 17 00:00:00 2001 From: alvinouille Date: Wed, 30 Apr 2025 18:07:27 +0200 Subject: [PATCH 13/47] fix: ci --- .github/workflows/backend.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 73668537..fea75410 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -96,15 +96,15 @@ jobs: POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" EOL - - name: Run tests + - name: Run unit tests run: turbo test - name: Build docker image run: docker build -t cairo-coder-backend:${{ github.sha }} -f backend.dockerfile . - - name: Integration docker connection test with Docker Compose + - name: Run integration tests run: | - docker-compose up -d postgres backend + docker compose up -d postgres backend echo "Waiting for services to be ready..." sleep 20 From ca16de835020a705aee4c4c5449f7a0d8297677e Mon Sep 17 00:00:00 2001 From: alvinouille Date: Wed, 30 Apr 2025 18:11:31 +0200 Subject: [PATCH 14/47] fix: env ci --- .github/workflows/backend.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index fea75410..58f267f4 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -87,8 +87,7 @@ jobs: - name: Create env file run: | - mkdir -p packages/agents - cat > packages/agents/.env << 'EOL' + cat > .env << 'EOL' POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" POSTGRES_HOST = "localhost" POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" From 1adab800f08670caf10daf46a0138e85f40d3147 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Wed, 30 Apr 2025 18:32:02 +0200 Subject: [PATCH 15/47] fix: setup pnpm cache --- .github/workflows/backend.yml | 2 +- .github/workflows/ingester.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 58f267f4..25b2767b 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -39,7 +39,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') || github.sha }} restore-keys: | ${{ runner.os }}-pnpm-store- diff --git a/.github/workflows/ingester.yml b/.github/workflows/ingester.yml index 1edfeaa4..8790cab5 100644 --- a/.github/workflows/ingester.yml +++ b/.github/workflows/ingester.yml @@ -40,7 +40,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') || github.sha }} restore-keys: | ${{ runner.os }}-pnpm-store- @@ -69,9 +69,9 @@ jobs: mkdir -p packages/agents cat > packages/agents/config.toml << 'EOL' [API_KEYS] - OPENAI = "${{ secrets.OPENAI_API_KEY }}" - ANTHROPIC = "${{ secrets.ANTHROPIC_API_KEY }}" - GEMINI = "${{ secrets.GEMINI_API_KEY }}" + OPENAI = "${{ secrets.OPENAI }}" + ANTHROPIC = "${{ secrets.ANTHROPIC }}" + GEMINI = "${{ secrets.GEMINI }}" [VECTOR_DB] POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" From 23186b93d12beceb200f584e7eac0262c583d275 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 1 May 2025 23:41:12 +0200 Subject: [PATCH 16/47] core: add ingester, snak import and generation code test logic in ci, add code generation test file in packages/backend/__tests__ --- .../backend/__tests__/compilation.test.ts | 112 ++++++++++++++++++ scripts/database-connection.sh | 2 +- scripts/integration-tests.sh | 11 -- 3 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 packages/backend/__tests__/compilation.test.ts diff --git a/packages/backend/__tests__/compilation.test.ts b/packages/backend/__tests__/compilation.test.ts new file mode 100644 index 00000000..f7a2c5db --- /dev/null +++ b/packages/backend/__tests__/compilation.test.ts @@ -0,0 +1,112 @@ +import request from 'supertest'; + +const API_KEY = process.env.API_KEY; +const API_URL = process.env.API_URL; +const GENERATION_TIMEOUT = 50000; +const COMPILATION_TIMEOUT = 15000; + +describe('Code Generation and Compilation Tests', () => { + const agent = request(API_URL); + + async function generateAndCompile(prompt: string): Promise { + + const generateResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ request: `${prompt}. Do not compile it or do any other action after generating it, just return the code` }) + .timeout(GENERATION_TIMEOUT); + + if (generateResponse.body.output[0].status !== 'success') { + console.error('Generation failed:', generateResponse.body); + return false; + } + + console.log('Generated code successfully'); + + const compileResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ request: 'Compile the previous code, if there is an error do NOT fix it, just return the error' }) + .timeout(COMPILATION_TIMEOUT); + + if (compileResponse.body.output[0].status !== 'success') { + console.error('Compilation request failed:', compileResponse.body); + return false; + } + + const responseText = compileResponse.body.response || ''; + const isSuccessful = !responseText.includes('error') && + !responseText.includes('failed') && + !responseText.includes('cannot'); + + if (!isSuccessful) { + console.error('Compilation failed:', responseText); + } else { + console.log('Compilation successful'); + } + + return isSuccessful; + } + + describe('Cairo Functions and Basic Algorithms', () => { + test('Factorial function', async () => { + const prompt = "Generate a Cairo function that calculates the factorial of a number."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + + test('Max value in array', async () => { + const prompt = "Generate a Cairo function that finds the maximum value in an array."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + + test('Simple sorting algorithm', async () => { + const prompt = "Generate a Cairo implementation of a simple sorting algorithm."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + }); + + describe('Simple Starknet Contracts', () => { + test('Basic contract with storage', async () => { + const prompt = "Generate a basic Starknet contract with a storage variable and getter/setter functions."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + + test('Counter contract', async () => { + const prompt = "Generate a Starknet contract that maintains a counter with increment and decrement functions."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + + test('Simple voting system', async () => { + const prompt = "Generate a Starknet contract for a simple voting system where users can vote only once."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + }); + + describe('Standard and Complex Contracts', () => { + test('ERC-20 token contract', async () => { + const prompt = "Generate a minimal Starknet ERC-20 token contract."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + + test('ERC-721 NFT contract', async () => { + const prompt = "Generate a Starknet ERC-721 NFT contract with minting functionality."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + + test('Multisig wallet contract', async () => { + const prompt = "Generate a Starknet multisig wallet contract that requires multiple approvals for transactions."; + const success = await generateAndCompile(prompt); + expect(success).toBe(true); + }, 100000); + }); +}); \ No newline at end of file diff --git a/scripts/database-connection.sh b/scripts/database-connection.sh index 4f20b41f..877d0604 100755 --- a/scripts/database-connection.sh +++ b/scripts/database-connection.sh @@ -4,7 +4,7 @@ echo "=== Testing database connection via /chat/completions endpoint ===" # Préparation des données pour l'appel API (requête simple) REQUEST_DATA='{ - "model": "gemini-2.0-flash", + "model": "gemini-2.5-flash", "messages": [ { "role": "user", diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 7a6ebcc6..f0d96979 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -1,31 +1,21 @@ #!/bin/bash -# Variable pour suivre les erreurs -ERROR_COUNT=0 - -echo "=== Checking containers status ===" docker ps -echo -e "\n=== Checking PostgreSQL ===" docker exec postgresql pg_isready -U postgres -h localhost if [ $? -eq 0 ]; then echo "✅ PostgreSQL is ready!" else echo "❌ PostgreSQL is not ready" - ((ERROR_COUNT++)) fi -echo -e "\n=== Checking network between backend and PostgreSQL ===" -# Using ping since it's installed in your backend docker exec backend ping -c 2 postgres if [ $? -eq 0 ]; then echo "✅ Network connectivity to PostgreSQL works!" else echo "❌ Network connectivity issue to PostgreSQL" - ((ERROR_COUNT++)) fi -echo -e "\n=== Testing backend API ===" RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3001/ 2>/dev/null) if [ "$RESPONSE" == "200" ]; then echo "✅ Backend API is working correctly!" @@ -34,5 +24,4 @@ else # Get more details about the error echo "Error details:" curl -v http://localhost:3001/ - ((ERROR_COUNT++)) fi From ecebf14a4b477e80892918b3a258fe1a2f1095c6 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 1 May 2025 23:43:38 +0200 Subject: [PATCH 17/47] fix: remove langsmith --- .github/workflows/backend.yml | 84 +++++++++++++++++++++-- {scripts => tests}/database-connection.sh | 0 {scripts => tests}/integration-tests.sh | 0 3 files changed, 79 insertions(+), 5 deletions(-) rename {scripts => tests}/database-connection.sh (100%) rename {scripts => tests}/integration-tests.sh (100%) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 25b2767b..3ec44bda 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -101,21 +101,21 @@ jobs: - name: Build docker image run: docker build -t cairo-coder-backend:${{ github.sha }} -f backend.dockerfile . - - name: Run integration tests + - name: Run backend and database integration tests run: | docker compose up -d postgres backend echo "Waiting for services to be ready..." sleep 20 - chmod +x ./scripts/integration-tests.sh - chmod +x ./scripts/database-connection.sh + chmod +x ./tests/integration-tests.sh + chmod +x ./tests/database-connection.sh echo -e "\n=== Running basic integration tests ===" - ./scripts/integration-tests.sh + ./tests/integration-tests.sh INTEGRATION_RESULT=$? echo -e "\n=== Running database connection test via chat/completions endpoint ===" - ./scripts/database-connection.sh + ./tests/database-connection.sh DB_CONNECTION_RESULT=$? if [ $INTEGRATION_RESULT -ne 0 ] || [ $DB_CONNECTION_RESULT -ne 0 ]; then @@ -124,6 +124,80 @@ jobs: else echo "✅ All integration tests passed!" fi + + - name: Build ingester image + run: docker compose --profile ingester build ingester + + - name: Run data ingestion + run: docker compose --profile ingester up -d ingester + + - name: Import snak repository + run: | + git clone https://github.com/KasarLabs/snak ../snak + cd ../snak + git checkout fix/server-for-reuse + + - name: Create snak env file + run: | + cd ../snak + cat > .env << 'EOL' + + STARKNET_PUBLIC_ADDRESS="${{ secrets.STARKNET_PUBLIC_ADDRESS }}" + STARKNET_PRIVATE_KEY="${{ secrets.STARKNET_PRIVATE_KEY }}" + STARKNET_RPC_URL="${{ secrets.STARKNET_RPC_URL }}" + + AI_PROVIDER_API_KEY="${{ secrets.OPENAI }}" + AI_MODEL="claude-3-5-sonnet-latest" + AI_PROVIDER="anthropic" + + NODE_ENV="development" + + SERVER_API_KEY="${{ secrets.SNAK_SERVER_KEY }}" + SERVER_PORT="${{ secrets.SNAK_SERVER_PORT }}" + + POSTGRES_USER="${{ secrets.POSTGRES_USER }}" + POSTGRES_PASSWORD="${{ secrets.POSTGRES_PASSWORD }}" + POSTGRES_ROOT_DB="${{ secrets.POSTGRES_ROOT_DB }}" + POSTGRES_HOST="localhost" + POSTGRES_PORT="${{ secrets.POSTGRES_PORT }}" + + CAIRO_UPLOAD_DIR="plugins/cairocoder/uploads/" + CAIRO_GENERATION_API_URL="http://127.0.0.1:3001/chat/completions" + + EOL + + - name: Install snak dependencies + run: | + cd ../snak + pnpm install + + - name: Start snak server + run: | + cd ../snak + pnpm start:server & + + echo "Waiting for server to start..." + sleep 30 + + - name: Run cairo code generation test + run: | + cd packages/backend + + export API_KEY="${{ secrets.SNAK_SERVER_KEY }}" + export API_URL="http://localhost:${{ secrets.SNAK_SERVER_PORT }}" + + npx jest __tests__/compilation.test.ts --verbose + + TEST_RESULT=$? + + pkill -f "start:server" || true + + if [ $TEST_RESULT -ne 0 ]; then + echo "❌ API test failed!" + exit 1 + else + echo "✅ API test passed!" + fi # - name: Push docker image # run: docker push ${{ github.repository }}:${{ github.sha }} diff --git a/scripts/database-connection.sh b/tests/database-connection.sh similarity index 100% rename from scripts/database-connection.sh rename to tests/database-connection.sh diff --git a/scripts/integration-tests.sh b/tests/integration-tests.sh similarity index 100% rename from scripts/integration-tests.sh rename to tests/integration-tests.sh From e9dcc6789d33426c0b753395c968dede5ca1d18b Mon Sep 17 00:00:00 2001 From: alvinouille Date: Fri, 2 May 2025 09:37:34 +0200 Subject: [PATCH 18/47] fix: remove backend/test/compilation.test.ts from unit testing --- .github/workflows/backend.yml | 18 +++---- .../backend/__tests__/compilation.test.ts | 51 ++++++++++++------- packages/backend/jest.config.js | 1 + 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 3ec44bda..3fce4715 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -124,10 +124,10 @@ jobs: else echo "✅ All integration tests passed!" fi - + - name: Build ingester image run: docker compose --profile ingester build ingester - + - name: Run data ingestion run: docker compose --profile ingester up -d ingester @@ -136,7 +136,7 @@ jobs: git clone https://github.com/KasarLabs/snak ../snak cd ../snak git checkout fix/server-for-reuse - + - name: Create snak env file run: | cd ../snak @@ -175,23 +175,23 @@ jobs: run: | cd ../snak pnpm start:server & - + echo "Waiting for server to start..." sleep 30 - name: Run cairo code generation test run: | cd packages/backend - + export API_KEY="${{ secrets.SNAK_SERVER_KEY }}" export API_URL="http://localhost:${{ secrets.SNAK_SERVER_PORT }}" - + npx jest __tests__/compilation.test.ts --verbose - + TEST_RESULT=$? - + pkill -f "start:server" || true - + if [ $TEST_RESULT -ne 0 ]; then echo "❌ API test failed!" exit 1 diff --git a/packages/backend/__tests__/compilation.test.ts b/packages/backend/__tests__/compilation.test.ts index f7a2c5db..309048ac 100644 --- a/packages/backend/__tests__/compilation.test.ts +++ b/packages/backend/__tests__/compilation.test.ts @@ -1,20 +1,25 @@ +/** + * @group integration + */ + import request from 'supertest'; const API_KEY = process.env.API_KEY; const API_URL = process.env.API_URL; -const GENERATION_TIMEOUT = 50000; +const GENERATION_TIMEOUT = 50000; const COMPILATION_TIMEOUT = 15000; describe('Code Generation and Compilation Tests', () => { const agent = request(API_URL); async function generateAndCompile(prompt: string): Promise { - const generateResponse = await agent .post('/api/key/request') .set('Content-Type', 'application/json') .set('x-api-key', API_KEY) - .send({ request: `${prompt}. Do not compile it or do any other action after generating it, just return the code` }) + .send({ + request: `${prompt}. Do not compile it or do any other action after generating it, just return the code`, + }) .timeout(GENERATION_TIMEOUT); if (generateResponse.body.output[0].status !== 'success') { @@ -28,7 +33,10 @@ describe('Code Generation and Compilation Tests', () => { .post('/api/key/request') .set('Content-Type', 'application/json') .set('x-api-key', API_KEY) - .send({ request: 'Compile the previous code, if there is an error do NOT fix it, just return the error' }) + .send({ + request: + 'Compile the previous code, if there is an error do NOT fix it, just return the error', + }) .timeout(COMPILATION_TIMEOUT); if (compileResponse.body.output[0].status !== 'success') { @@ -37,9 +45,10 @@ describe('Code Generation and Compilation Tests', () => { } const responseText = compileResponse.body.response || ''; - const isSuccessful = !responseText.includes('error') && - !responseText.includes('failed') && - !responseText.includes('cannot'); + const isSuccessful = + !responseText.includes('error') && + !responseText.includes('failed') && + !responseText.includes('cannot'); if (!isSuccessful) { console.error('Compilation failed:', responseText); @@ -52,19 +61,22 @@ describe('Code Generation and Compilation Tests', () => { describe('Cairo Functions and Basic Algorithms', () => { test('Factorial function', async () => { - const prompt = "Generate a Cairo function that calculates the factorial of a number."; + const prompt = + 'Generate a Cairo function that calculates the factorial of a number.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); test('Max value in array', async () => { - const prompt = "Generate a Cairo function that finds the maximum value in an array."; + const prompt = + 'Generate a Cairo function that finds the maximum value in an array.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); test('Simple sorting algorithm', async () => { - const prompt = "Generate a Cairo implementation of a simple sorting algorithm."; + const prompt = + 'Generate a Cairo implementation of a simple sorting algorithm.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); @@ -72,19 +84,22 @@ describe('Code Generation and Compilation Tests', () => { describe('Simple Starknet Contracts', () => { test('Basic contract with storage', async () => { - const prompt = "Generate a basic Starknet contract with a storage variable and getter/setter functions."; + const prompt = + 'Generate a basic Starknet contract with a storage variable and getter/setter functions.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); test('Counter contract', async () => { - const prompt = "Generate a Starknet contract that maintains a counter with increment and decrement functions."; + const prompt = + 'Generate a Starknet contract that maintains a counter with increment and decrement functions.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); test('Simple voting system', async () => { - const prompt = "Generate a Starknet contract for a simple voting system where users can vote only once."; + const prompt = + 'Generate a Starknet contract for a simple voting system where users can vote only once.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); @@ -92,21 +107,23 @@ describe('Code Generation and Compilation Tests', () => { describe('Standard and Complex Contracts', () => { test('ERC-20 token contract', async () => { - const prompt = "Generate a minimal Starknet ERC-20 token contract."; + const prompt = 'Generate a minimal Starknet ERC-20 token contract.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); test('ERC-721 NFT contract', async () => { - const prompt = "Generate a Starknet ERC-721 NFT contract with minting functionality."; + const prompt = + 'Generate a Starknet ERC-721 NFT contract with minting functionality.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); test('Multisig wallet contract', async () => { - const prompt = "Generate a Starknet multisig wallet contract that requires multiple approvals for transactions."; + const prompt = + 'Generate a Starknet multisig wallet contract that requires multiple approvals for transactions.'; const success = await generateAndCompile(prompt); expect(success).toBe(true); }, 100000); }); -}); \ No newline at end of file +}); diff --git a/packages/backend/jest.config.js b/packages/backend/jest.config.js index 8ddf4c7c..ec661178 100644 --- a/packages/backend/jest.config.js +++ b/packages/backend/jest.config.js @@ -3,6 +3,7 @@ module.exports = { testEnvironment: 'node', roots: ['/src', '/__tests__'], testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'], + testPathIgnorePatterns: ['/__tests__/.*compilation\\.test\\.ts$'], transform: { '^.+\\.ts$': [ 'ts-jest', From c68d1c69fb608425eb1258080d803af55e78085d Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 5 May 2025 18:18:35 +0200 Subject: [PATCH 19/47] feat: add snak test in addition of snak server launching to ci, improve testing organization within repo and update turbo command --- .github/workflows/backend.yml | 39 ++-- .gitignore | 5 + package.json | 4 + .../__tests__/code-quality/snak.test.ts | 167 ++++++++++++++++++ .../{ => unit}/answerGenerator.test.ts | 13 +- .../{ => unit}/documentRetriever.test.ts | 10 +- .../{ => unit}/queryProcessor.test.ts | 12 +- .../{ => unit}/ragAgentFactory.test.ts | 12 +- .../__tests__/{ => unit}/ragPipeline.test.ts | 20 +-- packages/agents/package.json | 4 +- .../samples/all_quality.json | 0 .../samples/cairo_book_quality.json | 0 .../samples/openzeppelin_quality.json | 0 .../samples/starknet_docs_quality.json | 0 .../{tests => doc-quality}/testDocQuality.ts | 0 .../backend/__tests__/compilation.test.ts | 129 -------------- .../__tests__/{types => unit}/context.test.ts | 0 .../__tests__/{ => unit}/server.test.ts | 9 +- packages/backend/jest.config.js | 1 - packages/backend/package.json | 3 +- packages/backend/tsconfig.json | 6 +- .../{ => unit}/AsciiDocIngester.test.ts | 4 +- .../{ => unit}/IngesterFactory.test.ts | 24 +-- .../{ => unit}/MarkdownIngester.test.ts | 4 +- .../__tests__/{ => unit}/contentUtils.test.ts | 2 +- .../__tests__/{ => unit}/shared.test.ts | 2 +- .../{ => unit}/vectorStoreUtils.test.ts | 2 +- packages/ingester/package.json | 1 + tests/database-connection.sh | 2 - turbo.json | 12 ++ 30 files changed, 278 insertions(+), 209 deletions(-) create mode 100644 packages/agents/__tests__/code-quality/snak.test.ts rename packages/agents/__tests__/{ => unit}/answerGenerator.test.ts (97%) rename packages/agents/__tests__/{ => unit}/documentRetriever.test.ts (96%) rename packages/agents/__tests__/{ => unit}/queryProcessor.test.ts (94%) rename packages/agents/__tests__/{ => unit}/ragAgentFactory.test.ts (89%) rename packages/agents/__tests__/{ => unit}/ragPipeline.test.ts (90%) rename packages/agents/src/{tests => doc-quality}/samples/all_quality.json (100%) rename packages/agents/src/{tests => doc-quality}/samples/cairo_book_quality.json (100%) rename packages/agents/src/{tests => doc-quality}/samples/openzeppelin_quality.json (100%) rename packages/agents/src/{tests => doc-quality}/samples/starknet_docs_quality.json (100%) rename packages/agents/src/{tests => doc-quality}/testDocQuality.ts (100%) delete mode 100644 packages/backend/__tests__/compilation.test.ts rename packages/backend/__tests__/{types => unit}/context.test.ts (100%) rename packages/backend/__tests__/{ => unit}/server.test.ts (93%) rename packages/ingester/__tests__/{ => unit}/AsciiDocIngester.test.ts (98%) rename packages/ingester/__tests__/{ => unit}/IngesterFactory.test.ts (75%) rename packages/ingester/__tests__/{ => unit}/MarkdownIngester.test.ts (97%) rename packages/ingester/__tests__/{ => unit}/contentUtils.test.ts (97%) rename packages/ingester/__tests__/{ => unit}/shared.test.ts (94%) rename packages/ingester/__tests__/{ => unit}/vectorStoreUtils.test.ts (96%) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 3fce4715..49f1bc2b 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -73,10 +73,10 @@ jobs: SIMILARITY_MEASURE = "cosine" [HOSTED_MODE] - DEFAULT_CHAT_PROVIDER = "anthropic" - DEFAULT_CHAT_MODEL = "Claude 3.5 Sonnet" - DEFAULT_FAST_CHAT_PROVIDER = "anthropic" - DEFAULT_FAST_CHAT_MODEL = "Claude 3.5 Sonnet" + DEFAULT_CHAT_PROVIDER = "gemini" + DEFAULT_CHAT_MODEL = "Gemini Flash 2.5" + DEFAULT_FAST_CHAT_PROVIDER = "gemini" + DEFAULT_FAST_CHAT_MODEL = "Gemini Flash 2.5" DEFAULT_EMBEDDING_PROVIDER = "openai" DEFAULT_EMBEDDING_MODEL = "Text embedding 3 large" @@ -96,7 +96,7 @@ jobs: EOL - name: Run unit tests - run: turbo test + run: pnpm run test:unit - name: Build docker image run: docker build -t cairo-coder-backend:${{ github.sha }} -f backend.dockerfile . @@ -133,13 +133,14 @@ jobs: - name: Import snak repository run: | - git clone https://github.com/KasarLabs/snak ../snak - cd ../snak + mkdir -p ./snak + git clone https://github.com/KasarLabs/snak ./snak + cd ./snak git checkout fix/server-for-reuse - name: Create snak env file run: | - cd ../snak + cd ./snak cat > .env << 'EOL' STARKNET_PUBLIC_ADDRESS="${{ secrets.STARKNET_PUBLIC_ADDRESS }}" @@ -165,28 +166,30 @@ jobs: CAIRO_GENERATION_API_URL="http://127.0.0.1:3001/chat/completions" EOL + - name: Cache snak node modules + uses: actions/cache@v4 + with: + path: ./snak/node_modules + key: ${{ runner.os }}-snak-modules-${{ hashFiles('./snak/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-snak-modules- - name: Install snak dependencies run: | - cd ../snak - pnpm install + cd ./snak + pnpm install --filter="@snakagent/server..." - name: Start snak server run: | - cd ../snak - pnpm start:server & + cd ./snak + pnpm run build --filter="@snakagent/server..." && lerna run start --scope "@snakagent/server" & echo "Waiting for server to start..." sleep 30 - name: Run cairo code generation test run: | - cd packages/backend - - export API_KEY="${{ secrets.SNAK_SERVER_KEY }}" - export API_URL="http://localhost:${{ secrets.SNAK_SERVER_PORT }}" - - npx jest __tests__/compilation.test.ts --verbose + API_KEY="${{ secrets.SNAK_SERVER_KEY }}" API_URL="http://localhost:${{ secrets.SNAK_SERVER_PORT }}" pnpm run test:code-quality TEST_RESULT=$? diff --git a/.gitignore b/.gitignore index 8273d3d8..978215d3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ yarn-error.log .env.development.local .env.test.local .env.production.local +.env.test packages/ui/.env @@ -43,3 +44,7 @@ packages/**/node_modules packages/**/dist /data +.secrets +.actrc + +.pnpm-store/ diff --git a/package.json b/package.json index 149463c4..43ed52d7 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,10 @@ "dev": "turbo run dev", "lint": "turbo run lint", "start": "turbo run start", + "test": "turbo run test", + "doc-quality": "turbo run doc-quality", + "test:unit": "turbo run test:unit", + "test:code-quality": "turbo run test:code-quality", "generate-embeddings": "turbo run generate-embeddings", "generate-embeddings:yes": "turbo run generate-embeddings:yes", "clean": "find packages -type d -name 'dist' -exec rm -rf {} +; find packages -type d -name '.turbo' -exec rm -rf {} +", diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts new file mode 100644 index 00000000..b090ee31 --- /dev/null +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -0,0 +1,167 @@ +import request from 'supertest'; + +const API_KEY = process.env.API_KEY || 'test'; +const API_URL = process.env.API_URL || 'http://localhost:3005'; +const GENERATION_TIMEOUT = 50000; +const COMPILATION_TIMEOUT = 30000; + +describe('Code Generation and Compilation Tests', () => { + const agent = request(API_URL); + + async function generateAndCompile( + project_name: string, + prompt_content: string, + index: number, + ): Promise { + const generation_prompt = `Test #${index}: Generate Cairo code for ${prompt_content} + + 1. First, register a new project named "${project_name}" using the cairocoder_register_project tool + 2. Then, generate the Cairo code using the cairocoder_generate_code tool + + If generation is successful: + - Return the generated Cairo code with syntax highlighting + + If generation fails: + - Return only the error message from the tool + - Do not try to fix or retry the generation + + Do not perform any additional actions.`; + const generateResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ + request: generation_prompt, + }); + + console.log( + 'GENERATION RESPONSE:', + JSON.stringify(generateResponse.body, null, 2), + ); + const sucessfulGeneration = generateResponse.body.output[0].text + .toLowerCase() + .includes('```cairo'); + + if ( + generateResponse.body.output[0].status !== 'success' || + !sucessfulGeneration + ) { + console.error('Generation failed:', generateResponse.body); + return false; + } + + console.log('Generated code successfully'); + + const compilation_prompt = `Test #${index}: Compile the project "${project_name}" using the scarb_compile_contract tool. + + After compilation, report whether it succeeded or failed. + + For successful compilation: Report "Compilation successful" and include any relevant output. + For failed compilation: Report "Compilation failed" and include the specific error messages. + + Only use the compilation tool and no other tools. + If another tool is used, instead or additionally to the compilation tool, report it as a failure.`; + + const compileResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ + request: compilation_prompt, + }); + + console.log( + 'COMPILATION RESPONSE:', + JSON.stringify(compileResponse.body, null, 2), + ); + + const sucessfulCompilation = + compileResponse.body.output[0].text + .toLowerCase() + .includes('compilation') && + !compileResponse.body.output[0].text.toLowerCase().includes('failure') && + !compileResponse.body.output[0].text.toLowerCase().includes('failed') && + !compileResponse.body.output[0].text.toLowerCase().includes('error'); + if ( + compileResponse.body.output[0].status !== 'success' || + !sucessfulCompilation + ) { + console.error('Compilation request failed:', compileResponse.body); + return false; + } + + console.log('END REQUEST ////////'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + return true; + } + + describe('Cairo Functions and Basic Algorithms', () => { + test('Factorial function', async () => { + // const project_name = 'factorial'; + // const prompt_content = "a Cairo function that calculates the factorial of a number"; + // const success = await generateAndCompile(project_name, prompt_content, 1); + // expect(success).toBe(true); + }, 100000); + + // test('Max value in array', async () => { + // const project_name = 'max_value'; + // const prompt_content = "a Cairo function that finds the maximum value in an array"; + // const success = await generateAndCompile(project_name, prompt_content, 2); + // expect(success).toBe(true); + // }, 100000); + + // test('Simple sorting algorithm', async () => { + // const project_name = 'sorting'; + // const prompt_content = "a sorting algorithm"; + // const success = await generateAndCompile(project_name, prompt_content, 3); + // expect(success).toBe(true); + // }, 100000); + // }); + + // describe('Simple Starknet Contracts', () => { + // test('Basic contract with storage', async () => { + // const project_name = 'basic_contract'; + // const prompt_content = "a basic Starknet contract with a storage variable and getter/setter functions"; + // const success = await generateAndCompile(project_name, prompt_content, 4); + // expect(success).toBe(true); + // }, 100000); + + // test('Counter contract', async () => { + // const project_name = 'counter'; + // const prompt_content = "a Starknet contract that maintains a counter with increment and decrement functions"; + // const success = await generateAndCompile(project_name, prompt_content, 5); + // expect(success).toBe(true); + // }, 100000); + + // test('Simple voting system', async () => { + // const project_name = 'voting'; + // const prompt_content = "a Starknet contract for a simple voting system where users can vote only once"; + // const success = await generateAndCompile(project_name, prompt_content, 6); + // expect(success).toBe(true); + // }, 100000); + // }); + + // describe('Standard and Complex Contracts', () => { + // test('ERC-20 token contract', async () => { + // const project_name = 'erc20'; + // const prompt_content = "a minimal Starknet ERC-20 token contract"; + // const success = await generateAndCompile(project_name, prompt_content, 7); + // expect(success).toBe(true); + // }, 100000); + + // test('ERC-721 NFT contract', async () => { + // const project_name = 'erc721'; + // const prompt_content = "a Starknet ERC-721 NFT contract with minting functionality"; + // const success = await generateAndCompile(project_name, prompt_content, 8); + // expect(success).toBe(true); + // }, 100000); + + // test('Multisig wallet contract', async () => { + // const project_name = 'multisig'; + // const prompt_content = "a Starknet multisig wallet contract that requires multiple approvals for transactions"; + // const success = await generateAndCompile(project_name, prompt_content, 9); + // expect(success).toBe(true); + // }, 100000); + }); +}); diff --git a/packages/agents/__tests__/answerGenerator.test.ts b/packages/agents/__tests__/unit/answerGenerator.test.ts similarity index 97% rename from packages/agents/__tests__/answerGenerator.test.ts rename to packages/agents/__tests__/unit/answerGenerator.test.ts index 8a758475..6db476be 100644 --- a/packages/agents/__tests__/answerGenerator.test.ts +++ b/packages/agents/__tests__/unit/answerGenerator.test.ts @@ -1,4 +1,4 @@ -import { AnswerGenerator } from '../src/core/pipeline/answerGenerator'; +import { AnswerGenerator } from '../../src/core/pipeline/answerGenerator'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { RagInput, @@ -6,23 +6,24 @@ import { RetrievedDocuments, RagSearchConfig, DocumentSource, -} from '../src/types/index'; +} from '../../src/types/index'; import { Document } from '@langchain/core/documents'; import { mockDeep, MockProxy } from 'jest-mock-extended'; import { IterableReadableStream } from '@langchain/core/utils/stream'; import { BaseMessage, BaseMessageChunk } from '@langchain/core/messages'; import { BaseLanguageModelInput } from '@langchain/core/language_models/base'; - // Mock the formatChatHistoryAsString utility -jest.mock('../src/utils/index', () => ({ +jest.mock('../../src/utils/index', () => ({ __esModule: true, - formatChatHistoryAsString: jest.fn().mockImplementation(() => 'mocked chat history'), + formatChatHistoryAsString: jest + .fn() + .mockImplementation(() => 'mocked chat history'), logger: { info: jest.fn(), debug: jest.fn(), error: jest.fn(), - } + }, })); // No need to separately mock the logger since it's now mocked as part of utils/index diff --git a/packages/agents/__tests__/documentRetriever.test.ts b/packages/agents/__tests__/unit/documentRetriever.test.ts similarity index 96% rename from packages/agents/__tests__/documentRetriever.test.ts rename to packages/agents/__tests__/unit/documentRetriever.test.ts index e7af11e9..74007efb 100644 --- a/packages/agents/__tests__/documentRetriever.test.ts +++ b/packages/agents/__tests__/unit/documentRetriever.test.ts @@ -1,22 +1,22 @@ -import { DocumentRetriever } from '../src/core/pipeline/documentRetriever'; +import { DocumentRetriever } from '../../src/core/pipeline/documentRetriever'; import { Embeddings } from '@langchain/core/embeddings'; import { DocumentSource, ProcessedQuery, RagSearchConfig, -} from '../src/types/index'; +} from '../../src/types/index'; import { Document } from '@langchain/core/documents'; import { mockDeep, MockProxy } from 'jest-mock-extended'; // Mock all utils including computeSimilarity and logger -jest.mock('../src/utils/index', () => ({ +jest.mock('../../src/utils/index', () => ({ __esModule: true, computeSimilarity: jest.fn().mockImplementation(() => 0.75), // Default high similarity logger: { info: jest.fn(), debug: jest.fn(), error: jest.fn(), - } + }, })); describe('DocumentRetriever', () => { @@ -158,7 +158,7 @@ describe('DocumentRetriever', () => { // Import the real computeSimilarity function to control scores const computeSimilarityMock = jest.requireMock( - '../src/utils/index', + '../../src/utils/index', ).computeSimilarity; // Set up different similarity scores for different documents diff --git a/packages/agents/__tests__/queryProcessor.test.ts b/packages/agents/__tests__/unit/queryProcessor.test.ts similarity index 94% rename from packages/agents/__tests__/queryProcessor.test.ts rename to packages/agents/__tests__/unit/queryProcessor.test.ts index 6c007041..9eec2c1a 100644 --- a/packages/agents/__tests__/queryProcessor.test.ts +++ b/packages/agents/__tests__/unit/queryProcessor.test.ts @@ -1,22 +1,24 @@ -import { QueryProcessor } from '../src/core/pipeline/queryProcessor'; +import { QueryProcessor } from '../../src/core/pipeline/queryProcessor'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { RagInput, RagSearchConfig, DocumentSource, -} from '../src/types/index'; +} from '../../src/types/index'; import { mockDeep, MockProxy } from 'jest-mock-extended'; import { AIMessage } from '@langchain/core/messages'; // Mock the logger -jest.mock('../src/utils/index', () => ({ +jest.mock('../../src/utils/index', () => ({ logger: { info: jest.fn(), debug: jest.fn(), error: jest.fn(), }, - formatChatHistoryAsString: jest.fn((history) => - history.map((message) => `${message._getType()}: ${message.content}`).join('\n') + formatChatHistoryAsString: jest.fn((history) => + history + .map((message) => `${message._getType()}: ${message.content}`) + .join('\n'), ), parseXMLContent: jest.fn((xml, tag) => { const regex = new RegExp(`<${tag}>(.*?)`, 'gs'); diff --git a/packages/agents/__tests__/ragAgentFactory.test.ts b/packages/agents/__tests__/unit/ragAgentFactory.test.ts similarity index 89% rename from packages/agents/__tests__/ragAgentFactory.test.ts rename to packages/agents/__tests__/unit/ragAgentFactory.test.ts index 0e6ecdff..60f3a554 100644 --- a/packages/agents/__tests__/ragAgentFactory.test.ts +++ b/packages/agents/__tests__/unit/ragAgentFactory.test.ts @@ -1,15 +1,15 @@ -import { RagAgentFactory } from '../src/core/agentFactory'; -import { RagPipeline } from '../src/core/pipeline/ragPipeline'; -import { AvailableAgents, LLMConfig, DocumentSource } from '../src/types'; +import { RagAgentFactory } from '../../src/core/agentFactory'; +import { RagPipeline } from '../../src/core/pipeline/ragPipeline'; +import { AvailableAgents, LLMConfig, DocumentSource } from '../../src/types'; import { Embeddings } from '@langchain/core/embeddings'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import { VectorStore } from '../src/db/postgresVectorStore'; +import { VectorStore } from '../../src/db/postgresVectorStore'; import { mockDeep, MockProxy } from 'jest-mock-extended'; import { BaseMessage } from '@langchain/core/messages'; import EventEmitter from 'events'; // Mock the agent configuration and RagPipeline -jest.mock('../src/config/agent', () => ({ +jest.mock('../../src/config/agent', () => ({ getAgentConfig: jest.fn().mockImplementation(() => ({ name: 'Cairo Coder', prompts: { @@ -27,7 +27,7 @@ jest.mock('../src/config/agent', () => ({ })), })); -jest.mock('../src/core/pipeline/ragPipeline', () => ({ +jest.mock('../../src/core/pipeline/ragPipeline', () => ({ RagPipeline: jest.fn().mockImplementation(() => ({ execute: jest.fn().mockReturnValue(new EventEmitter()), })), diff --git a/packages/agents/__tests__/ragPipeline.test.ts b/packages/agents/__tests__/unit/ragPipeline.test.ts similarity index 90% rename from packages/agents/__tests__/ragPipeline.test.ts rename to packages/agents/__tests__/unit/ragPipeline.test.ts index 20e23b1f..35a0b012 100644 --- a/packages/agents/__tests__/ragPipeline.test.ts +++ b/packages/agents/__tests__/unit/ragPipeline.test.ts @@ -1,7 +1,7 @@ -import { RagPipeline } from '../src/core/pipeline/ragPipeline'; -import { QueryProcessor } from '../src/core/pipeline/queryProcessor'; -import { DocumentRetriever } from '../src/core/pipeline/documentRetriever'; -import { AnswerGenerator } from '../src/core/pipeline/answerGenerator'; +import { RagPipeline } from '../../src/core/pipeline/ragPipeline'; +import { QueryProcessor } from '../../src/core/pipeline/queryProcessor'; +import { DocumentRetriever } from '../../src/core/pipeline/documentRetriever'; +import { AnswerGenerator } from '../../src/core/pipeline/answerGenerator'; import { Embeddings } from '@langchain/core/embeddings'; import { BookChunk, @@ -9,7 +9,7 @@ import { RagInput, RagSearchConfig, RetrievedDocuments, -} from '../src/types/index'; +} from '../../src/types/index'; import { Document } from '@langchain/core/documents'; import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { IterableReadableStream } from '@langchain/core/utils/stream'; @@ -18,17 +18,17 @@ import { mockDeep, MockProxy } from 'jest-mock-extended'; import EventEmitter from 'events'; // Mock the dependencies at the module level -jest.mock('../src/core/pipeline/queryProcessor'); -jest.mock('../src/core/pipeline/documentRetriever'); -jest.mock('../src/core/pipeline/answerGenerator'); +jest.mock('../../src/core/pipeline/queryProcessor'); +jest.mock('../../src/core/pipeline/documentRetriever'); +jest.mock('../../src/core/pipeline/answerGenerator'); // Mock the utils including logger -jest.mock('../src/utils/index', () => ({ +jest.mock('../../src/utils/index', () => ({ logger: { info: jest.fn(), debug: jest.fn(), error: jest.fn(), - } + }, })); describe('RagPipeline', () => { diff --git a/packages/agents/package.json b/packages/agents/package.json index 076c2685..db1fd45e 100644 --- a/packages/agents/package.json +++ b/packages/agents/package.json @@ -5,7 +5,9 @@ "scripts": { "build": "tsc -p tsconfig.json", "test": "jest", - "test-doc-quality": "ts-node src/tests/testDocQuality.ts" + "doc-quality": "ts-node src/doc-quality/testDocQuality.ts", + "test:unit": "jest --config jest.config.js --testMatch=\"**/__tests__/unit/**/*.test.[jt]s?(x)\"", + "test:code-quality": "jest --config jest.config.js --testMatch=\"**/__tests__/code-quality/**/*.test.[jt]s?(x)\"" }, "dependencies": { "@iarna/toml": "^2.2.5", diff --git a/packages/agents/src/tests/samples/all_quality.json b/packages/agents/src/doc-quality/samples/all_quality.json similarity index 100% rename from packages/agents/src/tests/samples/all_quality.json rename to packages/agents/src/doc-quality/samples/all_quality.json diff --git a/packages/agents/src/tests/samples/cairo_book_quality.json b/packages/agents/src/doc-quality/samples/cairo_book_quality.json similarity index 100% rename from packages/agents/src/tests/samples/cairo_book_quality.json rename to packages/agents/src/doc-quality/samples/cairo_book_quality.json diff --git a/packages/agents/src/tests/samples/openzeppelin_quality.json b/packages/agents/src/doc-quality/samples/openzeppelin_quality.json similarity index 100% rename from packages/agents/src/tests/samples/openzeppelin_quality.json rename to packages/agents/src/doc-quality/samples/openzeppelin_quality.json diff --git a/packages/agents/src/tests/samples/starknet_docs_quality.json b/packages/agents/src/doc-quality/samples/starknet_docs_quality.json similarity index 100% rename from packages/agents/src/tests/samples/starknet_docs_quality.json rename to packages/agents/src/doc-quality/samples/starknet_docs_quality.json diff --git a/packages/agents/src/tests/testDocQuality.ts b/packages/agents/src/doc-quality/testDocQuality.ts similarity index 100% rename from packages/agents/src/tests/testDocQuality.ts rename to packages/agents/src/doc-quality/testDocQuality.ts diff --git a/packages/backend/__tests__/compilation.test.ts b/packages/backend/__tests__/compilation.test.ts deleted file mode 100644 index 309048ac..00000000 --- a/packages/backend/__tests__/compilation.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -/** - * @group integration - */ - -import request from 'supertest'; - -const API_KEY = process.env.API_KEY; -const API_URL = process.env.API_URL; -const GENERATION_TIMEOUT = 50000; -const COMPILATION_TIMEOUT = 15000; - -describe('Code Generation and Compilation Tests', () => { - const agent = request(API_URL); - - async function generateAndCompile(prompt: string): Promise { - const generateResponse = await agent - .post('/api/key/request') - .set('Content-Type', 'application/json') - .set('x-api-key', API_KEY) - .send({ - request: `${prompt}. Do not compile it or do any other action after generating it, just return the code`, - }) - .timeout(GENERATION_TIMEOUT); - - if (generateResponse.body.output[0].status !== 'success') { - console.error('Generation failed:', generateResponse.body); - return false; - } - - console.log('Generated code successfully'); - - const compileResponse = await agent - .post('/api/key/request') - .set('Content-Type', 'application/json') - .set('x-api-key', API_KEY) - .send({ - request: - 'Compile the previous code, if there is an error do NOT fix it, just return the error', - }) - .timeout(COMPILATION_TIMEOUT); - - if (compileResponse.body.output[0].status !== 'success') { - console.error('Compilation request failed:', compileResponse.body); - return false; - } - - const responseText = compileResponse.body.response || ''; - const isSuccessful = - !responseText.includes('error') && - !responseText.includes('failed') && - !responseText.includes('cannot'); - - if (!isSuccessful) { - console.error('Compilation failed:', responseText); - } else { - console.log('Compilation successful'); - } - - return isSuccessful; - } - - describe('Cairo Functions and Basic Algorithms', () => { - test('Factorial function', async () => { - const prompt = - 'Generate a Cairo function that calculates the factorial of a number.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - - test('Max value in array', async () => { - const prompt = - 'Generate a Cairo function that finds the maximum value in an array.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - - test('Simple sorting algorithm', async () => { - const prompt = - 'Generate a Cairo implementation of a simple sorting algorithm.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - }); - - describe('Simple Starknet Contracts', () => { - test('Basic contract with storage', async () => { - const prompt = - 'Generate a basic Starknet contract with a storage variable and getter/setter functions.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - - test('Counter contract', async () => { - const prompt = - 'Generate a Starknet contract that maintains a counter with increment and decrement functions.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - - test('Simple voting system', async () => { - const prompt = - 'Generate a Starknet contract for a simple voting system where users can vote only once.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - }); - - describe('Standard and Complex Contracts', () => { - test('ERC-20 token contract', async () => { - const prompt = 'Generate a minimal Starknet ERC-20 token contract.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - - test('ERC-721 NFT contract', async () => { - const prompt = - 'Generate a Starknet ERC-721 NFT contract with minting functionality.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - - test('Multisig wallet contract', async () => { - const prompt = - 'Generate a Starknet multisig wallet contract that requires multiple approvals for transactions.'; - const success = await generateAndCompile(prompt); - expect(success).toBe(true); - }, 100000); - }); -}); diff --git a/packages/backend/__tests__/types/context.test.ts b/packages/backend/__tests__/unit/context.test.ts similarity index 100% rename from packages/backend/__tests__/types/context.test.ts rename to packages/backend/__tests__/unit/context.test.ts diff --git a/packages/backend/__tests__/server.test.ts b/packages/backend/__tests__/unit/server.test.ts similarity index 93% rename from packages/backend/__tests__/server.test.ts rename to packages/backend/__tests__/unit/server.test.ts index 6c3dbd96..6f4f1b67 100644 --- a/packages/backend/__tests__/server.test.ts +++ b/packages/backend/__tests__/unit/server.test.ts @@ -1,11 +1,11 @@ -import { createApplication } from '../src/server'; -import { Container } from '../src/config/context'; +import { createApplication } from '../../src/server'; +import { Container } from '../../src/config/context'; import express from 'express'; import { Server } from 'http'; import supertest from 'supertest'; describe('Server', () => { - jest.mock('../src/config/llm', () => ({ + jest.mock('../../src/config/llm', () => ({ initializeLLMConfig: jest.fn().mockResolvedValue({ defaultLLM: {}, fastLLM: {}, @@ -25,9 +25,8 @@ describe('Server', () => { getCairoByExampleDbConfig: jest.fn().mockReturnValue({}), })); - // Mock HTTP handling to avoid actual initialization - jest.mock('../src/config/http', () => ({ + jest.mock('../../src/config/http', () => ({ initializeHTTP: jest.fn(), })); diff --git a/packages/backend/jest.config.js b/packages/backend/jest.config.js index ec661178..8ddf4c7c 100644 --- a/packages/backend/jest.config.js +++ b/packages/backend/jest.config.js @@ -3,7 +3,6 @@ module.exports = { testEnvironment: 'node', roots: ['/src', '/__tests__'], testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'], - testPathIgnorePatterns: ['/__tests__/.*compilation\\.test\\.ts$'], transform: { '^.+\\.ts$': [ 'ts-jest', diff --git a/packages/backend/package.json b/packages/backend/package.json index 8d7ebaaf..d06b42ca 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -9,7 +9,8 @@ "test": "jest", "test:watch": "jest --watch", "test:coverage": "jest --coverage", - "check-types": "tsc --noEmit" + "check-types": "tsc --noEmit", + "test:unit": "jest --config jest.config.js --testMatch=\"**/__tests__/unit/**/*.test.[jt]s?(x)\"" }, "dependencies": { "@iarna/toml": "^2.2.5", diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index 71370409..b4d92734 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -11,6 +11,10 @@ "skipDefaultLibCheck": true, "types": ["node", "jest"] }, - "include": ["src", "**/*.test.ts"], + "include": [ + "src", + "**/*.test.ts", + "../agents/__tests__/code-quality/snak.test.ts" + ], "exclude": ["node_modules", "**/*.spec.ts"] } diff --git a/packages/ingester/__tests__/AsciiDocIngester.test.ts b/packages/ingester/__tests__/unit/AsciiDocIngester.test.ts similarity index 98% rename from packages/ingester/__tests__/AsciiDocIngester.test.ts rename to packages/ingester/__tests__/unit/AsciiDocIngester.test.ts index cd1cb2eb..48a8acf8 100644 --- a/packages/ingester/__tests__/AsciiDocIngester.test.ts +++ b/packages/ingester/__tests__/unit/AsciiDocIngester.test.ts @@ -3,8 +3,8 @@ import { BookChunk, DocumentSource } from '@cairo-coder/agents/types/index'; import { AsciiDocIngester, AsciiDocIngesterConfig, -} from '../src/ingesters/AsciiDocIngester'; -import { BookConfig, BookPageDto, ParsedSection } from '../src/utils/types'; +} from '../../src/ingesters/AsciiDocIngester'; +import { BookConfig, BookPageDto, ParsedSection } from '../../src/utils/types'; // Create a concrete implementation of AsciiDocIngester for testing class TestAsciiDocIngester extends AsciiDocIngester { diff --git a/packages/ingester/__tests__/IngesterFactory.test.ts b/packages/ingester/__tests__/unit/IngesterFactory.test.ts similarity index 75% rename from packages/ingester/__tests__/IngesterFactory.test.ts rename to packages/ingester/__tests__/unit/IngesterFactory.test.ts index d647453c..6c7643db 100644 --- a/packages/ingester/__tests__/IngesterFactory.test.ts +++ b/packages/ingester/__tests__/unit/IngesterFactory.test.ts @@ -1,18 +1,18 @@ -import { IngesterFactory } from '../src/IngesterFactory'; -import { CairoBookIngester } from '../src/ingesters/CairoBookIngester'; -import { StarknetDocsIngester } from '../src/ingesters/StarknetDocsIngester'; -import { StarknetFoundryIngester } from '../src/ingesters/StarknetFoundryIngester'; -import { CairoByExampleIngester } from '../src/ingesters/CairoByExampleIngester'; -import { OpenZeppelinDocsIngester } from '../src/ingesters/OpenZeppelinDocsIngester'; -import { BaseIngester } from '../src/BaseIngester'; +import { IngesterFactory } from '../../src/IngesterFactory'; +import { CairoBookIngester } from '../../src/ingesters/CairoBookIngester'; +import { StarknetDocsIngester } from '../../src/ingesters/StarknetDocsIngester'; +import { StarknetFoundryIngester } from '../../src/ingesters/StarknetFoundryIngester'; +import { CairoByExampleIngester } from '../../src/ingesters/CairoByExampleIngester'; +import { OpenZeppelinDocsIngester } from '../../src/ingesters/OpenZeppelinDocsIngester'; +import { BaseIngester } from '../../src/BaseIngester'; import { DocumentSource } from '@cairo-coder/agents/types/index'; // Mock the ingesters -jest.mock('../src/ingesters/CairoBookIngester'); -jest.mock('../src/ingesters/StarknetDocsIngester'); -jest.mock('../src/ingesters/StarknetFoundryIngester'); -jest.mock('../src/ingesters/CairoByExampleIngester'); -jest.mock('../src/ingesters/OpenZeppelinDocsIngester'); +jest.mock('../../src/ingesters/CairoBookIngester'); +jest.mock('../../src/ingesters/StarknetDocsIngester'); +jest.mock('../../src/ingesters/StarknetFoundryIngester'); +jest.mock('../../src/ingesters/CairoByExampleIngester'); +jest.mock('../../src/ingesters/OpenZeppelinDocsIngester'); describe('IngesterFactory', () => { beforeEach(() => { diff --git a/packages/ingester/__tests__/MarkdownIngester.test.ts b/packages/ingester/__tests__/unit/MarkdownIngester.test.ts similarity index 97% rename from packages/ingester/__tests__/MarkdownIngester.test.ts rename to packages/ingester/__tests__/unit/MarkdownIngester.test.ts index 1b8ed1a0..98a59afd 100644 --- a/packages/ingester/__tests__/MarkdownIngester.test.ts +++ b/packages/ingester/__tests__/unit/MarkdownIngester.test.ts @@ -1,6 +1,6 @@ -import { BookPageDto, isInsideCodeBlock } from '../src/shared'; +import { BookPageDto, isInsideCodeBlock } from '../../src/shared'; import { Document } from '@langchain/core/documents'; -import { MarkdownIngester } from '../src/ingesters/MarkdownIngester'; +import { MarkdownIngester } from '../../src/ingesters/MarkdownIngester'; import { DocumentSource } from '@cairo-coder/agents/types/index'; // Create a concrete implementation of the abstract MarkdownIngester for testing diff --git a/packages/ingester/__tests__/contentUtils.test.ts b/packages/ingester/__tests__/unit/contentUtils.test.ts similarity index 97% rename from packages/ingester/__tests__/contentUtils.test.ts rename to packages/ingester/__tests__/unit/contentUtils.test.ts index 90e8fee7..13b27cea 100644 --- a/packages/ingester/__tests__/contentUtils.test.ts +++ b/packages/ingester/__tests__/unit/contentUtils.test.ts @@ -1,4 +1,4 @@ -import { createAnchor, isInsideCodeBlock } from '../src/utils/contentUtils'; +import { createAnchor, isInsideCodeBlock } from '../../src/utils/contentUtils'; describe('createAnchor', () => { it('should handle undefined input', () => { diff --git a/packages/ingester/__tests__/shared.test.ts b/packages/ingester/__tests__/unit/shared.test.ts similarity index 94% rename from packages/ingester/__tests__/shared.test.ts rename to packages/ingester/__tests__/unit/shared.test.ts index 4855b284..10b58adb 100644 --- a/packages/ingester/__tests__/shared.test.ts +++ b/packages/ingester/__tests__/unit/shared.test.ts @@ -1,4 +1,4 @@ -import { createAnchor } from '../src/utils/contentUtils'; +import { createAnchor } from '../../src/utils/contentUtils'; describe('createAnchor', () => { it('should handle undefined input', () => { diff --git a/packages/ingester/__tests__/vectorStoreUtils.test.ts b/packages/ingester/__tests__/unit/vectorStoreUtils.test.ts similarity index 96% rename from packages/ingester/__tests__/vectorStoreUtils.test.ts rename to packages/ingester/__tests__/unit/vectorStoreUtils.test.ts index 7336bd86..77723341 100644 --- a/packages/ingester/__tests__/vectorStoreUtils.test.ts +++ b/packages/ingester/__tests__/unit/vectorStoreUtils.test.ts @@ -1,4 +1,4 @@ -import { findChunksToUpdateAndRemove } from '../src/utils/vectorStoreUtils'; +import { findChunksToUpdateAndRemove } from '../../src/utils/vectorStoreUtils'; import { Document } from '@langchain/core/documents'; describe('findChunksToUpdateAndRemove', () => { diff --git a/packages/ingester/package.json b/packages/ingester/package.json index af408c28..38bee6b3 100644 --- a/packages/ingester/package.json +++ b/packages/ingester/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "scripts": { "test": "jest", + "test:unit": "jest --config jest.config.js --testMatch=\"**/__tests__/unit/**/*.test.[jt]s?(x)\"", "build": "tsc -p tsconfig.json", "generate-embeddings": "node ./dist/src/generateEmbeddings.js", "generate-embeddings:yes": "node ./dist/src/generateEmbeddings.js -y" diff --git a/tests/database-connection.sh b/tests/database-connection.sh index 877d0604..a57d18f6 100755 --- a/tests/database-connection.sh +++ b/tests/database-connection.sh @@ -1,7 +1,5 @@ #!/bin/bash -echo "=== Testing database connection via /chat/completions endpoint ===" - # Préparation des données pour l'appel API (requête simple) REQUEST_DATA='{ "model": "gemini-2.5-flash", diff --git a/turbo.json b/turbo.json index 4985fcad..a0909faa 100644 --- a/turbo.json +++ b/turbo.json @@ -20,6 +20,18 @@ "dependsOn": ["^build"], "outputs": [] }, + "test:unit": { + "dependsOn": ["^build"], + "outputs": [] + }, + "doc-quality": { + "dependsOn": ["^build"], + "outputs": [] + }, + "test:code-quality": { + "dependsOn": ["^build"], + "outputs": [] + }, "start": { "dependsOn": ["build"], "cache": false, From 87e177dd482556da4ec4135c927d46b9bd3d200a Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 5 May 2025 19:08:57 +0200 Subject: [PATCH 20/47] fix: better handle env parameters for snak test file --- .github/workflows/backend.yml | 12 +- .gitignore | 1 - .../__tests__/code-quality/snak.test.ts | 141 +++++++++--------- 3 files changed, 85 insertions(+), 69 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 49f1bc2b..c7725d2c 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -187,9 +187,19 @@ jobs: echo "Waiting for server to start..." sleep 30 + - name: Create cairo code generation test env file + run: | + cd ./packages/agents + cat > .env.test << 'EOL' + + API_KEY="${{ secrets.SNAK_SERVER_KEY }}" + API_URL="http://localhost:${{ secrets.SNAK_SERVER_PORT }}" + + EOL + - name: Run cairo code generation test run: | - API_KEY="${{ secrets.SNAK_SERVER_KEY }}" API_URL="http://localhost:${{ secrets.SNAK_SERVER_PORT }}" pnpm run test:code-quality + pnpm run test:code-quality TEST_RESULT=$? diff --git a/.gitignore b/.gitignore index 978215d3..2ba7606f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ yarn-error.log .env.development.local .env.test.local .env.production.local -.env.test packages/ui/.env diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts index b090ee31..107e1970 100644 --- a/packages/agents/__tests__/code-quality/snak.test.ts +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -1,9 +1,15 @@ import request from 'supertest'; +require('dotenv').config({ path: '.env.test' }); -const API_KEY = process.env.API_KEY || 'test'; -const API_URL = process.env.API_URL || 'http://localhost:3005'; -const GENERATION_TIMEOUT = 50000; -const COMPILATION_TIMEOUT = 30000; +if (!process.env.API_KEY) { + throw new Error('API_KEY not found in .env.test file'); +} +if (!process.env.API_URL) { + throw new Error('API_URL not found in .env.test file'); +} + +const API_KEY = process.env.API_KEY; +const API_URL = process.env.API_URL; describe('Code Generation and Compilation Tests', () => { const agent = request(API_URL); @@ -98,70 +104,71 @@ describe('Code Generation and Compilation Tests', () => { describe('Cairo Functions and Basic Algorithms', () => { test('Factorial function', async () => { - // const project_name = 'factorial'; - // const prompt_content = "a Cairo function that calculates the factorial of a number"; - // const success = await generateAndCompile(project_name, prompt_content, 1); - // expect(success).toBe(true); + const project_name = 'factorial'; + const prompt_content = + 'a Cairo function that calculates the factorial of a number'; + const success = await generateAndCompile(project_name, prompt_content, 1); + expect(success).toBe(true); }, 100000); - // test('Max value in array', async () => { - // const project_name = 'max_value'; - // const prompt_content = "a Cairo function that finds the maximum value in an array"; - // const success = await generateAndCompile(project_name, prompt_content, 2); - // expect(success).toBe(true); - // }, 100000); - - // test('Simple sorting algorithm', async () => { - // const project_name = 'sorting'; - // const prompt_content = "a sorting algorithm"; - // const success = await generateAndCompile(project_name, prompt_content, 3); - // expect(success).toBe(true); - // }, 100000); - // }); - - // describe('Simple Starknet Contracts', () => { - // test('Basic contract with storage', async () => { - // const project_name = 'basic_contract'; - // const prompt_content = "a basic Starknet contract with a storage variable and getter/setter functions"; - // const success = await generateAndCompile(project_name, prompt_content, 4); - // expect(success).toBe(true); - // }, 100000); - - // test('Counter contract', async () => { - // const project_name = 'counter'; - // const prompt_content = "a Starknet contract that maintains a counter with increment and decrement functions"; - // const success = await generateAndCompile(project_name, prompt_content, 5); - // expect(success).toBe(true); - // }, 100000); - - // test('Simple voting system', async () => { - // const project_name = 'voting'; - // const prompt_content = "a Starknet contract for a simple voting system where users can vote only once"; - // const success = await generateAndCompile(project_name, prompt_content, 6); - // expect(success).toBe(true); - // }, 100000); - // }); - - // describe('Standard and Complex Contracts', () => { - // test('ERC-20 token contract', async () => { - // const project_name = 'erc20'; - // const prompt_content = "a minimal Starknet ERC-20 token contract"; - // const success = await generateAndCompile(project_name, prompt_content, 7); - // expect(success).toBe(true); - // }, 100000); - - // test('ERC-721 NFT contract', async () => { - // const project_name = 'erc721'; - // const prompt_content = "a Starknet ERC-721 NFT contract with minting functionality"; - // const success = await generateAndCompile(project_name, prompt_content, 8); - // expect(success).toBe(true); - // }, 100000); - - // test('Multisig wallet contract', async () => { - // const project_name = 'multisig'; - // const prompt_content = "a Starknet multisig wallet contract that requires multiple approvals for transactions"; - // const success = await generateAndCompile(project_name, prompt_content, 9); - // expect(success).toBe(true); - // }, 100000); + // test('Max value in array', async () => { + // const project_name = 'max_value'; + // const prompt_content = "a Cairo function that finds the maximum value in an array"; + // const success = await generateAndCompile(project_name, prompt_content, 2); + // expect(success).toBe(true); + // }, 100000); + + // test('Simple sorting algorithm', async () => { + // const project_name = 'sorting'; + // const prompt_content = "a sorting algorithm"; + // const success = await generateAndCompile(project_name, prompt_content, 3); + // expect(success).toBe(true); + // }, 100000); }); + + // describe('Simple Starknet Contracts', () => { + // test('Basic contract with storage', async () => { + // const project_name = 'basic_contract'; + // const prompt_content = "a basic Starknet contract with a storage variable and getter/setter functions"; + // const success = await generateAndCompile(project_name, prompt_content, 4); + // expect(success).toBe(true); + // }, 100000); + + // test('Counter contract', async () => { + // const project_name = 'counter'; + // const prompt_content = "a Starknet contract that maintains a counter with increment and decrement functions"; + // const success = await generateAndCompile(project_name, prompt_content, 5); + // expect(success).toBe(true); + // }, 100000); + + // test('Simple voting system', async () => { + // const project_name = 'voting'; + // const prompt_content = "a Starknet contract for a simple voting system where users can vote only once"; + // const success = await generateAndCompile(project_name, prompt_content, 6); + // expect(success).toBe(true); + // }, 100000); + // }); + + // describe('Standard and Complex Contracts', () => { + // test('ERC-20 token contract', async () => { + // const project_name = 'erc20'; + // const prompt_content = "a minimal Starknet ERC-20 token contract"; + // const success = await generateAndCompile(project_name, prompt_content, 7); + // expect(success).toBe(true); + // }, 100000); + + // test('ERC-721 NFT contract', async () => { + // const project_name = 'erc721'; + // const prompt_content = "a Starknet ERC-721 NFT contract with minting functionality"; + // const success = await generateAndCompile(project_name, prompt_content, 8); + // expect(success).toBe(true); + // }, 100000); + + // test('Multisig wallet contract', async () => { + // const project_name = 'multisig'; + // const prompt_content = "a Starknet multisig wallet contract that requires multiple approvals for transactions"; + // const success = await generateAndCompile(project_name, prompt_content, 9); + // expect(success).toBe(true); + // }, 100000); + // }); }); From 524df2ec60a5f3231c09dd0280facb78c4104489 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 5 May 2025 19:28:49 +0200 Subject: [PATCH 21/47] fix: fix import ingester --- packages/agents/.env.test | 4 ++++ packages/ingester/jest.config.js | 7 ++++++- packages/ingester/tsconfig.test.json | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 packages/agents/.env.test create mode 100644 packages/ingester/tsconfig.test.json diff --git a/packages/agents/.env.test b/packages/agents/.env.test new file mode 100644 index 00000000..ffe802c9 --- /dev/null +++ b/packages/agents/.env.test @@ -0,0 +1,4 @@ + +API_KEY="test" +API_URL="http://localhost:3005" + diff --git a/packages/ingester/jest.config.js b/packages/ingester/jest.config.js index 864a15d9..8ddf4c7c 100644 --- a/packages/ingester/jest.config.js +++ b/packages/ingester/jest.config.js @@ -4,7 +4,12 @@ module.exports = { roots: ['/src', '/__tests__'], testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'], transform: { - '^.+\\.ts$': ['ts-jest'], + '^.+\\.ts$': [ + 'ts-jest', + { + tsconfig: 'tsconfig.test.json', + }, + ], }, moduleNameMapper: { '^@/(.*)$': '/src/$1', diff --git a/packages/ingester/tsconfig.test.json b/packages/ingester/tsconfig.test.json new file mode 100644 index 00000000..877ede0a --- /dev/null +++ b/packages/ingester/tsconfig.test.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": ["node", "jest"], + "esModuleInterop": true + }, + "include": ["src", "__tests__"], + "exclude": ["node_modules"] +} From e67808bcaa7bbecf35d13e988b5b43646e7b19ad Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 5 May 2025 19:43:41 +0200 Subject: [PATCH 22/47] debug: debugging package export issue for ingester unit test --- .github/workflows/backend.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index c7725d2c..cf094285 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -36,6 +36,7 @@ jobs: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT - name: Setup pnpm cache + if: false uses: actions/cache@v4 with: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} @@ -49,9 +50,24 @@ jobs: - name: Install turbo run: pnpm add turbo@latest -g + - name: Clean previous builds + run: | + rm -rf packages/*/dist + rm -rf packages/*/node_modules/.cache + - name: Build run: turbo build + - name: Check files after build + run: | + find packages/backend/dist -type f | grep -i openai + ls -la packages/backend/dist/config/provider || echo "Directory does not exist" + + - name: Debug module resolution + run: | + cd packages/ingester + NODE_PATH=../.. node -e "try { const path = require.resolve('@cairo-coder/backend/config/provider/openai'); console.log('Module found at:', path); } catch(e) { console.log('Module not found:', e.message); console.log('Available modules:', require('fs').readdirSync('../backend/dist/config').join(', ')); }" + - name: Create config file run: | mkdir -p packages/agents From 5e01ec455dc6e9dce95d5e4a835ddb06b2487e57 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 5 May 2025 20:06:54 +0200 Subject: [PATCH 23/47] fix: modify tsconfig to remove wrong file that would make build of backend weird --- .../__tests__/code-quality/snak.test.js | 156 ++++++++++++++++++ .../__tests__/code-quality/snak.test.js.map | 1 + packages/backend/tsconfig.json | 6 +- 3 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 packages/agents/__tests__/code-quality/snak.test.js create mode 100644 packages/agents/__tests__/code-quality/snak.test.js.map diff --git a/packages/agents/__tests__/code-quality/snak.test.js b/packages/agents/__tests__/code-quality/snak.test.js new file mode 100644 index 00000000..97b55da7 --- /dev/null +++ b/packages/agents/__tests__/code-quality/snak.test.js @@ -0,0 +1,156 @@ +'use strict'; +var __importDefault = + (this && this.__importDefault) || + function (mod) { + return mod && mod.__esModule ? mod : { default: mod }; + }; +Object.defineProperty(exports, '__esModule', { value: true }); +const supertest_1 = __importDefault(require('supertest')); +require('dotenv').config({ path: '.env.test' }); +if (!process.env.API_KEY) { + throw new Error('API_KEY not found in .env.test file'); +} +if (!process.env.API_URL) { + throw new Error('API_URL not found in .env.test file'); +} +const API_KEY = process.env.API_KEY; +const API_URL = process.env.API_URL; +describe('Code Generation and Compilation Tests', () => { + const agent = (0, supertest_1.default)(API_URL); + async function generateAndCompile(project_name, prompt_content, index) { + const generation_prompt = `Test #${index}: Generate Cairo code for ${prompt_content} + + 1. First, register a new project named "${project_name}" using the cairocoder_register_project tool + 2. Then, generate the Cairo code using the cairocoder_generate_code tool + + If generation is successful: + - Return the generated Cairo code with syntax highlighting + + If generation fails: + - Return only the error message from the tool + - Do not try to fix or retry the generation + + Do not perform any additional actions.`; + const generateResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ + request: generation_prompt, + }); + console.log( + 'GENERATION RESPONSE:', + JSON.stringify(generateResponse.body, null, 2), + ); + const sucessfulGeneration = generateResponse.body.output[0].text + .toLowerCase() + .includes('```cairo'); + if ( + generateResponse.body.output[0].status !== 'success' || + !sucessfulGeneration + ) { + console.error('Generation failed:', generateResponse.body); + return false; + } + console.log('Generated code successfully'); + const compilation_prompt = `Test #${index}: Compile the project "${project_name}" using the scarb_compile_contract tool. + + After compilation, report whether it succeeded or failed. + + For successful compilation: Report "Compilation successful" and include any relevant output. + For failed compilation: Report "Compilation failed" and include the specific error messages. + + Only use the compilation tool and no other tools. + If another tool is used, instead or additionally to the compilation tool, report it as a failure.`; + const compileResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ + request: compilation_prompt, + }); + console.log( + 'COMPILATION RESPONSE:', + JSON.stringify(compileResponse.body, null, 2), + ); + const sucessfulCompilation = + compileResponse.body.output[0].text + .toLowerCase() + .includes('compilation') && + !compileResponse.body.output[0].text.toLowerCase().includes('failure') && + !compileResponse.body.output[0].text.toLowerCase().includes('failed') && + !compileResponse.body.output[0].text.toLowerCase().includes('error'); + if ( + compileResponse.body.output[0].status !== 'success' || + !sucessfulCompilation + ) { + console.error('Compilation request failed:', compileResponse.body); + return false; + } + console.log('END REQUEST ////////'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + return true; + } + describe('Cairo Functions and Basic Algorithms', () => { + test('Factorial function', async () => { + const project_name = 'factorial'; + const prompt_content = + 'a Cairo function that calculates the factorial of a number'; + const success = await generateAndCompile(project_name, prompt_content, 1); + expect(success).toBe(true); + }, 100000); + // test('Max value in array', async () => { + // const project_name = 'max_value'; + // const prompt_content = "a Cairo function that finds the maximum value in an array"; + // const success = await generateAndCompile(project_name, prompt_content, 2); + // expect(success).toBe(true); + // }, 100000); + // test('Simple sorting algorithm', async () => { + // const project_name = 'sorting'; + // const prompt_content = "a sorting algorithm"; + // const success = await generateAndCompile(project_name, prompt_content, 3); + // expect(success).toBe(true); + // }, 100000); + }); + // describe('Simple Starknet Contracts', () => { + // test('Basic contract with storage', async () => { + // const project_name = 'basic_contract'; + // const prompt_content = "a basic Starknet contract with a storage variable and getter/setter functions"; + // const success = await generateAndCompile(project_name, prompt_content, 4); + // expect(success).toBe(true); + // }, 100000); + // test('Counter contract', async () => { + // const project_name = 'counter'; + // const prompt_content = "a Starknet contract that maintains a counter with increment and decrement functions"; + // const success = await generateAndCompile(project_name, prompt_content, 5); + // expect(success).toBe(true); + // }, 100000); + // test('Simple voting system', async () => { + // const project_name = 'voting'; + // const prompt_content = "a Starknet contract for a simple voting system where users can vote only once"; + // const success = await generateAndCompile(project_name, prompt_content, 6); + // expect(success).toBe(true); + // }, 100000); + // }); + // describe('Standard and Complex Contracts', () => { + // test('ERC-20 token contract', async () => { + // const project_name = 'erc20'; + // const prompt_content = "a minimal Starknet ERC-20 token contract"; + // const success = await generateAndCompile(project_name, prompt_content, 7); + // expect(success).toBe(true); + // }, 100000); + // test('ERC-721 NFT contract', async () => { + // const project_name = 'erc721'; + // const prompt_content = "a Starknet ERC-721 NFT contract with minting functionality"; + // const success = await generateAndCompile(project_name, prompt_content, 8); + // expect(success).toBe(true); + // }, 100000); + // test('Multisig wallet contract', async () => { + // const project_name = 'multisig'; + // const prompt_content = "a Starknet multisig wallet contract that requires multiple approvals for transactions"; + // const success = await generateAndCompile(project_name, prompt_content, 9); + // expect(success).toBe(true); + // }, 100000); + // }); +}); +//# sourceMappingURL=snak.test.js.map diff --git a/packages/agents/__tests__/code-quality/snak.test.js.map b/packages/agents/__tests__/code-quality/snak.test.js.map new file mode 100644 index 00000000..bff66c18 --- /dev/null +++ b/packages/agents/__tests__/code-quality/snak.test.js.map @@ -0,0 +1 @@ +{"version":3,"file":"snak.test.js","sourceRoot":"","sources":["snak.test.ts"],"names":[],"mappings":";;;;;AAAA,0DAAgC;AAChC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAEhD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;AACzD,CAAC;AACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;AACpC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;AAEpC,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;IACrD,MAAM,KAAK,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,CAAC;IAE/B,KAAK,UAAU,kBAAkB,CAC/B,YAAoB,EACpB,cAAsB,EACtB,KAAa;QAEb,MAAM,iBAAiB,GAAG,SAAS,KAAK,6BAA6B,cAAc;;8CAEzC,YAAY;;;;;;;;;;2CAUf,CAAC;QACxC,MAAM,gBAAgB,GAAG,MAAM,KAAK;aACjC,IAAI,CAAC,kBAAkB,CAAC;aACxB,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC;aACvC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC;aACzB,IAAI,CAAC;YACJ,OAAO,EAAE,iBAAiB;SAC3B,CAAC,CAAC;QAEL,OAAO,CAAC,GAAG,CACT,sBAAsB,EACtB,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAC/C,CAAC;QACF,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;aAC7D,WAAW,EAAE;aACb,QAAQ,CAAC,UAAU,CAAC,CAAC;QAExB,IACE,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS;YACpD,CAAC,mBAAmB,EACpB,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAC3D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAE3C,MAAM,kBAAkB,GAAG,SAAS,KAAK,0BAA0B,YAAY;;;;;;;;sGAQmB,CAAC;QAEnG,MAAM,eAAe,GAAG,MAAM,KAAK;aAChC,IAAI,CAAC,kBAAkB,CAAC;aACxB,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC;aACvC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC;aACzB,IAAI,CAAC;YACJ,OAAO,EAAE,kBAAkB;SAC5B,CAAC,CAAC;QAEL,OAAO,CAAC,GAAG,CACT,uBAAuB,EACvB,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAC9C,CAAC;QAEF,MAAM,oBAAoB,GACxB,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;aAChC,WAAW,EAAE;aACb,QAAQ,CAAC,aAAa,CAAC;YAC1B,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtE,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACrE,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvE,IACE,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS;YACnD,CAAC,oBAAoB,EACrB,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC;YACnE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAE1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ,CAAC,sCAAsC,EAAE,GAAG,EAAE;QACpD,IAAI,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;YACpC,MAAM,YAAY,GAAG,WAAW,CAAC;YACjC,MAAM,cAAc,GAClB,4DAA4D,CAAC;YAC/D,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;YAC1E,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC,EAAE,MAAM,CAAC,CAAC;QAEX,2CAA2C;QAC3C,sCAAsC;QACtC,wFAAwF;QACxF,+EAA+E;QAC/E,gCAAgC;QAChC,cAAc;QAEd,iDAAiD;QACjD,oCAAoC;QACpC,kDAAkD;QAClD,+EAA+E;QAC/E,gCAAgC;QAChC,cAAc;IAChB,CAAC,CAAC,CAAC;IAEH,gDAAgD;IAChD,sDAAsD;IACtD,6CAA6C;IAC7C,8GAA8G;IAC9G,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,2CAA2C;IAC3C,sCAAsC;IACtC,oHAAoH;IACpH,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,+CAA+C;IAC/C,qCAAqC;IACrC,8GAA8G;IAC9G,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAChB,MAAM;IAEN,qDAAqD;IACrD,gDAAgD;IAChD,oCAAoC;IACpC,yEAAyE;IACzE,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,+CAA+C;IAC/C,qCAAqC;IACrC,2FAA2F;IAC3F,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,mDAAmD;IACnD,uCAAuC;IACvC,sHAAsH;IACtH,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAChB,MAAM;AACR,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index b4d92734..71370409 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -11,10 +11,6 @@ "skipDefaultLibCheck": true, "types": ["node", "jest"] }, - "include": [ - "src", - "**/*.test.ts", - "../agents/__tests__/code-quality/snak.test.ts" - ], + "include": ["src", "**/*.test.ts"], "exclude": ["node_modules", "**/*.spec.ts"] } From b748085baeebcc82e7c3a577eea72e801b389096 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 5 May 2025 20:07:57 +0200 Subject: [PATCH 24/47] fix: remove useless .js in code quality repo --- .../__tests__/code-quality/snak.test.js | 156 ------------------ .../__tests__/code-quality/snak.test.js.map | 1 - 2 files changed, 157 deletions(-) delete mode 100644 packages/agents/__tests__/code-quality/snak.test.js delete mode 100644 packages/agents/__tests__/code-quality/snak.test.js.map diff --git a/packages/agents/__tests__/code-quality/snak.test.js b/packages/agents/__tests__/code-quality/snak.test.js deleted file mode 100644 index 97b55da7..00000000 --- a/packages/agents/__tests__/code-quality/snak.test.js +++ /dev/null @@ -1,156 +0,0 @@ -'use strict'; -var __importDefault = - (this && this.__importDefault) || - function (mod) { - return mod && mod.__esModule ? mod : { default: mod }; - }; -Object.defineProperty(exports, '__esModule', { value: true }); -const supertest_1 = __importDefault(require('supertest')); -require('dotenv').config({ path: '.env.test' }); -if (!process.env.API_KEY) { - throw new Error('API_KEY not found in .env.test file'); -} -if (!process.env.API_URL) { - throw new Error('API_URL not found in .env.test file'); -} -const API_KEY = process.env.API_KEY; -const API_URL = process.env.API_URL; -describe('Code Generation and Compilation Tests', () => { - const agent = (0, supertest_1.default)(API_URL); - async function generateAndCompile(project_name, prompt_content, index) { - const generation_prompt = `Test #${index}: Generate Cairo code for ${prompt_content} - - 1. First, register a new project named "${project_name}" using the cairocoder_register_project tool - 2. Then, generate the Cairo code using the cairocoder_generate_code tool - - If generation is successful: - - Return the generated Cairo code with syntax highlighting - - If generation fails: - - Return only the error message from the tool - - Do not try to fix or retry the generation - - Do not perform any additional actions.`; - const generateResponse = await agent - .post('/api/key/request') - .set('Content-Type', 'application/json') - .set('x-api-key', API_KEY) - .send({ - request: generation_prompt, - }); - console.log( - 'GENERATION RESPONSE:', - JSON.stringify(generateResponse.body, null, 2), - ); - const sucessfulGeneration = generateResponse.body.output[0].text - .toLowerCase() - .includes('```cairo'); - if ( - generateResponse.body.output[0].status !== 'success' || - !sucessfulGeneration - ) { - console.error('Generation failed:', generateResponse.body); - return false; - } - console.log('Generated code successfully'); - const compilation_prompt = `Test #${index}: Compile the project "${project_name}" using the scarb_compile_contract tool. - - After compilation, report whether it succeeded or failed. - - For successful compilation: Report "Compilation successful" and include any relevant output. - For failed compilation: Report "Compilation failed" and include the specific error messages. - - Only use the compilation tool and no other tools. - If another tool is used, instead or additionally to the compilation tool, report it as a failure.`; - const compileResponse = await agent - .post('/api/key/request') - .set('Content-Type', 'application/json') - .set('x-api-key', API_KEY) - .send({ - request: compilation_prompt, - }); - console.log( - 'COMPILATION RESPONSE:', - JSON.stringify(compileResponse.body, null, 2), - ); - const sucessfulCompilation = - compileResponse.body.output[0].text - .toLowerCase() - .includes('compilation') && - !compileResponse.body.output[0].text.toLowerCase().includes('failure') && - !compileResponse.body.output[0].text.toLowerCase().includes('failed') && - !compileResponse.body.output[0].text.toLowerCase().includes('error'); - if ( - compileResponse.body.output[0].status !== 'success' || - !sucessfulCompilation - ) { - console.error('Compilation request failed:', compileResponse.body); - return false; - } - console.log('END REQUEST ////////'); - await new Promise((resolve) => setTimeout(resolve, 5000)); - return true; - } - describe('Cairo Functions and Basic Algorithms', () => { - test('Factorial function', async () => { - const project_name = 'factorial'; - const prompt_content = - 'a Cairo function that calculates the factorial of a number'; - const success = await generateAndCompile(project_name, prompt_content, 1); - expect(success).toBe(true); - }, 100000); - // test('Max value in array', async () => { - // const project_name = 'max_value'; - // const prompt_content = "a Cairo function that finds the maximum value in an array"; - // const success = await generateAndCompile(project_name, prompt_content, 2); - // expect(success).toBe(true); - // }, 100000); - // test('Simple sorting algorithm', async () => { - // const project_name = 'sorting'; - // const prompt_content = "a sorting algorithm"; - // const success = await generateAndCompile(project_name, prompt_content, 3); - // expect(success).toBe(true); - // }, 100000); - }); - // describe('Simple Starknet Contracts', () => { - // test('Basic contract with storage', async () => { - // const project_name = 'basic_contract'; - // const prompt_content = "a basic Starknet contract with a storage variable and getter/setter functions"; - // const success = await generateAndCompile(project_name, prompt_content, 4); - // expect(success).toBe(true); - // }, 100000); - // test('Counter contract', async () => { - // const project_name = 'counter'; - // const prompt_content = "a Starknet contract that maintains a counter with increment and decrement functions"; - // const success = await generateAndCompile(project_name, prompt_content, 5); - // expect(success).toBe(true); - // }, 100000); - // test('Simple voting system', async () => { - // const project_name = 'voting'; - // const prompt_content = "a Starknet contract for a simple voting system where users can vote only once"; - // const success = await generateAndCompile(project_name, prompt_content, 6); - // expect(success).toBe(true); - // }, 100000); - // }); - // describe('Standard and Complex Contracts', () => { - // test('ERC-20 token contract', async () => { - // const project_name = 'erc20'; - // const prompt_content = "a minimal Starknet ERC-20 token contract"; - // const success = await generateAndCompile(project_name, prompt_content, 7); - // expect(success).toBe(true); - // }, 100000); - // test('ERC-721 NFT contract', async () => { - // const project_name = 'erc721'; - // const prompt_content = "a Starknet ERC-721 NFT contract with minting functionality"; - // const success = await generateAndCompile(project_name, prompt_content, 8); - // expect(success).toBe(true); - // }, 100000); - // test('Multisig wallet contract', async () => { - // const project_name = 'multisig'; - // const prompt_content = "a Starknet multisig wallet contract that requires multiple approvals for transactions"; - // const success = await generateAndCompile(project_name, prompt_content, 9); - // expect(success).toBe(true); - // }, 100000); - // }); -}); -//# sourceMappingURL=snak.test.js.map diff --git a/packages/agents/__tests__/code-quality/snak.test.js.map b/packages/agents/__tests__/code-quality/snak.test.js.map deleted file mode 100644 index bff66c18..00000000 --- a/packages/agents/__tests__/code-quality/snak.test.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"snak.test.js","sourceRoot":"","sources":["snak.test.ts"],"names":[],"mappings":";;;;;AAAA,0DAAgC;AAChC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAEhD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;AACzD,CAAC;AACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;AACpC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;AAEpC,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;IACrD,MAAM,KAAK,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,CAAC;IAE/B,KAAK,UAAU,kBAAkB,CAC/B,YAAoB,EACpB,cAAsB,EACtB,KAAa;QAEb,MAAM,iBAAiB,GAAG,SAAS,KAAK,6BAA6B,cAAc;;8CAEzC,YAAY;;;;;;;;;;2CAUf,CAAC;QACxC,MAAM,gBAAgB,GAAG,MAAM,KAAK;aACjC,IAAI,CAAC,kBAAkB,CAAC;aACxB,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC;aACvC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC;aACzB,IAAI,CAAC;YACJ,OAAO,EAAE,iBAAiB;SAC3B,CAAC,CAAC;QAEL,OAAO,CAAC,GAAG,CACT,sBAAsB,EACtB,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAC/C,CAAC;QACF,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;aAC7D,WAAW,EAAE;aACb,QAAQ,CAAC,UAAU,CAAC,CAAC;QAExB,IACE,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS;YACpD,CAAC,mBAAmB,EACpB,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAC3D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAE3C,MAAM,kBAAkB,GAAG,SAAS,KAAK,0BAA0B,YAAY;;;;;;;;sGAQmB,CAAC;QAEnG,MAAM,eAAe,GAAG,MAAM,KAAK;aAChC,IAAI,CAAC,kBAAkB,CAAC;aACxB,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC;aACvC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC;aACzB,IAAI,CAAC;YACJ,OAAO,EAAE,kBAAkB;SAC5B,CAAC,CAAC;QAEL,OAAO,CAAC,GAAG,CACT,uBAAuB,EACvB,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAC9C,CAAC;QAEF,MAAM,oBAAoB,GACxB,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;aAChC,WAAW,EAAE;aACb,QAAQ,CAAC,aAAa,CAAC;YAC1B,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtE,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACrE,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvE,IACE,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS;YACnD,CAAC,oBAAoB,EACrB,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC;YACnE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAE1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ,CAAC,sCAAsC,EAAE,GAAG,EAAE;QACpD,IAAI,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;YACpC,MAAM,YAAY,GAAG,WAAW,CAAC;YACjC,MAAM,cAAc,GAClB,4DAA4D,CAAC;YAC/D,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;YAC1E,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC,EAAE,MAAM,CAAC,CAAC;QAEX,2CAA2C;QAC3C,sCAAsC;QACtC,wFAAwF;QACxF,+EAA+E;QAC/E,gCAAgC;QAChC,cAAc;QAEd,iDAAiD;QACjD,oCAAoC;QACpC,kDAAkD;QAClD,+EAA+E;QAC/E,gCAAgC;QAChC,cAAc;IAChB,CAAC,CAAC,CAAC;IAEH,gDAAgD;IAChD,sDAAsD;IACtD,6CAA6C;IAC7C,8GAA8G;IAC9G,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,2CAA2C;IAC3C,sCAAsC;IACtC,oHAAoH;IACpH,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,+CAA+C;IAC/C,qCAAqC;IACrC,8GAA8G;IAC9G,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAChB,MAAM;IAEN,qDAAqD;IACrD,gDAAgD;IAChD,oCAAoC;IACpC,yEAAyE;IACzE,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,+CAA+C;IAC/C,qCAAqC;IACrC,2FAA2F;IAC3F,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAEhB,mDAAmD;IACnD,uCAAuC;IACvC,sHAAsH;IACtH,iFAAiF;IACjF,kCAAkC;IAClC,gBAAgB;IAChB,MAAM;AACR,CAAC,CAAC,CAAC"} \ No newline at end of file From 9632f5d166274c700dfd3a1f615316cf550a0bf0 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 5 May 2025 20:34:26 +0200 Subject: [PATCH 25/47] fix: make snak.test return true to test whole ci, remove .env.test --- packages/agents/.env.test | 4 ---- packages/agents/__tests__/code-quality/snak.test.ts | 10 +++++----- 2 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 packages/agents/.env.test diff --git a/packages/agents/.env.test b/packages/agents/.env.test deleted file mode 100644 index ffe802c9..00000000 --- a/packages/agents/.env.test +++ /dev/null @@ -1,4 +0,0 @@ - -API_KEY="test" -API_URL="http://localhost:3005" - diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts index 107e1970..46485162 100644 --- a/packages/agents/__tests__/code-quality/snak.test.ts +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -104,11 +104,11 @@ describe('Code Generation and Compilation Tests', () => { describe('Cairo Functions and Basic Algorithms', () => { test('Factorial function', async () => { - const project_name = 'factorial'; - const prompt_content = - 'a Cairo function that calculates the factorial of a number'; - const success = await generateAndCompile(project_name, prompt_content, 1); - expect(success).toBe(true); + // const project_name = 'factorial'; + // const prompt_content = + // 'a Cairo function that calculates the factorial of a number'; + // const success = await generateAndCompile(project_name, prompt_content, 1); + // expect(success).toBe(true); }, 100000); // test('Max value in array', async () => { From f5e3759a6c84dcf2d652d20486ad9a41f4da6e4f Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 11:14:04 +0200 Subject: [PATCH 26/47] feat: improve error displaying when snak test failures --- .github/workflows/backend.yml | 22 +- .../__tests__/code-quality/snak.test.ts | 207 ++++++++++-------- {tests => scripts}/database-connection.sh | 0 {tests => scripts}/integration-tests.sh | 0 scripts/snak-test.sh | 14 ++ 5 files changed, 140 insertions(+), 103 deletions(-) rename {tests => scripts}/database-connection.sh (100%) rename {tests => scripts}/integration-tests.sh (100%) create mode 100755 scripts/snak-test.sh diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index cf094285..8c0794ec 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -123,15 +123,15 @@ jobs: echo "Waiting for services to be ready..." sleep 20 - chmod +x ./tests/integration-tests.sh - chmod +x ./tests/database-connection.sh + chmod +x ./scripts/integration-tests.sh + chmod +x ./scripts/database-connection.sh echo -e "\n=== Running basic integration tests ===" - ./tests/integration-tests.sh + ./scripts/integration-tests.sh INTEGRATION_RESULT=$? echo -e "\n=== Running database connection test via chat/completions endpoint ===" - ./tests/database-connection.sh + ./scripts/database-connection.sh DB_CONNECTION_RESULT=$? if [ $INTEGRATION_RESULT -ne 0 ] || [ $DB_CONNECTION_RESULT -ne 0 ]; then @@ -215,18 +215,8 @@ jobs: - name: Run cairo code generation test run: | - pnpm run test:code-quality - - TEST_RESULT=$? - - pkill -f "start:server" || true - - if [ $TEST_RESULT -ne 0 ]; then - echo "❌ API test failed!" - exit 1 - else - echo "✅ API test passed!" - fi + chmod +x ./scripts/snak-test.sh + bash scripts/snak-test.sh # - name: Push docker image # run: docker push ${{ github.repository }}:${{ github.sha }} diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts index 46485162..84e60c0e 100644 --- a/packages/agents/__tests__/code-quality/snak.test.ts +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -18,97 +18,130 @@ describe('Code Generation and Compilation Tests', () => { project_name: string, prompt_content: string, index: number, - ): Promise { - const generation_prompt = `Test #${index}: Generate Cairo code for ${prompt_content} - - 1. First, register a new project named "${project_name}" using the cairocoder_register_project tool - 2. Then, generate the Cairo code using the cairocoder_generate_code tool - - If generation is successful: - - Return the generated Cairo code with syntax highlighting - - If generation fails: - - Return only the error message from the tool - - Do not try to fix or retry the generation - - Do not perform any additional actions.`; - const generateResponse = await agent - .post('/api/key/request') - .set('Content-Type', 'application/json') - .set('x-api-key', API_KEY) - .send({ - request: generation_prompt, - }); - - console.log( - 'GENERATION RESPONSE:', - JSON.stringify(generateResponse.body, null, 2), - ); - const sucessfulGeneration = generateResponse.body.output[0].text - .toLowerCase() - .includes('```cairo'); - - if ( - generateResponse.body.output[0].status !== 'success' || - !sucessfulGeneration - ) { - console.error('Generation failed:', generateResponse.body); - return false; - } - - console.log('Generated code successfully'); - - const compilation_prompt = `Test #${index}: Compile the project "${project_name}" using the scarb_compile_contract tool. - - After compilation, report whether it succeeded or failed. - - For successful compilation: Report "Compilation successful" and include any relevant output. - For failed compilation: Report "Compilation failed" and include the specific error messages. - - Only use the compilation tool and no other tools. - If another tool is used, instead or additionally to the compilation tool, report it as a failure.`; - - const compileResponse = await agent - .post('/api/key/request') - .set('Content-Type', 'application/json') - .set('x-api-key', API_KEY) - .send({ - request: compilation_prompt, - }); - - console.log( - 'COMPILATION RESPONSE:', - JSON.stringify(compileResponse.body, null, 2), - ); - - const sucessfulCompilation = - compileResponse.body.output[0].text + ): Promise<{success: boolean; error?: string}> { + console.log(`\n=== Test #${index}: ${project_name} ===`); + console.log(`Generating code for: ${prompt_content}`); + + try { + const generation_prompt = `Test #${index}: Generate Cairo code for ${prompt_content} + + 1. First, register a new project named "${project_name}" using the cairocoder_register_project tool + 2. Then, generate the Cairo code using the cairocoder_generate_code tool + + If generation is successful: + - Return the generated Cairo code with syntax highlighting + + If generation fails: + - Return only the error message from the tool + - Do not try to fix or retry the generation + + Do not perform any additional actions.`; + const generateResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ + request: generation_prompt, + }); + + console.log('CODE GENERATION STATUS:', generateResponse.status); + + if (generateResponse.status !== 201) { + return { + success: false, + error: `Generation HTTP request failed with status ${generateResponse.status}: ${JSON.stringify(generateResponse.body)}` + }; + } + + console.log('CODE GENERATION RESPONSE:', JSON.stringify(generateResponse.body.output[0], null, 2)); + const sucessfulGeneration = generateResponse.body.output[0].text .toLowerCase() - .includes('compilation') && - !compileResponse.body.output[0].text.toLowerCase().includes('failure') && - !compileResponse.body.output[0].text.toLowerCase().includes('failed') && - !compileResponse.body.output[0].text.toLowerCase().includes('error'); - if ( - compileResponse.body.output[0].status !== 'success' || - !sucessfulCompilation - ) { - console.error('Compilation request failed:', compileResponse.body); - return false; - } - - console.log('END REQUEST ////////'); - await new Promise((resolve) => setTimeout(resolve, 5000)); - - return true; + .includes('```cairo'); + + if ( + generateResponse.body.output[0].status !== 'success' || + !sucessfulGeneration + ) { + return { + success: false, + error: `Generation failed: ${JSON.stringify(generateResponse.body.output[0].text)}` + }; + } + + console.log('✅ Code generated successfully'); + + const compilation_prompt = `Test #${index}: Compile the project "${project_name}" using the scarb_compile_contract tool. + + After compilation, report whether it succeeded or failed. + + For successful compilation: Report "Compilation successful" and include any relevant output. + For failed compilation: Report "Compilation failed" and include the specific error messages. + + Only use the compilation tool and no other tools. + If another tool is used, instead or additionally to the compilation tool, report it as a failure.`; + + const compileResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ + request: compilation_prompt, + }); + + + console.log('COMPILATION STATUS:', compileResponse.status); + + if (compileResponse.status !== 201) { + return { + success: false, + error: `Compilation HTTP request failed with status ${compileResponse.status}: ${JSON.stringify(compileResponse.body)}` + }; + } + + console.log('COMPILATION RESPONSE:', JSON.stringify(compileResponse.body.output[0], null, 2)); + + const sucessfulCompilation = + compileResponse.body.output[0].text + .toLowerCase() + .includes('compilation') && + !compileResponse.body.output[0].text.toLowerCase().includes('failure') && + !compileResponse.body.output[0].text.toLowerCase().includes('failed') && + !compileResponse.body.output[0].text.toLowerCase().includes('error'); + + if ( + compileResponse.body.output[0].status !== 'success' || + !sucessfulCompilation + ) { + return { + success: false, + error: `Compilation failed: ${JSON.stringify(compileResponse.body.output[0].text)}` + }; + } + + console.log('✅ Compilation successful'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + return { success: true }; + } catch (error) { + console.error(`❌ Unexpected error in Test #${index}:`, error); + return { + success: false, + error: `Unexpected error: ${error.message}` + }; } +} describe('Cairo Functions and Basic Algorithms', () => { - test('Factorial function', async () => { - // const project_name = 'factorial'; - // const prompt_content = - // 'a Cairo function that calculates the factorial of a number'; - // const success = await generateAndCompile(project_name, prompt_content, 1); - // expect(success).toBe(true); + test('Fibonacci function', async () => { + const project_name = 'fibonacci'; + const prompt_content = 'a Cairo function that calculates the Fibonacci sequence'; + const result = await generateAndCompile(project_name, prompt_content, 1); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); }, 100000); // test('Max value in array', async () => { diff --git a/tests/database-connection.sh b/scripts/database-connection.sh similarity index 100% rename from tests/database-connection.sh rename to scripts/database-connection.sh diff --git a/tests/integration-tests.sh b/scripts/integration-tests.sh similarity index 100% rename from tests/integration-tests.sh rename to scripts/integration-tests.sh diff --git a/scripts/snak-test.sh b/scripts/snak-test.sh new file mode 100755 index 00000000..6fe52f54 --- /dev/null +++ b/scripts/snak-test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +pnpm run test:code-quality --force | tee test-output.log + +TEST_RESULT=${PIPESTATUS[0]} + +if [ $TEST_RESULT -ne 0 ]; then + echo "❌ API test failed with exit code $TEST_RESULT!" + echo "=============== Test Logs ===============" + grep -A 5 -E "TEST FAILED|❌" test-output.log || true + exit 1 +else + echo "✅ API test passed!" +fi \ No newline at end of file From a68f3eafe1419c10e3c37affd69b7dbb7a20f89a Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 11:32:34 +0200 Subject: [PATCH 27/47] debug: server connection to test-snak --- .github/workflows/backend.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 8c0794ec..2ad0dbde 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -201,7 +201,27 @@ jobs: pnpm run build --filter="@snakagent/server..." && lerna run start --scope "@snakagent/server" & echo "Waiting for server to start..." - sleep 30 + sleep 45 + + max_retries=10 + retry_count=0 + server_ready=false + + while [ $retry_count -lt $max_retries ] && [ "$server_ready" = false ]; do + if curl -s http://localhost:${{ secrets.SNAK_SERVER_PORT }}/api/key/status > /dev/null; then + echo "Server is up and running!" + server_ready=true + else + echo "Server not ready yet, retrying in 5 seconds..." + sleep 5 + retry_count=$((retry_count+1)) + fi + done + + if [ "$server_ready" = false ]; then + echo "Server failed to start after $max_retries retries" + exit 1 + fi - name: Create cairo code generation test env file run: | From d32b660c61daa2b1a90bcc720dd4b15c1966d90b Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 11:50:46 +0200 Subject: [PATCH 28/47] debug: add test basic API connectivity and fix api key in first connection --- .github/workflows/backend.yml | 18 +++++++++++++++--- .../agents/__tests__/code-quality/snak.test.ts | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 2ad0dbde..eb247076 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -201,14 +201,14 @@ jobs: pnpm run build --filter="@snakagent/server..." && lerna run start --scope "@snakagent/server" & echo "Waiting for server to start..." - sleep 45 + sleep 60 max_retries=10 retry_count=0 server_ready=false while [ $retry_count -lt $max_retries ] && [ "$server_ready" = false ]; do - if curl -s http://localhost:${{ secrets.SNAK_SERVER_PORT }}/api/key/status > /dev/null; then + if curl -s -H "x-api-key: ${{ secrets.SNAK_SERVER_KEY }}" http://localhost:${{ secrets.SNAK_SERVER_PORT }}/api/key/status > /dev/null; then echo "Server is up and running!" server_ready=true else @@ -222,7 +222,19 @@ jobs: echo "Server failed to start after $max_retries retries" exit 1 fi - + + - name: Test basic API connectivity + run: | + echo "Testing basic API connectivity..." + response=$(curl -s -H "Content-Type: application/json" -H "x-api-key: ${{ secrets.SNAK_SERVER_KEY }}" -X POST -d '{"request": "Hello World"}' http://localhost:${{ secrets.SNAK_SERVER_PORT }}/api/key/request) + echo "Response: $response" + if echo "$response" | grep -q "Hello"; then + echo "✅ Basic API test passed!" + else + echo "❌ Basic API test failed!" + exit 1 + fi + - name: Create cairo code generation test env file run: | cd ./packages/agents diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts index 84e60c0e..98c063d6 100644 --- a/packages/agents/__tests__/code-quality/snak.test.ts +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -131,7 +131,20 @@ describe('Code Generation and Compilation Tests', () => { } } - describe('Cairo Functions and Basic Algorithms', () => { +describe('Cairo Functions and Basic Algorithms', () => { + + test('Hello World test', async () => { + const project_name = 'hello_world'; + const prompt_content = 'a simple Hello World function in Cairo'; + const result = await generateAndCompile(project_name, prompt_content, 0); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + test('Fibonacci function', async () => { const project_name = 'fibonacci'; const prompt_content = 'a Cairo function that calculates the Fibonacci sequence'; From 6fcfae74ebabea5fbd33ee8e19e7794a3703ba1e Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 11:57:40 +0200 Subject: [PATCH 29/47] fix: ai provder api key in snak env --- .github/workflows/backend.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index eb247076..5560e000 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -163,7 +163,7 @@ jobs: STARKNET_PRIVATE_KEY="${{ secrets.STARKNET_PRIVATE_KEY }}" STARKNET_RPC_URL="${{ secrets.STARKNET_RPC_URL }}" - AI_PROVIDER_API_KEY="${{ secrets.OPENAI }}" + AI_PROVIDER_API_KEY="${{ secrets.ANTHROPIC }}" AI_MODEL="claude-3-5-sonnet-latest" AI_PROVIDER="anthropic" @@ -234,7 +234,7 @@ jobs: echo "❌ Basic API test failed!" exit 1 fi - + - name: Create cairo code generation test env file run: | cd ./packages/agents From bf3a5f1ce1529da35dd2bafaaa2caaecbe57ad3e Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 12:20:50 +0200 Subject: [PATCH 30/47] fix: add scarb installation request before tests in snak-test, remove debug steps in ci --- .github/workflows/backend.yml | 29 +----------- .../__tests__/code-quality/snak.test.ts | 47 +++++++++++++++++-- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 5560e000..a0c521d2 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -36,7 +36,6 @@ jobs: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT - name: Setup pnpm cache - if: false uses: actions/cache@v4 with: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} @@ -50,24 +49,9 @@ jobs: - name: Install turbo run: pnpm add turbo@latest -g - - name: Clean previous builds - run: | - rm -rf packages/*/dist - rm -rf packages/*/node_modules/.cache - - name: Build run: turbo build - - name: Check files after build - run: | - find packages/backend/dist -type f | grep -i openai - ls -la packages/backend/dist/config/provider || echo "Directory does not exist" - - - name: Debug module resolution - run: | - cd packages/ingester - NODE_PATH=../.. node -e "try { const path = require.resolve('@cairo-coder/backend/config/provider/openai'); console.log('Module found at:', path); } catch(e) { console.log('Module not found:', e.message); console.log('Available modules:', require('fs').readdirSync('../backend/dist/config').join(', ')); }" - - name: Create config file run: | mkdir -p packages/agents @@ -182,6 +166,7 @@ jobs: CAIRO_GENERATION_API_URL="http://127.0.0.1:3001/chat/completions" EOL + - name: Cache snak node modules uses: actions/cache@v4 with: @@ -222,18 +207,6 @@ jobs: echo "Server failed to start after $max_retries retries" exit 1 fi - - - name: Test basic API connectivity - run: | - echo "Testing basic API connectivity..." - response=$(curl -s -H "Content-Type: application/json" -H "x-api-key: ${{ secrets.SNAK_SERVER_KEY }}" -X POST -d '{"request": "Hello World"}' http://localhost:${{ secrets.SNAK_SERVER_PORT }}/api/key/request) - echo "Response: $response" - if echo "$response" | grep -q "Hello"; then - echo "✅ Basic API test passed!" - else - echo "❌ Basic API test failed!" - exit 1 - fi - name: Create cairo code generation test env file run: | diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts index 98c063d6..e350e992 100644 --- a/packages/agents/__tests__/code-quality/snak.test.ts +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -11,8 +11,49 @@ if (!process.env.API_URL) { const API_KEY = process.env.API_KEY; const API_URL = process.env.API_URL; +// Agent est défini au niveau global pour être utilisé dans beforeAll et dans les tests +const agent = request(API_URL); + +// Le beforeAll est placé au niveau global, en dehors du describe +beforeAll(async () => { + console.log('Setting up test environment - Installing Scarb...'); + + try { + const installResponse = await agent + .post('/api/key/request') + .set('Content-Type', 'application/json') + .set('x-api-key', API_KEY) + .send({ + request: "Can you install scarb?", + }); + + console.log('Scarb Installation Status:', installResponse.status); + console.log('Scarb Installation Response:', + installResponse.body.output ? + JSON.stringify(installResponse.body.output[0], null, 2) : + 'No output' + ); + + const isSuccess = installResponse.status === 201 && + installResponse.body.output && + installResponse.body.output[0].status === 'success'; + + if (!isSuccess) { + console.error('⚠️ Warning: Scarb installation failed. : ', installResponse.body.output[0].text); + } else { + console.log('✅ Scarb installation successful'); + } + + // Attendre que l'installation soit traitée + await new Promise((resolve) => setTimeout(resolve, 5000)); + + } catch (error) { + console.error('❌ Error during Scarb installation:', error); + console.warn('⚠️ Tests may fail if Scarb is not properly installed'); + } +}, 60000); // Timeout de 60 secondes pour l'installation + describe('Code Generation and Compilation Tests', () => { - const agent = request(API_URL); async function generateAndCompile( project_name: string, @@ -132,10 +173,10 @@ describe('Code Generation and Compilation Tests', () => { } describe('Cairo Functions and Basic Algorithms', () => { - + test('Hello World test', async () => { const project_name = 'hello_world'; - const prompt_content = 'a simple Hello World function in Cairo'; + const prompt_content = 'a cairo function that returns "Hello World"'; const result = await generateAndCompile(project_name, prompt_content, 0); if (!result.success) { From c77f7d3c284df15068fd57b9ef68f648ba5d116f Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 12:27:39 +0200 Subject: [PATCH 31/47] trunked --- .github/workflows/backend.yml | 4 +- .../__tests__/code-quality/snak.test.ts | 92 ++++++++++--------- 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index a0c521d2..aa8eed0e 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -191,7 +191,7 @@ jobs: max_retries=10 retry_count=0 server_ready=false - + while [ $retry_count -lt $max_retries ] && [ "$server_ready" = false ]; do if curl -s -H "x-api-key: ${{ secrets.SNAK_SERVER_KEY }}" http://localhost:${{ secrets.SNAK_SERVER_PORT }}/api/key/status > /dev/null; then echo "Server is up and running!" @@ -202,7 +202,7 @@ jobs: retry_count=$((retry_count+1)) fi done - + if [ "$server_ready" = false ]; then echo "Server failed to start after $max_retries retries" exit 1 diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts index e350e992..83f0b3b4 100644 --- a/packages/agents/__tests__/code-quality/snak.test.ts +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -17,36 +17,40 @@ const agent = request(API_URL); // Le beforeAll est placé au niveau global, en dehors du describe beforeAll(async () => { console.log('Setting up test environment - Installing Scarb...'); - + try { const installResponse = await agent .post('/api/key/request') .set('Content-Type', 'application/json') .set('x-api-key', API_KEY) .send({ - request: "Can you install scarb?", + request: 'Can you install scarb?', }); - + console.log('Scarb Installation Status:', installResponse.status); - console.log('Scarb Installation Response:', - installResponse.body.output ? - JSON.stringify(installResponse.body.output[0], null, 2) : - 'No output' + console.log( + 'Scarb Installation Response:', + installResponse.body.output + ? JSON.stringify(installResponse.body.output[0], null, 2) + : 'No output', ); - - const isSuccess = installResponse.status === 201 && - installResponse.body.output && - installResponse.body.output[0].status === 'success'; - + + const isSuccess = + installResponse.status === 201 && + installResponse.body.output && + installResponse.body.output[0].status === 'success'; + if (!isSuccess) { - console.error('⚠️ Warning: Scarb installation failed. : ', installResponse.body.output[0].text); + console.error( + '⚠️ Warning: Scarb installation failed. : ', + installResponse.body.output[0].text, + ); } else { console.log('✅ Scarb installation successful'); } - + // Attendre que l'installation soit traitée await new Promise((resolve) => setTimeout(resolve, 5000)); - } catch (error) { console.error('❌ Error during Scarb installation:', error); console.warn('⚠️ Tests may fail if Scarb is not properly installed'); @@ -54,12 +58,11 @@ beforeAll(async () => { }, 60000); // Timeout de 60 secondes pour l'installation describe('Code Generation and Compilation Tests', () => { - async function generateAndCompile( project_name: string, prompt_content: string, index: number, - ): Promise<{success: boolean; error?: string}> { + ): Promise<{ success: boolean; error?: string }> { console.log(`\n=== Test #${index}: ${project_name} ===`); console.log(`Generating code for: ${prompt_content}`); @@ -90,11 +93,14 @@ describe('Code Generation and Compilation Tests', () => { if (generateResponse.status !== 201) { return { success: false, - error: `Generation HTTP request failed with status ${generateResponse.status}: ${JSON.stringify(generateResponse.body)}` + error: `Generation HTTP request failed with status ${generateResponse.status}: ${JSON.stringify(generateResponse.body)}`, }; } - console.log('CODE GENERATION RESPONSE:', JSON.stringify(generateResponse.body.output[0], null, 2)); + console.log( + 'CODE GENERATION RESPONSE:', + JSON.stringify(generateResponse.body.output[0], null, 2), + ); const sucessfulGeneration = generateResponse.body.output[0].text .toLowerCase() .includes('```cairo'); @@ -105,7 +111,7 @@ describe('Code Generation and Compilation Tests', () => { ) { return { success: false, - error: `Generation failed: ${JSON.stringify(generateResponse.body.output[0].text)}` + error: `Generation failed: ${JSON.stringify(generateResponse.body.output[0].text)}`, }; } @@ -128,34 +134,38 @@ describe('Code Generation and Compilation Tests', () => { .send({ request: compilation_prompt, }); - console.log('COMPILATION STATUS:', compileResponse.status); - + if (compileResponse.status !== 201) { return { success: false, - error: `Compilation HTTP request failed with status ${compileResponse.status}: ${JSON.stringify(compileResponse.body)}` + error: `Compilation HTTP request failed with status ${compileResponse.status}: ${JSON.stringify(compileResponse.body)}`, }; } - console.log('COMPILATION RESPONSE:', JSON.stringify(compileResponse.body.output[0], null, 2)); + console.log( + 'COMPILATION RESPONSE:', + JSON.stringify(compileResponse.body.output[0], null, 2), + ); const sucessfulCompilation = compileResponse.body.output[0].text .toLowerCase() .includes('compilation') && - !compileResponse.body.output[0].text.toLowerCase().includes('failure') && + !compileResponse.body.output[0].text + .toLowerCase() + .includes('failure') && !compileResponse.body.output[0].text.toLowerCase().includes('failed') && !compileResponse.body.output[0].text.toLowerCase().includes('error'); - + if ( compileResponse.body.output[0].status !== 'success' || !sucessfulCompilation ) { return { success: false, - error: `Compilation failed: ${JSON.stringify(compileResponse.body.output[0].text)}` + error: `Compilation failed: ${JSON.stringify(compileResponse.body.output[0].text)}`, }; } @@ -163,38 +173,38 @@ describe('Code Generation and Compilation Tests', () => { await new Promise((resolve) => setTimeout(resolve, 5000)); return { success: true }; - } catch (error) { - console.error(`❌ Unexpected error in Test #${index}:`, error); - return { - success: false, - error: `Unexpected error: ${error.message}` - }; + } catch (error) { + console.error(`❌ Unexpected error in Test #${index}:`, error); + return { + success: false, + error: `Unexpected error: ${error.message}`, + }; + } } -} - -describe('Cairo Functions and Basic Algorithms', () => { + describe('Cairo Functions and Basic Algorithms', () => { test('Hello World test', async () => { const project_name = 'hello_world'; const prompt_content = 'a cairo function that returns "Hello World"'; const result = await generateAndCompile(project_name, prompt_content, 0); - + if (!result.success) { console.error(`❌ TEST FAILED: ${result.error}`); } - + expect(result.success).toBe(true); }, 100000); test('Fibonacci function', async () => { const project_name = 'fibonacci'; - const prompt_content = 'a Cairo function that calculates the Fibonacci sequence'; + const prompt_content = + 'a Cairo function that calculates the Fibonacci sequence'; const result = await generateAndCompile(project_name, prompt_content, 1); - + if (!result.success) { console.error(`❌ TEST FAILED: ${result.error}`); } - + expect(result.success).toBe(true); }, 100000); From 78f7bf537a556db4ed18a9eca1a00aaadf6e5bbe Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 12:31:40 +0200 Subject: [PATCH 32/47] unable all tests of snak-test --- .../__tests__/code-quality/snak.test.ts | 162 +++++++++++------- 1 file changed, 104 insertions(+), 58 deletions(-) diff --git a/packages/agents/__tests__/code-quality/snak.test.ts b/packages/agents/__tests__/code-quality/snak.test.ts index 83f0b3b4..d1440886 100644 --- a/packages/agents/__tests__/code-quality/snak.test.ts +++ b/packages/agents/__tests__/code-quality/snak.test.ts @@ -208,64 +208,110 @@ describe('Code Generation and Compilation Tests', () => { expect(result.success).toBe(true); }, 100000); - // test('Max value in array', async () => { - // const project_name = 'max_value'; - // const prompt_content = "a Cairo function that finds the maximum value in an array"; - // const success = await generateAndCompile(project_name, prompt_content, 2); - // expect(success).toBe(true); - // }, 100000); - - // test('Simple sorting algorithm', async () => { - // const project_name = 'sorting'; - // const prompt_content = "a sorting algorithm"; - // const success = await generateAndCompile(project_name, prompt_content, 3); - // expect(success).toBe(true); - // }, 100000); + test('Max value in array', async () => { + const project_name = 'max_value'; + const prompt_content = + 'a Cairo function that finds the maximum value in an array'; + const result = await generateAndCompile(project_name, prompt_content, 2); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + + test('Simple sorting algorithm', async () => { + const project_name = 'sorting'; + const prompt_content = 'a sorting algorithm'; + const result = await generateAndCompile(project_name, prompt_content, 3); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + }); + + describe('Simple Starknet Contracts', () => { + test('Basic contract with storage', async () => { + const project_name = 'basic_contract'; + const prompt_content = + 'a basic Starknet contract with a storage variable and getter/setter functions'; + const result = await generateAndCompile(project_name, prompt_content, 4); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + + test('Counter contract', async () => { + const project_name = 'counter'; + const prompt_content = + 'a Starknet contract that maintains a counter with increment and decrement functions'; + const result = await generateAndCompile(project_name, prompt_content, 5); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + + test('Simple voting system', async () => { + const project_name = 'voting'; + const prompt_content = + 'a Starknet contract for a simple voting system where users can vote only once'; + const result = await generateAndCompile(project_name, prompt_content, 6); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); }); - // describe('Simple Starknet Contracts', () => { - // test('Basic contract with storage', async () => { - // const project_name = 'basic_contract'; - // const prompt_content = "a basic Starknet contract with a storage variable and getter/setter functions"; - // const success = await generateAndCompile(project_name, prompt_content, 4); - // expect(success).toBe(true); - // }, 100000); - - // test('Counter contract', async () => { - // const project_name = 'counter'; - // const prompt_content = "a Starknet contract that maintains a counter with increment and decrement functions"; - // const success = await generateAndCompile(project_name, prompt_content, 5); - // expect(success).toBe(true); - // }, 100000); - - // test('Simple voting system', async () => { - // const project_name = 'voting'; - // const prompt_content = "a Starknet contract for a simple voting system where users can vote only once"; - // const success = await generateAndCompile(project_name, prompt_content, 6); - // expect(success).toBe(true); - // }, 100000); - // }); - - // describe('Standard and Complex Contracts', () => { - // test('ERC-20 token contract', async () => { - // const project_name = 'erc20'; - // const prompt_content = "a minimal Starknet ERC-20 token contract"; - // const success = await generateAndCompile(project_name, prompt_content, 7); - // expect(success).toBe(true); - // }, 100000); - - // test('ERC-721 NFT contract', async () => { - // const project_name = 'erc721'; - // const prompt_content = "a Starknet ERC-721 NFT contract with minting functionality"; - // const success = await generateAndCompile(project_name, prompt_content, 8); - // expect(success).toBe(true); - // }, 100000); - - // test('Multisig wallet contract', async () => { - // const project_name = 'multisig'; - // const prompt_content = "a Starknet multisig wallet contract that requires multiple approvals for transactions"; - // const success = await generateAndCompile(project_name, prompt_content, 9); - // expect(success).toBe(true); - // }, 100000); - // }); + describe('Standard and Complex Contracts', () => { + test('ERC-20 token contract', async () => { + const project_name = 'erc20'; + const prompt_content = 'a minimal Starknet ERC-20 token contract'; + const result = await generateAndCompile(project_name, prompt_content, 7); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + + test('ERC-721 NFT contract', async () => { + const project_name = 'erc721'; + const prompt_content = + 'a Starknet ERC-721 NFT contract with minting functionality'; + const result = await generateAndCompile(project_name, prompt_content, 8); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + + test('Multisig wallet contract', async () => { + const project_name = 'multisig'; + const prompt_content = + 'a Starknet multisig wallet contract that requires multiple approvals for transactions'; + const result = await generateAndCompile(project_name, prompt_content, 9); + + if (!result.success) { + console.error(`❌ TEST FAILED: ${result.error}`); + } + + expect(result.success).toBe(true); + }, 100000); + }); }); From 43d0f28d4dce3e4f77b0895b0f7cff8522cd66e6 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 12:50:37 +0200 Subject: [PATCH 33/47] fix: remove snak server checker to avoid security issue --- .github/workflows/backend.yml | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index aa8eed0e..7bfc3527 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -186,27 +186,7 @@ jobs: pnpm run build --filter="@snakagent/server..." && lerna run start --scope "@snakagent/server" & echo "Waiting for server to start..." - sleep 60 - - max_retries=10 - retry_count=0 - server_ready=false - - while [ $retry_count -lt $max_retries ] && [ "$server_ready" = false ]; do - if curl -s -H "x-api-key: ${{ secrets.SNAK_SERVER_KEY }}" http://localhost:${{ secrets.SNAK_SERVER_PORT }}/api/key/status > /dev/null; then - echo "Server is up and running!" - server_ready=true - else - echo "Server not ready yet, retrying in 5 seconds..." - sleep 5 - retry_count=$((retry_count+1)) - fi - done - - if [ "$server_ready" = false ]; then - echo "Server failed to start after $max_retries retries" - exit 1 - fi + sleep 120 - name: Create cairo code generation test env file run: | From a805a1007555efe9f7c72337159a02cdffc4d959 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 19:17:37 +0200 Subject: [PATCH 34/47] fix: remove unnecessary logs from script test --- scripts/snak-test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/snak-test.sh b/scripts/snak-test.sh index 6fe52f54..f08303bc 100755 --- a/scripts/snak-test.sh +++ b/scripts/snak-test.sh @@ -6,8 +6,6 @@ TEST_RESULT=${PIPESTATUS[0]} if [ $TEST_RESULT -ne 0 ]; then echo "❌ API test failed with exit code $TEST_RESULT!" - echo "=============== Test Logs ===============" - grep -A 5 -E "TEST FAILED|❌" test-output.log || true exit 1 else echo "✅ API test passed!" From d6cfe01662f7f36c5e5b3a0bf09d6dc0d27d483f Mon Sep 17 00:00:00 2001 From: alvinouille Date: Tue, 6 May 2025 19:29:25 +0200 Subject: [PATCH 35/47] fix: name model in ingester ci --- .github/workflows/ingester.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ingester.yml b/.github/workflows/ingester.yml index 8790cab5..1c534139 100644 --- a/.github/workflows/ingester.yml +++ b/.github/workflows/ingester.yml @@ -85,10 +85,10 @@ jobs: SIMILARITY_MEASURE = "cosine" [HOSTED_MODE] - DEFAULT_CHAT_PROVIDER = "anthropic" - DEFAULT_CHAT_MODEL = "Claude 3.5 Sonnet" - DEFAULT_FAST_CHAT_PROVIDER = "anthropic" - DEFAULT_FAST_CHAT_MODEL = "Claude 3.5 Sonnet" + DEFAULT_CHAT_PROVIDER = "gemini" + DEFAULT_CHAT_MODEL = "Gemini Flash 2.5" + DEFAULT_FAST_CHAT_PROVIDER = "gemini" + DEFAULT_FAST_CHAT_MODEL = "Gemini Flash 2.5" DEFAULT_EMBEDDING_PROVIDER = "openai" DEFAULT_EMBEDDING_MODEL = "Text embedding 3 large" From e4a5a77dd92d2bd1251cdcc3d3211e27ea6356fd Mon Sep 17 00:00:00 2001 From: alvinouille Date: Wed, 7 May 2025 09:39:07 +0200 Subject: [PATCH 36/47] remove unusued comment --- .github/workflows/backend.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 7bfc3527..ad8e9527 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -202,6 +202,3 @@ jobs: run: | chmod +x ./scripts/snak-test.sh bash scripts/snak-test.sh - - # - name: Push docker image - # run: docker push ${{ github.repository }}:${{ github.sha }} From a72adcc7115996bf9136331d015b2f8ef2add0f1 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Wed, 7 May 2025 09:45:26 +0200 Subject: [PATCH 37/47] fix: add backend env file --- .github/workflows/backend.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ad8e9527..0802be96 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -95,6 +95,10 @@ jobs: POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" EOL + - name: Create backend env file + run: | + touch packages/backend/.env + - name: Run unit tests run: pnpm run test:unit From f7a1f151dc0a09fa90084d925fbaec42d5fbd1b3 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 8 May 2025 16:33:28 +0200 Subject: [PATCH 38/47] feat: add image build workflow --- .github/workflows/test-build.yml | 103 +++++++++++++++++++++++++++++++ .gitignore | 2 +- 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-build.yml diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml new file mode 100644 index 00000000..da2bfc2d --- /dev/null +++ b/.github/workflows/test-build.yml @@ -0,0 +1,103 @@ +name: Create and publish a Docker image + +on: + workflow-dispatch: + pull-request: + branches: ['main'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=sha,format=short + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create config file + run: | + mkdir -p packages/agents + cat > packages/agents/config.toml << 'EOL' + [API_KEYS] + OPENAI = "${{ secrets.OPENAI }}" + ANTHROPIC = "${{ secrets.ANTHROPIC }}" + GEMINI = "${{ secrets.GEMINI }}" + + [VECTOR_DB] + POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" + POSTGRES_HOST = "postgres" + POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" + POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" + POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" + + [GENERAL] + PORT = 3001 + SIMILARITY_MEASURE = "cosine" + + [HOSTED_MODE] + DEFAULT_CHAT_PROVIDER = "gemini" + DEFAULT_CHAT_MODEL = "Gemini Flash 2.5" + DEFAULT_FAST_CHAT_PROVIDER = "gemini" + DEFAULT_FAST_CHAT_MODEL = "Gemini Flash 2.5" + DEFAULT_EMBEDDING_PROVIDER = "openai" + DEFAULT_EMBEDDING_MODEL = "Text embedding 3 large" + + [VERSIONS] + STARKNET_FOUNDRY = "0.37.0" + SCARB = "2.9.2" + EOL + + - name: Create env file + run: | + cat > .env << 'EOL' + POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" + POSTGRES_HOST = "localhost" + POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" + POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" + POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" + EOL + + - name: Create backend env file + run: | + touch packages/backend/.env + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v6 + with: + context: . + push: true + file: ./backend.dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 2ba7606f..586a656b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ yarn-error.log .env.test.local .env.production.local -packages/ui/.env +packages/agents/.env.test # Config files config.toml From 9e4305c2b562b27742f1174fadc9ecdeb7ce705a Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 8 May 2025 16:36:31 +0200 Subject: [PATCH 39/47] fix: add push event test-build workflow --- .github/workflows/test-build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index da2bfc2d..20f33ae6 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -2,8 +2,10 @@ name: Create and publish a Docker image on: workflow-dispatch: - pull-request: - branches: ['main'] + push: + branches: [main] + pull_request: + branches: [main] env: REGISTRY: ghcr.io From bee9c9f853a23e9d73ebcb71d83c69483af807ad Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 8 May 2025 16:38:19 +0200 Subject: [PATCH 40/47] fix: workflow_dispatch fix --- .github/workflows/test-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 20f33ae6..4e0a1ee9 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -1,7 +1,7 @@ name: Create and publish a Docker image on: - workflow-dispatch: + workflow_dispatch: push: branches: [main] pull_request: From 853b09dea7192a1f8174de6367d667b33ba1d21a Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 8 May 2025 19:20:50 +0200 Subject: [PATCH 41/47] fix: remove config file from image push --- .github/workflows/test-build.yml | 49 -------------------------------- 1 file changed, 49 deletions(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4e0a1ee9..652320f7 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -2,8 +2,6 @@ name: Create and publish a Docker image on: workflow_dispatch: - push: - branches: [main] pull_request: branches: [main] @@ -45,53 +43,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Create config file - run: | - mkdir -p packages/agents - cat > packages/agents/config.toml << 'EOL' - [API_KEYS] - OPENAI = "${{ secrets.OPENAI }}" - ANTHROPIC = "${{ secrets.ANTHROPIC }}" - GEMINI = "${{ secrets.GEMINI }}" - - [VECTOR_DB] - POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" - POSTGRES_HOST = "postgres" - POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" - POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" - POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" - - [GENERAL] - PORT = 3001 - SIMILARITY_MEASURE = "cosine" - - [HOSTED_MODE] - DEFAULT_CHAT_PROVIDER = "gemini" - DEFAULT_CHAT_MODEL = "Gemini Flash 2.5" - DEFAULT_FAST_CHAT_PROVIDER = "gemini" - DEFAULT_FAST_CHAT_MODEL = "Gemini Flash 2.5" - DEFAULT_EMBEDDING_PROVIDER = "openai" - DEFAULT_EMBEDDING_MODEL = "Text embedding 3 large" - - [VERSIONS] - STARKNET_FOUNDRY = "0.37.0" - SCARB = "2.9.2" - EOL - - - name: Create env file - run: | - cat > .env << 'EOL' - POSTGRES_USER = "${{ secrets.POSTGRES_USER }}" - POSTGRES_HOST = "localhost" - POSTGRES_ROOT_DB = "${{ secrets.POSTGRES_ROOT_DB }}" - POSTGRES_PASSWORD = "${{ secrets.POSTGRES_PASSWORD }}" - POSTGRES_PORT = "${{ secrets.POSTGRES_PORT }}" - EOL - - - name: Create backend env file - run: | - touch packages/backend/.env - - name: Build and push Docker image id: push uses: docker/build-push-action@v6 From 37030903517c12d54f9dabe0a1738ba911fc5d1f Mon Sep 17 00:00:00 2001 From: alvinouille Date: Thu, 8 May 2025 21:33:33 +0200 Subject: [PATCH 42/47] feat: add ingester image build in test-build and backend image test: OK --- .github/workflows/test-build.yml | 36 +++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 652320f7..1fd0b49b 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -23,11 +23,21 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Extract metadata (tags, labels) for Docker - id: meta + - name: Extract metadata for backend + id: meta-backend uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ github.repository }}/backend + tags: | + type=ref,event=branch + type=sha,format=short + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + + - name: Extract metadata for ingester + id: meta-ingester + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository }}/ingester tags: | type=ref,event=branch type=sha,format=short @@ -43,14 +53,26 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push Docker image - id: push + - name: Build and push Docker image for backend + id: push-backend uses: docker/build-push-action@v6 with: context: . push: true file: ./backend.dockerfile - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta-backend.outputs.tags }} + labels: ${{ steps.meta-backend.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push Docker image for ingester + id: push-ingester + uses: docker/build-push-action@v6 + with: + context: . + push: true + file: ./ingester.dockerfile + tags: ${{ steps.meta-ingester.outputs.tags }} + labels: ${{ steps.meta-ingester.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max From a8edb0ef2184951b4ae3e90257e814fb16df326d Mon Sep 17 00:00:00 2001 From: alvinouille Date: Fri, 9 May 2025 17:55:18 +0200 Subject: [PATCH 43/47] feat: fix ingester dockerfile name, add task-build-image generic workflow to build push image, add docker compose for production --- .github/workflows/task-build-image.yml | 64 +++++++++++++++++++ .github/workflows/test-build.yml | 2 +- docker-compose.yml => docker-compose.dev.yml | 16 +++-- docker-compose.prod.yml | 65 ++++++++++++++++++++ ingest.dockerfile => ingester.dockerfile | 0 5 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/task-build-image.yml rename docker-compose.yml => docker-compose.dev.yml (71%) create mode 100644 docker-compose.prod.yml rename ingest.dockerfile => ingester.dockerfile (100%) diff --git a/.github/workflows/task-build-image.yml b/.github/workflows/task-build-image.yml new file mode 100644 index 00000000..82f3f039 --- /dev/null +++ b/.github/workflows/task-build-image.yml @@ -0,0 +1,64 @@ +name: Create and publish a Docker image + +on: + workflow_dispatch: + workflow_call: + inputs: + registry: + description: Container registry domain + required: true + type: string + image-name: + description: Name for the Docker image + required: true + type: string + image-file: + description: Dockerfile used to build the image + required: true + type: string + +permissions: + contents: read + packages: write + attestations: write + id-token: write + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Extract metadata + id: meta-backend + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image-name }}/${{ inputs.image-file }} + tags: | + type=ref,event=branch + type=sha,format=short + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + id: push-image + uses: docker/build-push-action@v6 + with: + context: . + push: true + file: ./${{ inputs.image-file }}.dockerfile + tags: ${{ steps.meta-backend.outputs.tags }} + labels: ${{ steps.meta-backend.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 1fd0b49b..da3aec9c 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -71,7 +71,7 @@ jobs: with: context: . push: true - file: ./ingester.dockerfile + file: ./ingest.dockerfile tags: ${{ steps.meta-ingester.outputs.tags }} labels: ${{ steps.meta-ingester.outputs.labels }} cache-from: type=gha diff --git a/docker-compose.yml b/docker-compose.dev.yml similarity index 71% rename from docker-compose.yml rename to docker-compose.dev.yml index 7833df4c..2b1e9141 100644 --- a/docker-compose.yml +++ b/docker-compose.dev.yml @@ -5,13 +5,19 @@ services: shm_size: 1g env_file: - .env - ports: - - 5432:5432 + expose: + - 5432 volumes: - - ./data:/var/lib/postgresql/data + - postgres_data:/var/lib/postgresql/data restart: unless-stopped networks: - cairo_coder + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U yo -d cairo_coder_db'] + interval: 5s + timeout: 5s + retries: 5 + start_period: 10s backend: container_name: 'backend' @@ -26,7 +32,7 @@ services: - packages/backend/.env depends_on: postgres: - condition: service_started + condition: service_healthy restart: unless-stopped networks: - cairo_coder @@ -38,7 +44,7 @@ services: profiles: ['ingester'] depends_on: postgres: - condition: service_started + condition: service_healthy networks: - cairo_coder diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..bc93db00 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,65 @@ +services: + cairo-coder-postgres: + image: pgvector/pgvector:pg17 + container_name: cairo-coder-postgres + shm_size: 1g + env_file: + - .env + volumes: + - postgres_data:/var/lib/postgresql/data + restart: unless-stopped + networks: + - cairo_coder + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U yo -d cairo_coder_db'] + interval: 5s + timeout: 5s + retries: 5 + start_period: 10s + + cairo-coder-backend: + image: gchr.io/repo/cairo-coder-backend:123456 + container_name: cairo-coder-backend + env_file: + - packages/backend/.env + expose: + - 3001 + depends_on: + - postgres: + condition: service_healthy + restart: unless-stopped + networks: + - cairo_coder + - services + deploy: + resources: + limits: + memory: 4G + labels: + - 'traefik.enable=true' + - 'traefik.docker.network=services' + - 'traefik.http.routers.proxy.rule=Host(cairo-coder.kasar.io)' + - 'traefik.http.routers.proxy.entrypoints=websecure' + - 'traefik.http.routers.proxy.tls.certresolver=letsencrypt' + - 'traefik.http.services.proxy.loadbalancer.server.port=3001' + + cairo-coder-ingester: + image: gchr.io/repo/cairo-coder-ingester:123456 + container_name: cairo-coder-ingester + depends_on: + - postgres: + condition: service_healthy + - backend: + condition: service_started + restart: 'no' + networks: + - cairo_coder + +networks: + cairo_coder: + internal: true + services: + external: true + +volumes: + postgres_data: diff --git a/ingest.dockerfile b/ingester.dockerfile similarity index 100% rename from ingest.dockerfile rename to ingester.dockerfile From a3dbbc41f29ca6b85c8a52f8c3608df053214930 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Fri, 9 May 2025 17:57:06 +0200 Subject: [PATCH 44/47] fix: name ingester dockerfile in test-build --- .github/workflows/test-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index da3aec9c..1fd0b49b 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -71,7 +71,7 @@ jobs: with: context: . push: true - file: ./ingest.dockerfile + file: ./ingester.dockerfile tags: ${{ steps.meta-ingester.outputs.tags }} labels: ${{ steps.meta-ingester.outputs.labels }} cache-from: type=gha From e92ec3e2841e2acf9ab8d38119f0ee4f18ff4f4d Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 12 May 2025 12:29:05 +0200 Subject: [PATCH 45/47] debug: add debug log for assess gemini stream response delay and fix compose prod --- docker-compose.dev.yml | 16 ++++----- docker-compose.prod.yml | 35 ++++++++++--------- .../src/core/pipeline/answerGenerator.ts | 3 ++ .../agents/src/core/pipeline/ragPipeline.ts | 10 +++++- 4 files changed, 38 insertions(+), 26 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2b1e9141..13713ba0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,7 +1,7 @@ services: - postgres: + cairo-coder-postgres: image: pgvector/pgvector:pg17 - container_name: 'postgresql' + container_name: 'cairo-coder-postgres' shm_size: 1g env_file: - .env @@ -19,8 +19,8 @@ services: retries: 5 start_period: 10s - backend: - container_name: 'backend' + cairo-coder-backend: + container_name: 'cairo-coder-backend' build: context: . dockerfile: backend.dockerfile @@ -31,19 +31,19 @@ services: env_file: - packages/backend/.env depends_on: - postgres: + cairo-coder-postgres: condition: service_healthy restart: unless-stopped networks: - cairo_coder - ingester: + cairo-coder-ingester: build: context: . - dockerfile: ingest.dockerfile + dockerfile: ingester.dockerfile profiles: ['ingester'] depends_on: - postgres: + cairo-coder-postgres: condition: service_healthy networks: - cairo_coder diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index bc93db00..71677f9e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -4,7 +4,7 @@ services: container_name: cairo-coder-postgres shm_size: 1g env_file: - - .env + - ./config/.env volumes: - postgres_data:/var/lib/postgresql/data restart: unless-stopped @@ -18,15 +18,13 @@ services: start_period: 10s cairo-coder-backend: - image: gchr.io/repo/cairo-coder-backend:123456 + image: ghcr.io/kasarlabs/cairo-coder/backend:sha-7e38821 container_name: cairo-coder-backend - env_file: - - packages/backend/.env - expose: - - 3001 depends_on: - - postgres: - condition: service_healthy + cairo-coder-postgres: + condition: service_healthy + volumes: + - ./config/config.toml:/app/packages/agents/config.toml restart: unless-stopped networks: - cairo_coder @@ -38,19 +36,22 @@ services: labels: - 'traefik.enable=true' - 'traefik.docker.network=services' - - 'traefik.http.routers.proxy.rule=Host(cairo-coder.kasar.io)' - - 'traefik.http.routers.proxy.entrypoints=websecure' - - 'traefik.http.routers.proxy.tls.certresolver=letsencrypt' - - 'traefik.http.services.proxy.loadbalancer.server.port=3001' + - 'traefik.http.routers.cairo-coder-backend.rule=Host(`cairo-coder.kasar.io`)' + - 'traefik.http.routers.cairo-coder-backend.entrypoints=websecure' + - 'traefik.http.routers.cairo-coder-backend.tls.certresolver=letsencrypt' + - 'traefik.http.services.cairo-coder-backend.loadbalancer.server.port=3001' cairo-coder-ingester: - image: gchr.io/repo/cairo-coder-ingester:123456 + image: ghcr.io/kasarlabs/cairo-coder/ingester:sha-7e38821 container_name: cairo-coder-ingester + profiles: ['ingester'] + volumes: + - ./config/config.toml:/app/packages/agents/config.toml depends_on: - - postgres: - condition: service_healthy - - backend: - condition: service_started + cairo-coder-postgres: + condition: service_healthy + cairo-coder-backend: + condition: service_started restart: 'no' networks: - cairo_coder diff --git a/packages/agents/src/core/pipeline/answerGenerator.ts b/packages/agents/src/core/pipeline/answerGenerator.ts index 586f6188..6effd036 100644 --- a/packages/agents/src/core/pipeline/answerGenerator.ts +++ b/packages/agents/src/core/pipeline/answerGenerator.ts @@ -28,8 +28,11 @@ export class AnswerGenerator { logger.debug('Final Prompt:' + prompt); // Use stream instead of invoke, and pipe through StringOutputParser + logger.debug('Before streaming response'); + const startTime = Date.now(); const stream = await this.llm.stream(prompt); logger.debug('Started streaming response'); + logger.debug(`Time to stream: ${Date.now() - startTime}ms`); return stream; } diff --git a/packages/agents/src/core/pipeline/ragPipeline.ts b/packages/agents/src/core/pipeline/ragPipeline.ts index 6ef6316d..4b4172a2 100644 --- a/packages/agents/src/core/pipeline/ragPipeline.ts +++ b/packages/agents/src/core/pipeline/ragPipeline.ts @@ -1,5 +1,10 @@ import { Embeddings } from '@langchain/core/embeddings'; -import { RagInput, StreamHandler, RagSearchConfig, LLMConfig } from '../../types'; +import { + RagInput, + StreamHandler, + RagSearchConfig, + LLMConfig, +} from '../../types'; import { QueryProcessor } from './queryProcessor'; import { DocumentRetriever } from './documentRetriever'; import { AnswerGenerator } from './answerGenerator'; @@ -61,10 +66,13 @@ export class RagPipeline { // Step 3: Generate the answer as a stream const stream = await this.answerGenerator.generate(input, retrieved); + const startTime = Date.now(); for await (const chunk of stream) { handler.emitResponse(chunk); + logger.debug(`Time to get chunk: ${Date.now() - startTime}ms`); } logger.debug('Stream ended'); + logger.debug(`Total time: ${Date.now() - startTime}ms`); handler.emitEnd(); } catch (error) { logger.error('Pipeline error:', error); From 64fd8c9b0688994ee05608e52d171d69a680db81 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 12 May 2025 17:26:34 +0200 Subject: [PATCH 46/47] fix: update ingester dockerfile to add certificat installation --- docker-compose.dev.yml | 2 -- ingester.dockerfile | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 13713ba0..d699750d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -26,8 +26,6 @@ services: dockerfile: backend.dockerfile ports: - 3001:3001 - extra_hosts: - - host.docker.internal:host-gateway env_file: - packages/backend/.env depends_on: diff --git a/ingester.dockerfile b/ingester.dockerfile index d90b2dee..6e47c202 100644 --- a/ingester.dockerfile +++ b/ingester.dockerfile @@ -21,6 +21,8 @@ RUN pnpm install --frozen-lockfile RUN npm install -g turbo # Install Antora +RUN apt update && apt install -y ca-certificates openssl + RUN npm install -g @antora/cli @antora/site-generator # Install mdbook From 8dfcd58aeb0310ef0ce815cf08e0028ebdaa6de8 Mon Sep 17 00:00:00 2001 From: alvinouille Date: Mon, 12 May 2025 19:03:05 +0200 Subject: [PATCH 47/47] fix: compose update on the server --- docker-compose.prod.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 71677f9e..4cce8797 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,7 @@ services: start_period: 10s cairo-coder-backend: - image: ghcr.io/kasarlabs/cairo-coder/backend:sha-7e38821 + image: ghcr.io/kasarlabs/cairo-coder/backend:sha-9822262 container_name: cairo-coder-backend depends_on: cairo-coder-postgres: @@ -42,7 +42,7 @@ services: - 'traefik.http.services.cairo-coder-backend.loadbalancer.server.port=3001' cairo-coder-ingester: - image: ghcr.io/kasarlabs/cairo-coder/ingester:sha-7e38821 + image: ghcr.io/kasarlabs/cairo-coder/ingester:sha-9822262 container_name: cairo-coder-ingester profiles: ['ingester'] volumes: @@ -58,7 +58,7 @@ services: networks: cairo_coder: - internal: true + driver: bridge services: external: true