Merge branch 'dev' into swiftyos/optimse-views

# Conflicts:
#	autogpt_platform/backend/backend/TEST_DATA_README.md
#	autogpt_platform/backend/backend/check_db.py
#	autogpt_platform/backend/backend/check_store_data.py
#	autogpt_platform/backend/backend/test_data_updater.py
#	autogpt_platform/backend/backend/util/test_data_creator.py

Author: Swiftyos

.github/workflows/platform-backend-ci.yml

@@ -50,6 +50,23 @@ jobs:
         env:
           RABBITMQ_DEFAULT_USER: ${{ env.RABBITMQ_DEFAULT_USER }}
           RABBITMQ_DEFAULT_PASS: ${{ env.RABBITMQ_DEFAULT_PASS }}
+      clamav:
+        image: clamav/clamav-debian:latest
+        ports:
+          - 3310:3310
+        env:
+          CLAMAV_NO_FRESHCLAMD: false
+          CLAMD_CONF_StreamMaxLength: 50M
+          CLAMD_CONF_MaxFileSize: 100M
+          CLAMD_CONF_MaxScanSize: 100M
+          CLAMD_CONF_MaxThreads: 4
+          CLAMD_CONF_ReadTimeout: 300
+        options: >-
+          --health-cmd "clamdscan --version || exit 1"
+          --health-interval 30s
+          --health-timeout 10s
+          --health-retries 5
+          --health-start-period 180s
 
     steps:
       - name: Checkout repository
@@ -131,6 +148,35 @@ jobs:
         # outputs:
         # DB_URL, API_URL, GRAPHQL_URL, ANON_KEY, SERVICE_ROLE_KEY, JWT_SECRET
 
+      - name: Wait for ClamAV to be ready
+        run: |
+          echo "Waiting for ClamAV daemon to start..."
+          max_attempts=60
+          attempt=0
+          
+          until nc -z localhost 3310 || [ $attempt -eq $max_attempts ]; do
+            echo "ClamAV is unavailable - sleeping (attempt $((attempt+1))/$max_attempts)"
+            sleep 5
+            attempt=$((attempt+1))
+          done
+          
+          if [ $attempt -eq $max_attempts ]; then
+            echo "ClamAV failed to start after $((max_attempts*5)) seconds"
+            echo "Checking ClamAV service logs..."
+            docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
+            exit 1
+          fi
+          
+          echo "ClamAV is ready!"
+          
+          # Verify ClamAV is responsive
+          echo "Testing ClamAV connection..."
+          timeout 10 bash -c 'echo "PING" | nc localhost 3310' || {
+            echo "ClamAV is not responding to PING"
+            docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
+            exit 1
+          }
+
       - name: Run Database Migrations
         run: poetry run prisma migrate dev --name updates
         env:
@@ -144,9 +190,9 @@ jobs:
       - name: Run pytest with coverage
         run: |
           if [[ "${{ runner.debug }}" == "1" ]]; then
-            poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG test
+            poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG
           else
-            poetry run pytest -s -vv test
+            poetry run pytest -s -vv
           fi
         if: success() || (failure() && steps.lint.outcome == 'failure')
         env:
@@ -159,6 +205,7 @@ jobs:
           REDIS_HOST: "localhost"
           REDIS_PORT: "6379"
           REDIS_PASSWORD: "testpassword"
+          ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!
 
     env:
       CI: true

.github/workflows/platform-frontend-ci.yml

@@ -18,11 +18,45 @@ defaults:
     working-directory: autogpt_platform/frontend
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      cache-key: ${{ steps.cache-key.outputs.key }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "21"
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Generate cache key
+        id: cache-key
+        run: echo "key=${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}" >> $GITHUB_OUTPUT
+
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.pnpm-store
+          key: ${{ steps.cache-key.outputs.key }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
   lint:
     runs-on: ubuntu-latest
+    needs: setup
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
       - name: Set up Node.js
         uses: actions/setup-node@v4
@@ -32,6 +66,14 @@ jobs:
       - name: Enable corepack
         run: corepack enable
 
+      - name: Restore dependencies cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.pnpm-store
+          key: ${{ needs.setup.outputs.cache-key }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
@@ -40,9 +82,11 @@ jobs:
 
   type-check:
     runs-on: ubuntu-latest
+    needs: setup
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
       - name: Set up Node.js
         uses: actions/setup-node@v4
@@ -52,14 +96,62 @@ jobs:
       - name: Enable corepack
         run: corepack enable
 
+      - name: Restore dependencies cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.pnpm-store
+          key: ${{ needs.setup.outputs.cache-key }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
       - name: Run tsc check
         run: pnpm type-check
 
+  chromatic:
+    runs-on: ubuntu-latest
+    needs: setup
+    # Only run on dev branch pushes or PRs targeting dev
+    if: github.ref == 'refs/heads/dev' || github.base_ref == 'dev'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "21"
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Restore dependencies cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.pnpm-store
+          key: ${{ needs.setup.outputs.cache-key }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run Chromatic
+        uses: chromaui/action@latest
+        with:
+          projectToken: chpt_9e7c1a76478c9c8
+          onlyChanged: true
+          workingDir: autogpt_platform/frontend
+          token: ${{ secrets.GITHUB_TOKEN }}
+
   test:
     runs-on: ubuntu-latest
+    needs: setup
     strategy:
       fail-fast: false
       matrix:
@@ -97,6 +189,14 @@ jobs:
         run: |
           docker compose -f ../docker-compose.yml up -d
 
+      - name: Restore dependencies cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.pnpm-store
+          key: ${{ needs.setup.outputs.cache-key }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
@@ -112,6 +212,8 @@ jobs:
 
       - name: Run Playwright tests
         run: pnpm test:no-build --project=${{ matrix.browser }}
+        env:
+          BROWSER_TYPE: ${{ matrix.browser }}
 
       - name: Print Final Docker Compose logs
         if: always()

.gitignore

@@ -165,7 +165,7 @@ package-lock.json
 
 # Allow for locally private items
 # private
-pri* 
+pri*
 # ignore
 ig*
 .github_access_token

autogpt_platform/CLAUDE.md

@@ -19,7 +19,7 @@ cd backend && poetry install
 # Run database migrations
 poetry run prisma migrate dev
 
-# Start all services (database, redis, rabbitmq)
+# Start all services (database, redis, rabbitmq, clamav)
 docker compose up -d
 
 # Run the backend server
@@ -32,6 +32,7 @@ poetry run test
 poetry run pytest path/to/test_file.py::test_function_name
 
 # Lint and format
+# prefer format if you want to just "fix" it and only get the errors that can't be autofixed
 poetry run format  # Black + isort
 poetry run lint    # ruff
 ```
@@ -77,6 +78,7 @@ npm run type-check
 - **Queue System**: RabbitMQ for async task processing
 - **Execution Engine**: Separate executor service processes agent workflows
 - **Authentication**: JWT-based with Supabase integration
+- **Security**: Cache protection middleware prevents sensitive data caching in browsers/proxies
 
 ### Frontend Architecture
 - **Framework**: Next.js App Router with React Server Components
@@ -90,6 +92,7 @@ npm run type-check
 2. **Blocks**: Reusable components in `/backend/blocks/` that perform specific tasks
 3. **Integrations**: OAuth and API connections stored per user
 4. **Store**: Marketplace for sharing agent templates
+5. **Virus Scanning**: ClamAV integration for file upload security
 
 ### Testing Approach
 - Backend uses pytest with snapshot testing for API responses
@@ -118,6 +121,7 @@ Key models (defined in `/backend/schema.prisma`):
 3. Define input/output schemas
 4. Implement `run` method
 5. Register in block registry
+6. Generate the block uuid using `uuid.uuid4()`
 
 **Modifying the API:**
 1. Update route in `/backend/backend/server/routers/`
@@ -129,4 +133,15 @@ Key models (defined in `/backend/schema.prisma`):
 1. Components go in `/frontend/src/components/`
 2. Use existing UI components from `/frontend/src/components/ui/`
 3. Add Storybook stories for new components
-4. Test with Playwright if user-facing
+4. Test with Playwright if user-facing
+
+### Security Implementation
+
+**Cache Protection Middleware:**
+- Located in `/backend/backend/server/middleware/security.py`
+- Default behavior: Disables caching for ALL endpoints with `Cache-Control: no-store, no-cache, must-revalidate, private`
+- Uses an allow list approach - only explicitly permitted paths can be cached
+- Cacheable paths include: static assets (`/static/*`, `/_next/static/*`), health checks, public store pages, documentation
+- Prevents sensitive data (auth tokens, API keys, user data) from being cached by browsers/proxies
+- To allow caching for a new endpoint, add it to `CACHEABLE_PATHS` in the middleware
+- Applied to both main API server and external API applications

autogpt_platform/README.md

@@ -62,6 +62,12 @@ To run the AutoGPT Platform, follow these steps:
    pnpm i
    ```
 
+   Generate the API client (this step is required before running the frontend):
+
+   ```
+   pnpm generate:api-client
+   ```
+
    Then start the frontend application in development mode:
 
    ```
@@ -164,3 +170,27 @@ To persist data for PostgreSQL and Redis, you can modify the `docker-compose.yml
 3. Save the file and run `docker compose up -d` to apply the changes.
 
 This configuration will create named volumes for PostgreSQL and Redis, ensuring that your data persists across container restarts.
+
+### API Client Generation
+
+The platform includes scripts for generating and managing the API client:
+
+- `pnpm fetch:openapi`: Fetches the OpenAPI specification from the backend service (requires backend to be running on port 8006)
+- `pnpm generate:api-client`: Generates the TypeScript API client from the OpenAPI specification using Orval
+- `pnpm generate:api-all`: Runs both fetch and generate commands in sequence
+
+#### Manual API Client Updates
+
+If you need to update the API client after making changes to the backend API:
+
+1. Ensure the backend services are running:
+   ```
+   docker compose up -d
+   ```
+
+2. Generate the updated API client:
+   ```
+   pnpm generate:api-all
+   ```
+
+This will fetch the latest OpenAPI specification and regenerate the TypeScript client code.