Added Experiments (and small fixes) (#65)

Author: wesfloyd

.cursorrules

@@ -3,15 +3,20 @@
 ## Project Overview
 This is a Next.js prediction market application built by a **solo founder** that integrates with Polymarket and Colosseum APIs. The app uses Prisma with PostgreSQL for data management.
 
-**Infrastructure:** The entire application runs on Vercel infrastructure, including hosting, serverless functions, and database connections.
+**Infrastructure:** The entire application runs on Vercel infrastructure, including hosting, serverless functions, and database connections. Please see RUNBOOK.md for more information.
 
-**Solo Founder Context:** This project is built and maintained by a single developer. Prioritize:
+
+
+## High level guidance to Claude on coding and design
+**Solo Founder Context**: This project is built and maintained by a single developer. Prioritize:
 - **Simplicity over complexity** - Choose straightforward solutions that are easy to maintain
-- **Proven technologies** - Use well-established libraries and patterns  
+- **Less code is better** - Choose solutions that involve less custom code where possible.
+- **Proven technologies** - Use well-established libraries and patterns
 - **Minimal dependencies** - Avoid over-engineering for a 1-person team
 - **Clear documentation** - Write code that's easy to understand and modify later
 - **Gradual scaling** - Start simple, optimize later when needed
 
+
 ## Code Style & Patterns
 - Use TypeScript with strict typing
 - Follow Next.js 14+ App Router patterns
@@ -48,7 +53,7 @@ curl -H "Authorization: Bearer $CRON_SECRET" http://localhost:3000/api/cron/upda
 - Add new queries to the appropriate query object
 - Use transactions for multi-step database operations
 - Follow the existing schema patterns (snake_case for DB columns, camelCase for JS)
-- Try to use the migration commands in package.json such as "db:migrate:deploy"
+- Try to use the migration commands in package.json such as "db:migrate:deploy:dev"
 - **Prefer simple queries** - Complex joins can be hard to debug for a solo developer
 ### Database Operations
 - Use commands in package.json where possible for database migration in Dev and Preview builds.

CLAUDE.md

@@ -8,8 +8,11 @@ BetterAI is a Next.js 15 prediction market application built by a **solo founder
 
 **Core Value Proposition**: Everyone should be able to access world-class AI models with enriched data through a single click to enhance their prediction market decisions.
 
+
+## High level guidance to Claude on coding and design
 **Solo Founder Context**: This project is built and maintained by a single developer. Prioritize:
 - **Simplicity over complexity** - Choose straightforward solutions that are easy to maintain
+- **Less code is better** - Choose solutions that involve less custom code where possible.
 - **Proven technologies** - Use well-established libraries and patterns
 - **Minimal dependencies** - Avoid over-engineering for a 1-person team
 - **Clear documentation** - Write code that's easy to understand and modify later
@@ -102,8 +105,8 @@ Follow the project's `.cursorrules` for consistent development:
 
 ### Database Migrations
 - Use `pnpm run db:migrate:dev` for development migrations
-- Use `pnpm run db:migrate:deploy` for production deployments (requires `.env.local`)
-- Use `pnpm run db:migrate:status` to check migration status (requires `.env.local`)
+- Use `pnpm run db:migrate:deploy:dev` for development deployments (requires `.env.local`)
+- Use `pnpm run db:migrate:status:dev` to check migration status (requires `.env.local`)
 - **Migration naming**: Provide `--name descriptive_name` to avoid interactive prompts
 - **Example**: `pnpm run db:migrate:dev --name add_user_table`
 - Always test migrations in development before deploying to production

app/api/experiments/route.ts

@@ -0,0 +1,99 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { prisma } from '@/lib/db/prisma'
+
+export async function GET(request: NextRequest) {
+  try {
+    // Get experiment summary with accuracy metrics
+    const experiments = await prisma.prediction.findMany({
+      where: {
+        experimentTag: {
+          not: null
+        }
+      },
+      select: {
+        experimentTag: true,
+        experimentNotes: true,
+        modelName: true,
+        createdAt: true,
+        predictionChecks: {
+          select: {
+            absDelta: true,
+            marketClosed: true,
+            createdAt: true
+          }
+        }
+      },
+      orderBy: {
+        createdAt: 'desc'
+      }
+    })
+
+    // Group by experiment tag and calculate stats
+    const experimentStats = experiments.reduce((acc: any, pred: any) => {
+      const tag = pred.experimentTag!
+      if (!acc[tag]) {
+        acc[tag] = {
+          experimentTag: tag,
+          experimentNotes: pred.experimentNotes,
+          modelName: pred.modelName,
+          totalPredictions: 0,
+          checkedPredictions: 0,
+          avgAccuracy: null,
+          firstCreated: pred.createdAt,
+          lastCreated: pred.createdAt,
+        }
+      }
+
+      acc[tag].totalPredictions += 1
+      
+      // Update date range
+      if (pred.createdAt < acc[tag].firstCreated) {
+        acc[tag].firstCreated = pred.createdAt
+      }
+      if (pred.createdAt > acc[tag].lastCreated) {
+        acc[tag].lastCreated = pred.createdAt
+      }
+
+      // Calculate accuracy from prediction checks
+      if (pred.predictionChecks && pred.predictionChecks.length > 0) {
+        const validChecks = pred.predictionChecks.filter((check: any) => 
+          check.absDelta !== null && check.marketClosed === true
+        )
+        
+        if (validChecks.length > 0) {
+          acc[tag].checkedPredictions += 1
+          const avgDelta = validChecks.reduce((sum: number, check: any) => 
+            sum + parseFloat(check.absDelta.toString()), 0
+          ) / validChecks.length
+          
+          // Accuracy = 100 - (avgDelta * 100) to convert delta to percentage accuracy
+          const accuracy = Math.max(0, 100 - (avgDelta * 100))
+          
+          if (acc[tag].avgAccuracy === null) {
+            acc[tag].avgAccuracy = accuracy
+          } else {
+            // Running average
+            acc[tag].avgAccuracy = ((acc[tag].avgAccuracy * (acc[tag].checkedPredictions - 1)) + accuracy) / acc[tag].checkedPredictions
+          }
+        }
+      }
+
+      return acc
+    }, {})
+
+    const results = Object.values(experimentStats).sort((a: any, b: any) => 
+      new Date(b.lastCreated).getTime() - new Date(a.lastCreated).getTime()
+    )
+
+    return NextResponse.json({
+      success: true,
+      experiments: results
+    })
+  } catch (error) {
+    console.error('Error fetching experiments:', error)
+    return NextResponse.json(
+      { success: false, error: 'Failed to fetch experiments' },
+      { status: 500 }
+    )
+  }
+}

app/experiments/page.tsx

@@ -0,0 +1,151 @@
+'use client'
+
+import { useEffect, useState } from 'react'
+import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
+import { Badge } from '@/components/ui/badge'
+
+interface ExperimentStat {
+  experimentTag: string
+  experimentNotes: string | null
+  modelName: string | null
+  totalPredictions: number
+  checkedPredictions: number
+  avgAccuracy: number | null
+  firstCreated: string
+  lastCreated: string
+}
+
+interface ApiResponse {
+  success: boolean
+  experiments: ExperimentStat[]
+}
+
+export default function ExperimentsPage() {
+  const [experiments, setExperiments] = useState<ExperimentStat[]>([])
+  const [loading, setLoading] = useState(true)
+  const [error, setError] = useState<string | null>(null)
+
+  useEffect(() => {
+    fetchExperiments()
+  }, [])
+
+  const fetchExperiments = async () => {
+    try {
+      const response = await fetch('/api/experiments')
+      const data: ApiResponse = await response.json()
+      
+      if (data.success) {
+        setExperiments(data.experiments)
+      } else {
+        setError('Failed to fetch experiments')
+      }
+    } catch (err) {
+      setError('Network error fetching experiments')
+    } finally {
+      setLoading(false)
+    }
+  }
+
+  const formatDate = (dateStr: string) => {
+    return new Date(dateStr).toLocaleDateString('en-US', {
+      month: 'short',
+      day: 'numeric',
+      hour: '2-digit',
+      minute: '2-digit'
+    })
+  }
+
+  const formatAccuracy = (accuracy: number | null) => {
+    if (accuracy === null) return 'N/A'
+    return `${accuracy.toFixed(1)}%`
+  }
+
+  if (loading) {
+    return (
+      <div className="container mx-auto p-6">
+        <h1 className="text-3xl font-bold mb-8">🧪 Prediction Experiments</h1>
+        <div>Loading experiments...</div>
+      </div>
+    )
+  }
+
+  if (error) {
+    return (
+      <div className="container mx-auto p-6">
+        <h1 className="text-3xl font-bold mb-8">🧪 Prediction Experiments</h1>
+        <div className="text-red-600">Error: {error}</div>
+      </div>
+    )
+  }
+
+  return (
+    <div className="container mx-auto p-6">
+      <h1 className="text-3xl font-bold mb-8">🧪 Prediction Experiments</h1>
+      
+      {experiments.length === 0 ? (
+        <Card>
+          <CardContent className="p-8 text-center text-gray-500">
+            No experiments found. Run the batch prediction script with --experiment-tag to create your first experiment.
+          </CardContent>
+        </Card>
+      ) : (
+        <div className="grid gap-6">
+          {experiments.map((exp, index) => (
+            <Card key={index} className="w-full">
+              <CardHeader>
+                <div className="flex items-center justify-between">
+                  <CardTitle className="text-xl font-semibold">
+                    {exp.experimentTag}
+                  </CardTitle>
+                  <div className="flex gap-2">
+                    {exp.modelName && (
+                      <Badge variant="outline">{exp.modelName}</Badge>
+                    )}
+                    <Badge variant="secondary">
+                      {exp.totalPredictions} predictions
+                    </Badge>
+                  </div>
+                </div>
+              </CardHeader>
+              <CardContent>
+                <div className="grid md:grid-cols-2 lg:grid-cols-4 gap-4">
+                  <div>
+                    <div className="text-sm text-gray-500">Accuracy</div>
+                    <div className="text-2xl font-bold">
+                      {formatAccuracy(exp.avgAccuracy)}
+                    </div>
+                    <div className="text-sm text-gray-500">
+                      ({exp.checkedPredictions} checked)
+                    </div>
+                  </div>
+                  
+                  <div>
+                    <div className="text-sm text-gray-500">Total Predictions</div>
+                    <div className="text-2xl font-bold">{exp.totalPredictions}</div>
+                  </div>
+                  
+                  <div>
+                    <div className="text-sm text-gray-500">Started</div>
+                    <div className="text-lg">{formatDate(exp.firstCreated)}</div>
+                  </div>
+                  
+                  <div>
+                    <div className="text-sm text-gray-500">Last Activity</div>
+                    <div className="text-lg">{formatDate(exp.lastCreated)}</div>
+                  </div>
+                </div>
+                
+                {exp.experimentNotes && (
+                  <div className="mt-4 p-3 bg-gray-50 rounded-lg">
+                    <div className="text-sm text-gray-500">Notes</div>
+                    <div className="text-sm">{exp.experimentNotes}</div>
+                  </div>
+                )}
+              </CardContent>
+            </Card>
+          ))}
+        </div>
+      )}
+    </div>
+  )
+}

docs/DATABASE_MIGRATIONS.md

@@ -184,7 +184,7 @@ pnpm prisma migrate status
 
 ```bash
 # Apply to local DB
-pnpm run db:migrate:deploy
+pnpm run db:migrate:deploy:dev
 
 # Generate Prisma client
 pnpm run db:prisma:generate

docs/MIGRATION_ROLLBACK.md

@@ -90,7 +90,7 @@ The GitHub Actions workflow includes basic rollback detection:
 1. **Assess the situation**:
    ```bash
    # Check current migration status
-   pnpm run db:migrate:status
+   pnpm run db:migrate:status:dev
 
    # Check application health
    curl -f $PRODUCTION_URL/api/health
@@ -113,7 +113,7 @@ The GitHub Actions workflow includes basic rollback detection:
 4. **Verify rollback**:
    ```bash
    # Check schema state
-   pnpm run db:migrate:status
+   pnpm run db:migrate:status:dev
 
    # Test application functionality
    # Run critical user flows
@@ -350,7 +350,7 @@ pnpm run db:migrate:dev --name "rollback_${MIGRATION_NAME}"
 # scripts/check-db-state.sh
 
 echo "Checking database state..."
-pnpm run db:migrate:status
+pnpm run db:migrate:status:dev
 echo "Checking application health..."
 curl -f $NEXT_PUBLIC_APP_URL/api/health
 echo "Checking recent migrations..."