Mega Refactor: Reorganize into single package.json, fix all types, im…

…prove imports, improve organization (#120)

* remove package json for dev server api

* fix server not proxying to api properly

* checkpoint root package json can run frontend

* remove frontend package json

* remove package locks and example project package

* simplify example project package.json

* fix build issue with cli packages

* improve open test

* no need to run server tests explicitly

* fix import issue

* fix server imports

* fix minor bug with package exmaples not updating on change

* checkpoint, refactor to fix type performance and new dir structure

* checkpoint, fixing type errors, rename api directory

* more refactoring to simplify dir structure

* more api fixes for refactor

* more fixes to adapt index

* refactor to cli directory

* fix many type issues

* wip fixing type issues, ~20 remaining

* fix type errors

* add typecheck github action

* fix workflows to not use bootstrap

* fix typecheck to use bun

* typecheck workflow fix

* test fix

* fix formatbot to format

* formatbot: Automatically format code

* couple test to db instance

* fix test parallelization issue

---------

Co-authored-by: tscircuitbot <[email protected]>

.github/workflows/formatbot.yml

@@ -29,7 +29,7 @@ jobs:
       run: npm install @biomejs/biome@${{ steps.get-biome-version.outputs.BIOME_VERSION }}
 
     - name: Run formatter
-      run: npx @biomejs/biome format .
+      run: npx @biomejs/biome format . --write
 
     - name: Commit changes
       uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/release.yml

@@ -24,7 +24,12 @@ jobs:
           bun-version: latest
 
       - name: Install project dependencies
-        run: bun run bootstrap:ci
+        run: |
+          bun install
+
+          # Undo changes to the lockfile that might block release
+          # when this project is more mature use "bun install --frozen-lockfile"
+          git checkout -- bun.lockb
 
       - name: Build project
         run: bun run build

.github/workflows/test.yml

@@ -9,7 +9,7 @@ on:
       - main
 
 jobs:
-  build:
+  tests:
     runs-on: ubuntu-latest
 
     steps:
@@ -22,7 +22,7 @@ jobs:
           bun-version: latest
 
       - name: Install dependencies
-        run: bun run bootstrap
+        run: bun install
 
       - name: Install dependencies
         run: bun run build

.github/workflows/typecheck.yml

@@ -0,0 +1,29 @@
+# Created using @tscircuit/plop (npm install -g @tscircuit/plop)
+name: Type Check
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  type-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup bun
+        uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun i
+
+      - name: Build (required for dist imports)
+        run: bun run build
+
+      - name: Run format check
+        run: bunx tsc --noEmit

.gitignore

@@ -9,3 +9,5 @@ dist
 
 .vscode
 .aider*
+package-lock.json
+**/package-lock.json

DEVELOPMENT.md

@@ -1,4 +1,7 @@
 # Getting Started with Development for @tscircuit/cli
 
 1. Run `bun install`
-2. Run `bun bootstrap`
+2. Run `bun run dev`
+
+You now have a development server running on http://localhost:5173 (don't use
+the one on port 3020, it won't auto-reload frontend changes)

README.md

@@ -1,4 +1,4 @@
-# `tsci` - The TSCircuit Command Line Tool
+# `tsci` - The TSCircuit Command Line & Development Tools
 
 Command line tool for developing tscircuit projects and interacting with the
 tscircuit registry.
@@ -41,7 +41,7 @@ tsci publish
 This project is developed with [bun](https://bun.sh/), make sure you have
 that installed.
 
-Run `bun bootstrap` to install dependencies and `bun cli.ts` to run test the cli in development.
+Run `bun install` to install dependencies and `bun cli/cli.ts` to run test the cli in development, or `bun run dev` to run the dev server.
 
 To run tests, run `bun test`
 

dev-server-api/src/db/generic-json-level.ts → api/db/generic-json-level.ts

@@ -108,18 +108,15 @@ class GenericJsonLevel extends AbstractLevel<string, any> {
           return undefined
         }
 
-        return [entry.key, entry.value]
+        return [entry.key, entry.value] as [string, any]
       },
       async seek(target: string) {
         index = files.findIndex((file) => file.startsWith(target))
         if (index === -1) {
           index = files.length
         }
       },
-      async end() {
-        // No resources to clean up
-      },
-    }
+    } as any
   }
 }
 

dev-server-api/src/middlewares/with-db.ts → api/lib/middlewares/with-db.ts

@@ -1,6 +1,6 @@
 import type { Middleware } from "winterspec"
-import { getDb } from "../db/get-db"
-import type { ZodLevelDatabase } from "src/db/zod-level-db"
+import { getDb } from "api/db/get-db"
+import type { ZodLevelDatabase } from "api/db/zod-level-db"
 
 export const withDb: Middleware<
   {},

dev-server-api/routes/api/db/download.ts → api/routes/api/db/download.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/api/dev_package_examples/create.ts → api/routes/api/dev_package_examples/create.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/api/dev_package_examples/get.ts → api/routes/api/dev_package_examples/get.ts

@@ -1,5 +1,5 @@
-import { withWinterSpec } from "src/with-winter-spec"
-import { NotFoundError } from "edgespec/middleware"
+import { withWinterSpec } from "api/lib/with-winter-spec"
+import { NotFoundError } from "winterspec/middleware"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/api/dev_package_examples/list.ts → api/routes/api/dev_package_examples/list.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/api/dev_package_examples/update.ts → api/routes/api/dev_package_examples/update.ts

@@ -1,7 +1,7 @@
-import { withWinterSpec } from "src/with-winter-spec"
-import { NotFoundError } from "edgespec/middleware"
+import { withWinterSpec } from "api/lib/with-winter-spec"
+import { NotFoundError } from "winterspec/middleware"
 import { z } from "zod"
-import { DevPackageExampleSchema } from "src/db/schema"
+import { DevPackageExampleSchema } from "api/db/schema"
 
 export default withWinterSpec({
   methods: ["POST"],

dev-server-api/routes/api/dev_server/reset.ts → api/routes/api/dev_server/reset.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 
 export default withWinterSpec({
   methods: ["GET", "POST"],

dev-server-api/routes/api/export_files/create.ts → api/routes/api/export_files/create.ts

@@ -1,7 +1,5 @@
-import { ExportFileSchema } from "src/db/schema"
-import { publicMapExportFile } from "src/lib/public-mapping/public-map-export-file"
-import { export_file } from "src/lib/zod/export_file"
-import { withWinterSpec } from "src/with-winter-spec"
+import { ExportFileSchema } from "api/db/schema"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({
@@ -16,15 +14,13 @@ export default withWinterSpec({
   }),
   auth: "none",
 })(async (req, ctx) => {
-  console.log("putting file")
   const export_file = await ctx.db.put("export_file", {
     export_request_id: req.jsonBody.export_request_id,
     file_name: req.jsonBody.file_name,
     file_content_base64: req.jsonBody.file_content_base64,
     created_at: new Date().toISOString(),
   })
   console.log("done putting file")
-
   return ctx.json({
     export_file,
   })

dev-server-api/routes/api/export_files/download.ts → api/routes/api/export_files/download.ts

@@ -1,6 +1,4 @@
-import { publicMapExportFile } from "src/lib/public-mapping/public-map-export-file"
-import { export_file } from "src/lib/zod/export_file"
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { NotFoundError } from "winterspec/middleware"
 import { z } from "zod"
 

dev-server-api/routes/api/export_requests/create.ts → api/routes/api/export_requests/create.ts

@@ -1,8 +1,7 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
-import { publicMapExportRequest } from "src/lib/public-mapping/public-map-export-request"
-import { export_parameters } from "../../../src/lib/zod/export_parameters"
-import { ExportRequestSchema } from "src/db/schema"
+import { export_parameters } from "api/lib/zod/export_parameters"
+import { ExportRequestSchema } from "api/db/schema"
 
 export default withWinterSpec({
   methods: ["POST"],

dev-server-api/routes/api/export_requests/get.ts → api/routes/api/export_requests/get.ts

@@ -1,10 +1,7 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
-import { export_request } from "src/lib/zod/export_request"
-import { publicMapExportRequest } from "src/lib/public-mapping/public-map-export-request"
-import { NotFoundError } from "edgespec/middleware"
-import { ExportRequestSchema } from "src/db/schema"
-import { file } from "bun"
+import { NotFoundError } from "winterspec/middleware"
+import { ExportRequestSchema } from "api/db/schema"
 
 export default withWinterSpec({
   methods: ["GET", "POST"],

dev-server-api/routes/api/export_requests/list.ts → api/routes/api/export_requests/list.ts

@@ -1,7 +1,6 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
-import { publicMapExportRequest } from "src/lib/public-mapping/public-map-export-request"
-import { ExportRequestSchema } from "src/db/schema"
+import { ExportRequestSchema } from "api/db/schema"
 
 export default withWinterSpec({
   methods: ["GET", "POST"],

dev-server-api/routes/api/export_requests/update.ts → api/routes/api/export_requests/update.ts

@@ -1,9 +1,7 @@
-import { withWinterSpec } from "src/with-winter-spec"
-import { NotFoundError } from "edgespec/middleware"
+import { withWinterSpec } from "api/lib/with-winter-spec"
+import { NotFoundError } from "winterspec/middleware"
 import { z } from "zod"
-import { export_request } from "src/lib/zod/export_request"
-import { publicMapExportRequest } from "src/lib/public-mapping/public-map-export-request"
-import { ExportRequestSchema } from "src/db/schema"
+import { ExportRequestSchema } from "api/db/schema"
 
 export default withWinterSpec({
   methods: ["POST"],

dev-server-api/routes/health.ts → api/routes/api/health.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "../src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/api/package_info/create.ts → api/routes/api/package_info/create.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/api/package_info/get.ts → api/routes/api/package_info/get.ts

@@ -1,5 +1,6 @@
-import { PackageInfoSchema } from "src/db/schema"
-import { withWinterSpec } from "src/with-winter-spec"
+import { PackageInfoSchema } from "api/db/schema"
+
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/api/health.ts → api/routes/health.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "../../src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/routes/index.ts → api/routes/index.ts

@@ -1,4 +1,4 @@
-import { withWinterSpec } from "../src/with-winter-spec"
+import { withWinterSpec } from "api/lib/with-winter-spec"
 import { z } from "zod"
 
 export default withWinterSpec({

dev-server-api/server.ts → api/server.ts

@@ -6,7 +6,7 @@ const serverFetch = await createFetchHandlerFromDir(
   join(import.meta.dir, "./routes"),
 )
 
-console.log("starting dev-server-api on http://127.0.0.1:3021")
+console.log("starting dev server api on http://127.0.0.1:3021")
 Bun.serve({
   fetch: (bunReq) => {
     const req = new EdgeRuntimeRequest(bunReq.url, {

api/static-routes.ts

@@ -0,0 +1,24 @@
+
+// import { WinterSpecRouteMap } from "@winterspec/types"
+
+const routeMap = {
+  "/api/db/download": (await import('./routes/api/db/download.ts')).default,
+  "/api/dev_package_examples/create": (await import('./routes/api/dev_package_examples/create.ts')).default,
+  "/api/dev_package_examples/get": (await import('./routes/api/dev_package_examples/get.ts')).default,
+  "/api/dev_package_examples/list": (await import('./routes/api/dev_package_examples/list.ts')).default,
+  "/api/dev_package_examples/update": (await import('./routes/api/dev_package_examples/update.ts')).default,
+  "/api/dev_server/reset": (await import('./routes/api/dev_server/reset.ts')).default,
+  "/api/export_files/create": (await import('./routes/api/export_files/create.ts')).default,
+  "/api/export_files/download": (await import('./routes/api/export_files/download.ts')).default,
+  "/api/export_requests/create": (await import('./routes/api/export_requests/create.ts')).default,
+  "/api/export_requests/get": (await import('./routes/api/export_requests/get.ts')).default,
+  "/api/export_requests/list": (await import('./routes/api/export_requests/list.ts')).default,
+  "/api/export_requests/update": (await import('./routes/api/export_requests/update.ts')).default,
+  "/api/health": (await import('./routes/api/health.ts')).default,
+  "/api/package_info/create": (await import('./routes/api/package_info/create.ts')).default,
+  "/api/package_info/get": (await import('./routes/api/package_info/get.ts')).default,
+  "/health": (await import('./routes/health.ts')).default,
+  "/": (await import('./routes/index.ts')).default
+}
+
+export default routeMap

dev-server-api/tests/fixtures/get-test-server.ts → api/tests/fixtures/get-test-server.ts

@@ -12,7 +12,8 @@ interface TestFixture {
 export const getTestFixture = async (): Promise<TestFixture> => {
   process.env.TSCI_DEV_SERVER_DB = tmpdir() + `/${Math.random()}` + "/devdb"
   const port = 3001 + Math.floor(Math.random() * 999)
-  const server = startServer({ port })
+  const testInstanceId = Math.random().toString(36).substring(2, 15)
+  const server = await startServer({ port })
   const url = `http://127.0.0.1:${port}`
   const axios = defaultAxios.create({
     baseURL: url,