dev: reorg codebase structure (#85)

Author: enitrat

.gitignore

@@ -47,7 +47,6 @@ packages/**/dist
 !.trunk/trunk.yaml
 !.trunk/.gitignore
 
-starklings/
 debug/
 
 fixtures/runner_crate/target

.trunk/trunk.yaml

@@ -16,6 +16,10 @@ runtimes:
     - [email protected]
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
+  ignore:
+    - linters: [ALL]
+      paths:
+        - python/src/cairo_coder_tools/ingestion/generated
   enabled:
     - [email protected]
     - [email protected]

AGENTS.md

@@ -6,23 +6,24 @@ This file documents conventions and checklists for making changes that affect th
 
 When adding a new documentation source (e.g., a new docs site or SDK) make sure to complete all of the following steps:
 
-1. TypeScript ingestion (packages/ingester)
+1. TypeScript ingestion (ingesters)
 
-   - Create an ingester class extending `BaseIngester` or `MarkdownIngester` under `packages/ingester/src/ingesters/`.
-   - Register it in `packages/ingester/src/IngesterFactory.ts`.
+   - Create an ingester class extending `BaseIngester` or `MarkdownIngester` under `ingesters/src/ingesters/`.
+   - Register it in `ingesters/src/IngesterFactory.ts`.
    - Ensure chunks carry correct metadata: `uniqueId`, `contentHash`, `sourceLink`, and `source`.
-   - Run `pnpm generate-embeddings` (or `generate-embeddings:yes`) to populate/update the vector store.
+   - Run the embeddings generator to populate/update the vector store:
+     - Prefer: `bun run src/generateEmbeddings.ts` (or `bun run src/generateEmbeddings.ts -y`)
+     - If you have scripts wired: `bun run generate-embeddings` (or `generate-embeddings:yes`)
 
-2. Agents (TS)
+2. Agents (Python)
 
-   - Add the new enum value to `packages/agents/src/types/index.ts` under `DocumentSource`.
-   - Verify Postgres vector store accepts the new `source` and filters on it (`packages/agents/src/db/postgresVectorStore.ts`).
+   - Add the new enum value to `python/src/cairo_coder/core/types.py` under `DocumentSource`.
+   - Ensure filtering by `metadata->>'source'` works with the new value in `python/src/cairo_coder/dspy/document_retriever.py`.
+   - Update the query processor resource descriptions in `python/src/cairo_coder/dspy/query_processor.py` (`RESOURCE_DESCRIPTIONS`). The module validates that every `DocumentSource` has a description.
 
 3. Retrieval Pipeline (Python)
 
-   - Add the new enum value to `python/src/cairo_coder/core/types.py` under `DocumentSource`.
-   - Ensure filtering by `metadata->>'source'` works with the new value in `python/src/cairo_coder/dspy/document_retriever.py`.
-   - Update the query processor resource descriptions in `python/src/cairo_coder/dspy/query_processor.py` (`RESOURCE_DESCRIPTIONS`).
+   - No extra steps beyond the above; the retriever already supports filtering by `metadata->>'source'`.
 
 4. Optimized Program Files (Python) — required
 
@@ -34,7 +35,7 @@ When adding a new documentation source (e.g., a new docs site or SDK) make sure
 
    - Ensure the new source appears where appropriate (e.g., `/v1/agents` output and documentation tables):
      - `API_DOCUMENTATION.md`
-     - `packages/ingester/README.md`
+     - `ingesters/README.md`
      - Any user-facing lists of supported sources
 
 6. Quick Sanity Check

API_DOCUMENTATION.md

@@ -58,14 +58,25 @@ Lists every agent registered in Cairo Coder.
       "openzeppelin_docs",
       "corelib_docs",
       "scarb_docs",
-      "starknet_js"
+      "starknet_js",
+      "starknet_blog"
     ]
   },
   {
-    "id": "scarb-assistant",
-    "name": "Scarb Assistant",
-    "description": "Specialized assistant for Scarb build tool",
-    "sources": ["scarb_docs"]
+    "id": "starknet-agent",
+    "name": "Starknet Agent",
+    "description": "Assistant for the Starknet ecosystem (contracts, tools, docs).",
+    "sources": [
+      "cairo_book",
+      "starknet_docs",
+      "starknet_foundry",
+      "cairo_by_example",
+      "openzeppelin_docs",
+      "corelib_docs",
+      "scarb_docs",
+      "starknet_js",
+      "starknet_blog"
+    ]
   }
 ]
 ```
@@ -82,6 +93,7 @@ Lists every agent registered in Cairo Coder.
 | `corelib_docs`      | Cairo core library docs                    |
 | `scarb_docs`        | Scarb package manager documentation        |
 | `starknet_js`       | StarknetJS guides and SDK documentation    |
+| `starknet_blog`     | Starknet blog posts and announcements      |
 
 ## Chat Completions
 

CLAUDE.md

@@ -118,8 +118,8 @@ ingesters/
 
 ```text
 Python Summarizer → Generated Markdown → Ingester → PostgreSQL → RAG Pipeline → Code Generation
-     (python/)            (python/src/scripts/         (ingesters/)     (pgvector)    (python/)
-                          summarizer/generated/)
+     (python/)            (python/src/cairo_coder_tools/ (ingesters/)     (pgvector)    (python/)
+                          ingestion/generated/)
 ```
 
 ## Configuration

ingester.dockerfile

@@ -16,7 +16,7 @@ COPY ingesters ./ingesters
 
 
 # Copy ingester files generated from python
-COPY python/src/scripts/summarizer/generated ./python/src/scripts/summarizer/generated
+COPY python/src/cairo_coder_tools/ingestion/generated ./python/src/cairo_coder_tools/ingestion/generated
 
 # Install dependencies
 WORKDIR /app/ingesters

ingesters/README.md

@@ -32,6 +32,7 @@ The ingester currently supports the following documentation sources:
 6. **Core Library Docs** (`corelib_docs`): Cairo core library documentation
 7. **Scarb Docs** (`scarb_docs`): Scarb package manager documentation
 8. **StarknetJS Guides** (`starknet_js`): StarknetJS guides and tutorials
+9. **Starknet Blog** (`starknet_blog`): Starknet blog posts and announcements
 
 ## Architecture
 
@@ -112,17 +113,16 @@ const chunks = splitter.splitMarkdownToChunks(markdown);
 
 ## Usage
 
-To use the ingester package, run the `generateEmbeddings.ts` script:
+To use the ingester package, run the embeddings generator script:
 
 ```bash
-# From the root of the package
-pnpm run generate-embeddings
-
-# From the root of the project
-turbo run generate-embeddings
+# Preferred (direct bun):
+bun run src/generateEmbeddings.ts
+# Non-interactive (yes to prompts):
+bun run src/generateEmbeddings.ts -y
 ```
 
-This will prompt you to select a documentation source to ingest. You can also select "Everything" to ingest all sources.
+This will prompt you to select a documentation source to ingest (or use `-y` for all). You can also select "Everything" to ingest all sources.
 
 ## Adding a New Documentation Source
 

ingesters/src/ingesters/AsciiDocIngester.ts

@@ -237,7 +237,7 @@ export abstract class AsciiDocIngester extends BaseIngester {
       sections.forEach((section: ParsedSection, index: number) => {
         const hash: string = calculateHash(section.content);
         const sourceLink = `${this.config.baseUrl}/${page.name}${this.config.urlSuffix}${section.anchor ? '#' + section.anchor : ''}`;
-        console.debug(
+        logger.debug(
           `Section Title: ${section.title}, source: ${this.source}, sourceLink: ${sourceLink}`,
         );
         chunks.push(

ingesters/src/ingesters/CairoBookIngester.ts

@@ -46,8 +46,8 @@ export class CairoBookIngester extends MarkdownIngester {
   async readSummaryFile(): Promise<string> {
     const summaryPath = getPythonPath(
       'src',
-      'scripts',
-      'summarizer',
+      'cairo_coder_tools',
+      'ingestion',
       'generated',
       'cairo_book_summary.md',
     );

ingesters/src/ingesters/CoreLibDocsIngester.ts

@@ -46,8 +46,8 @@ export class CoreLibDocsIngester extends MarkdownIngester {
   async readCorelibSummaryFile(): Promise<string> {
     const summaryPath = getPythonPath(
       'src',
-      'scripts',
-      'summarizer',
+      'cairo_coder_tools',
+      'ingestion',
       'generated',
       'corelib_summary.md',
     );