wip

Author: heyjorgedev

app/Console/Commands/IngestDocumentation.php

@@ -72,9 +72,6 @@ private function ingestFile(SplFileInfo $fileInfo)
         $usedMemory = (memory_get_usage() - $memory) / 1024;
 
         $upserts = collect($documents)
-            ->filter(function (Document $document) {
-                return ! $this->contentIsJustHeadings($document);
-            })
             ->map(fn (Document $document) => new DataUpsert(
                 id: Str::uuid(),
                 data: $document->getContent(),
@@ -91,11 +88,4 @@ private function ingestFile(SplFileInfo $fileInfo)
 
         $this->info("File $fileName has $documentCount documents, used $usedMemory kb.");
     }
-
-    private function contentIsJustHeadings(Document $document): bool
-    {
-
-        return collect(explode("\n", $document->getContent()))
-            ->every(fn ($line) => Str::startsWith($line, '#') || Str::startsWith($line, ''));
-    }
 }

app/Console/Commands/SearchVector.php

@@ -74,7 +74,7 @@ public function processQuestion(string $question)
             ])
             ->toArray();
 
-        $this->js(sprintf('$wire.generateAnswer("%s")', $question));
+        // $this->js(sprintf('$wire.generateAnswer("%s")', $question));
     }
 
     public function generateAnswer(string $question)

app/Livewire/ChatPage.php

@@ -57,4 +57,11 @@ public function getLinks(): array
     {
         return $this->links;
     }
+
+    public function hasOnlyTitles(): bool
+    {
+        $lines = explode("\n", trim($this->content));
+
+        return count($lines) === count(array_filter($lines, fn ($line) => preg_match('/^#{1,6}/', $line)));
+    }
 }

app/Splitters/Document.php

@@ -53,7 +53,11 @@ public function split(string $contents): array
 
         $documents[] = $currentDocument;
 
-        return array_values(array_filter($documents));
+        return collect($documents)
+            ->filter()
+            ->filter(fn (Document $document) => ! $document->hasOnlyTitles())
+            ->values()
+            ->toArray();
     }
 
     private function getTitleIndex(string $line): int

app/Splitters/LaravelMarkdownSplitter.php

@@ -0,0 +1,13 @@
+<?php
+
+use App\Splitters\Document;
+
+test('document can identify if it has only titles', function () {
+    $document = new Document;
+    $document->pushContent('# Title');
+    $document->pushContent('## Title');
+    expect($document->hasOnlyTitles())->toBeTrue();
+
+    $document->pushContent('Content');
+    expect($document->hasOnlyTitles())->toBeFalse();
+});