Merge branch 'master' into pipertts-support

Mintplex-Labs · Aug 7, 2024 · 31df75d · 31df75d
2 parents 1739342 + 6666fff
commit 31df75d
Show file tree

Hide file tree

Showing 8 changed files with 518 additions and 23 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -78,8 +78,8 @@
   },
   "updateContentCommand": "cd server && yarn && cd ../collector && PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public yarn && cd ../frontend && yarn && cd .. && yarn setup:envs && yarn prisma:setup && echo \"Please run yarn dev:server, yarn dev:collector, and yarn dev:frontend in separate terminal tabs.\"",
   // Use 'postCreateCommand' to run commands after the container is created.
-  // This configures VITE for github codespaces
-  "postCreateCommand": "if [ \"${CODESPACES}\" = \"true\" ]; then echo 'VITE_API_BASE=\"https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api\"' > ./frontend/.env; fi",
+  // This configures VITE for github codespaces and installs gh cli
+  "postCreateCommand": "if [ \"${CODESPACES}\" = \"true\" ]; then echo 'VITE_API_BASE=\"https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api\"' > ./frontend/.env && (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) && sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg && echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main\" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null && sudo apt update && sudo apt install gh -y; fi",
   "portsAttributes": {
     "3001": {
       "label": "Backend",
@@ -208,4 +208,4 @@
   }
   // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
   // "remoteUser": "root"
-}
+}
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -46,7 +46,7 @@
           }
         }
       },
-      "command": "cd ${workspaceFolder}/server/ && yarn dev",
+      "command": "if [ \"${CODESPACES}\" = \"true\" ]; then while ! gh codespace ports -c $CODESPACE_NAME | grep 3001; do sleep 1; done; gh codespace ports visibility 3001:public -c $CODESPACE_NAME; fi & cd ${workspaceFolder}/server/ && yarn dev",
       "runOptions": {
         "instanceLimit": 1,
         "reevaluateOnRerun": true

diff --git a/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js
@@ -223,10 +223,6 @@ class GitLabRepoLoader {
         const objects = Array.isArray(data)
           ? data.filter((item) => item.type === "blob")
           : []; // only get files, not paths or submodules
-        if (objects.length === 0) {
-          fetching = false;
-          break;
-        }
 
         // Apply ignore path rules to found objects. If any rules match it is an invalid file path.
         console.log(

diff --git a/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js b/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js
@@ -47,10 +47,12 @@ class YoutubeTranscript {
       let transcript = "";
       const chunks = transcriptXML.getElementsByTagName("text");
       for (const chunk of chunks) {
-        transcript += chunk.textContent;
+        // Add space after each text chunk
+        transcript += chunk.textContent + " ";
       }
 
-      return transcript;
+      // Trim extra whitespace
+      return transcript.trim().replace(/\s+/g, " ");
     } catch (e) {
       throw new YoutubeTranscriptError(e);
     }