diff --git a/.env.template b/.env.template index cc430cd..9ca7bdf 100644 --- a/.env.template +++ b/.env.template @@ -1,6 +1,7 @@ REPO_NAME=MyRepo REPO_URL=https://github.com/user/MyRepo.git REPO_PATH=./examples +INGEST_SECRET=9640f515-fd2c-4403-845f-71b25fc9086c OPENAI_API_KEY=XXX MODEL_TYPE_INFERENCE=gpt-4o-mini MODEL_TYPE_EMBEDDING=text-embedding-3-small diff --git a/README.md b/README.md index 0ccebf8..62d73b0 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ make dev - [x] API data chat - [x] GitHub Actions release - [ ] Secure ingestion endpoint (/api/ingest) +- [ ] Secure RepoChat with optional password ## Why not use Vercel ? diff --git a/action/index.js b/action/index.js index 5317d05..46dc6cb 100644 --- a/action/index.js +++ b/action/index.js @@ -6,18 +6,24 @@ import { Container, createClient } from '@scaleway/sdk'; const providers = ['scaleway']; +function uuidv4() { + return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, c => + (+c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> +c / 4).toString(16) + ); +} + function isValidFile(filePath) { return fs.statSync(filePath).isFile() && !path.basename(filePath).startsWith('.'); } -async function sendFileToApi(filePath, apiUrl) { +async function sendFileToApi(filePath, apiUrl, ingestSecret) { const content = fs.readFileSync(filePath, { encoding: 'base64' }); const metadata = { 'source': path.basename(filePath) }; const payload = { 'content': content, 'metadata': metadata }; try { console.log(`Sending: ${filePath}`); - const response = await axios.post(apiUrl, payload, { headers: { 'Content-Type': 'application/json' } }); + const response = await axios.post(apiUrl, payload, { headers: { 'Content-Type': 'application/json', 'X-Ingest-Secret': ingestSecret } }); return response; } catch (error) { console.error(`Error sending file: ${error.message}`); @@ -34,7 +40,7 @@ function isExcluded(filePath, excludeFiles) { async function processFile(filePath, apiUrl) { console.log(`Sending file: ${filePath}`); - const response = await sendFileToApi(filePath, apiUrl); + const response = await sendFileToApi(filePath, apiUrl, ingestSecret); if (response.status === 200) { console.log(`Successfully ingested: ${filePath}`); } else { @@ -86,6 +92,7 @@ try { const providerProjectId = core.getInput('provider_project_id'); const providerDefaultRegion = core.getInput('provider_default_region'); const providerDefaultZone = core.getInput('provider_default_zone'); + const ingestSecret = uuidv4(); // Check required parameters if (!dirsToScan) { @@ -176,6 +183,7 @@ try { OPENAI_API_KEY: openaiApiKey, MODEL_TYPE_INFERENCE: openaiModelTypeInference, MODEL_TYPE_EMBEDDING: openaiModelTypeEmbedding, + INGEST_SECRET: ingestSecret, REPO_NAME: process.env.GITHUB_REPOSITORY, REPO_URL: `https://github.com/${process.env.GITHUB_REPOSITORY}`, MODE: 'api' @@ -247,7 +255,7 @@ try { // Wait for settings endpoint to be available const settingsEndpoint = 'https://' + containerEndpoint + '/api/settings'; try { - console.log('Checking settings endpoint:', settingsEndpoint); + console.log('Checking settings endpoint...'); const startTime = Date.now(); const timeout = 30000; // 30 seconds timeout diff --git a/api.docker-compose.yml b/api.docker-compose.yml index 14eb7bd..b7b6818 100644 --- a/api.docker-compose.yml +++ b/api.docker-compose.yml @@ -6,6 +6,7 @@ services: REPO_NAME: ${REPO_NAME} REPO_URL: ${REPO_URL} OPENAI_API_KEY: ${OPENAI_API_KEY} + INGEST_SECRET: ${INGEST_SECRET} MODEL_TYPE_INFERENCE: ${MODEL_TYPE_INFERENCE} MODEL_TYPE_EMBEDDING: ${MODEL_TYPE_EMBEDDING} CLEAR_DB_AT_RESTART: ${CLEAR_DB_AT_RESTART} diff --git a/api/config.py b/api/config.py index f1d6aef..d2e3994 100644 --- a/api/config.py +++ b/api/config.py @@ -11,6 +11,7 @@ REPO_NAME = os.environ['REPO_NAME'].strip() REPO_URL = os.getenv('REPO_URL', '').strip() REPO_PATH = os.getenv('REPO_PATH', '').strip() +INGEST_SECRET = os.environ['INGEST_SECRET'].strip() OPEN_AI_API_KEY = os.environ['OPENAI_API_KEY'].strip() MODEL_TYPE_INFERENCE = os.environ['MODEL_TYPE_INFERENCE'].strip() MODEL_TYPE_EMBEDDING = os.environ['MODEL_TYPE_EMBEDDING'].strip() diff --git a/api/index.py b/api/index.py index c6b5e14..5d168ac 100644 --- a/api/index.py +++ b/api/index.py @@ -3,7 +3,7 @@ import binascii import shutil import logging -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Header from pydantic import BaseModel from api.dir_loader import DirLoader from api.api_loader import APILoader @@ -14,6 +14,7 @@ MODEL_TYPE_INFERENCE, MODEL_TYPE_EMBEDDING, PERSIST_DIRECTORY, + INGEST_SECRET, REPO_NAME, REPO_PATH, REPO_URL, @@ -88,9 +89,11 @@ class IngestData(BaseModel): metadata: dict @app.post("/api/ingest") -async def ingest(data: IngestData): +async def ingest(data: IngestData, x_ingest_secret: str = Header(..., alias="X-Ingest-Secret")): if MODE != "api": - raise HTTPException(status_code=400, detail="Ingestion is only available in API mode") + raise HTTPException(status_code=400, detail="Ingestion is only available in API mode (set env MODE=api)") + if x_ingest_secret != INGEST_SECRET: + raise HTTPException(status_code=401, detail="Invalid ingest secret") try: decoded_content = base64.b64decode(data.content).decode('utf-8') if not decoded_content.strip(): diff --git a/app/app/page.jsx b/app/app/page.jsx index 6434818..c5483c2 100644 --- a/app/app/page.jsx +++ b/app/app/page.jsx @@ -37,7 +37,7 @@ export default function Home() { diff --git a/dir.docker-compose.yml b/dir.docker-compose.yml index 3169992..d4ed2a4 100644 --- a/dir.docker-compose.yml +++ b/dir.docker-compose.yml @@ -7,6 +7,7 @@ services: REPO_URL: ${REPO_URL} REPO_PATH: ${REPO_PATH} OPENAI_API_KEY: ${OPENAI_API_KEY} + INGEST_SECRET: ${INGEST_SECRET} MODEL_TYPE_INFERENCE: ${MODEL_TYPE_INFERENCE} MODEL_TYPE_EMBEDDING: ${MODEL_TYPE_EMBEDDING} CLEAR_DB_AT_RESTART: ${CLEAR_DB_AT_RESTART} diff --git a/lib/index.js b/lib/index.js index eb1de62..a777382 100644 --- a/lib/index.js +++ b/lib/index.js @@ -3099,7 +3099,7 @@ exports.colors = [6, 2, 3, 4, 5, 1]; try { // Optional dependency (as in, doesn't need to be installed, NOT like optionalDependencies in package.json) // eslint-disable-next-line import/no-extraneous-dependencies - const supportsColor = __nccwpck_require__(132); + const supportsColor = __nccwpck_require__(9318); if (supportsColor && (supportsColor.stderr || supportsColor).level >= 2) { exports.colors = [ @@ -4687,6 +4687,21 @@ module.exports = function(dst, src) { }; +/***/ }), + +/***/ 1621: +/***/ ((module) => { + + + +module.exports = (flag, argv = process.argv) => { + const prefix = flag.startsWith('-') ? '' : (flag.length === 1 ? '-' : '--'); + const position = argv.indexOf(prefix + flag); + const terminatorPosition = argv.indexOf('--'); + return position !== -1 && (terminatorPosition === -1 || position < terminatorPosition); +}; + + /***/ }), /***/ 7426: @@ -5185,6 +5200,165 @@ function getEnv(key) { exports.j = getProxyForUrl; +/***/ }), + +/***/ 9318: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + + +const os = __nccwpck_require__(2037); +const tty = __nccwpck_require__(6224); +const hasFlag = __nccwpck_require__(1621); + +const {env} = process; + +let flagForceColor; +if (hasFlag('no-color') || + hasFlag('no-colors') || + hasFlag('color=false') || + hasFlag('color=never')) { + flagForceColor = 0; +} else if (hasFlag('color') || + hasFlag('colors') || + hasFlag('color=true') || + hasFlag('color=always')) { + flagForceColor = 1; +} + +function envForceColor() { + if ('FORCE_COLOR' in env) { + if (env.FORCE_COLOR === 'true') { + return 1; + } + + if (env.FORCE_COLOR === 'false') { + return 0; + } + + return env.FORCE_COLOR.length === 0 ? 1 : Math.min(Number.parseInt(env.FORCE_COLOR, 10), 3); + } +} + +function translateLevel(level) { + if (level === 0) { + return false; + } + + return { + level, + hasBasic: true, + has256: level >= 2, + has16m: level >= 3 + }; +} + +function supportsColor(haveStream, {streamIsTTY, sniffFlags = true} = {}) { + const noFlagForceColor = envForceColor(); + if (noFlagForceColor !== undefined) { + flagForceColor = noFlagForceColor; + } + + const forceColor = sniffFlags ? flagForceColor : noFlagForceColor; + + if (forceColor === 0) { + return 0; + } + + if (sniffFlags) { + if (hasFlag('color=16m') || + hasFlag('color=full') || + hasFlag('color=truecolor')) { + return 3; + } + + if (hasFlag('color=256')) { + return 2; + } + } + + if (haveStream && !streamIsTTY && forceColor === undefined) { + return 0; + } + + const min = forceColor || 0; + + if (env.TERM === 'dumb') { + return min; + } + + if (process.platform === 'win32') { + // Windows 10 build 10586 is the first Windows release that supports 256 colors. + // Windows 10 build 14931 is the first release that supports 16m/TrueColor. + const osRelease = os.release().split('.'); + if ( + Number(osRelease[0]) >= 10 && + Number(osRelease[2]) >= 10586 + ) { + return Number(osRelease[2]) >= 14931 ? 3 : 2; + } + + return 1; + } + + if ('CI' in env) { + if (['TRAVIS', 'CIRCLECI', 'APPVEYOR', 'GITLAB_CI', 'GITHUB_ACTIONS', 'BUILDKITE', 'DRONE'].some(sign => sign in env) || env.CI_NAME === 'codeship') { + return 1; + } + + return min; + } + + if ('TEAMCITY_VERSION' in env) { + return /^(9\.(0*[1-9]\d*)\.|\d{2,}\.)/.test(env.TEAMCITY_VERSION) ? 1 : 0; + } + + if (env.COLORTERM === 'truecolor') { + return 3; + } + + if ('TERM_PROGRAM' in env) { + const version = Number.parseInt((env.TERM_PROGRAM_VERSION || '').split('.')[0], 10); + + switch (env.TERM_PROGRAM) { + case 'iTerm.app': + return version >= 3 ? 3 : 2; + case 'Apple_Terminal': + return 2; + // No default + } + } + + if (/-256(color)?$/i.test(env.TERM)) { + return 2; + } + + if (/^screen|^xterm|^vt100|^vt220|^rxvt|color|ansi|cygwin|linux/i.test(env.TERM)) { + return 1; + } + + if ('COLORTERM' in env) { + return 1; + } + + return min; +} + +function getSupportLevel(stream, options = {}) { + const level = supportsColor(stream, { + streamIsTTY: stream && stream.isTTY, + ...options + }); + + return translateLevel(level); +} + +module.exports = { + supportsColor: getSupportLevel, + stdout: getSupportLevel({isTTY: tty.isatty(1)}), + stderr: getSupportLevel({isTTY: tty.isatty(2)}) +}; + + /***/ }), /***/ 4294: @@ -28183,14 +28357,6 @@ function version(uuid) { var _default = version; exports["default"] = _default; -/***/ }), - -/***/ 132: -/***/ ((module) => { - -module.exports = eval("require")("supports-color"); - - /***/ }), /***/ 9491: @@ -73521,18 +73687,24 @@ __nccwpck_require__.a(__webpack_module__, async (__webpack_handle_async_dependen const providers = ['scaleway']; +function uuidv4() { + return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, c => + (+c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> +c / 4).toString(16) + ); +} + function isValidFile(filePath) { return fs__WEBPACK_IMPORTED_MODULE_0__.statSync(filePath).isFile() && !path__WEBPACK_IMPORTED_MODULE_1__.basename(filePath).startsWith('.'); } -async function sendFileToApi(filePath, apiUrl) { +async function sendFileToApi(filePath, apiUrl, ingestSecret) { const content = fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(filePath, { encoding: 'base64' }); const metadata = { 'source': path__WEBPACK_IMPORTED_MODULE_1__.basename(filePath) }; const payload = { 'content': content, 'metadata': metadata }; try { console.log(`Sending: ${filePath}`); - const response = await axios__WEBPACK_IMPORTED_MODULE_4__/* ["default"].post */ .Z.post(apiUrl, payload, { headers: { 'Content-Type': 'application/json' } }); + const response = await axios__WEBPACK_IMPORTED_MODULE_4__/* ["default"].post */ .Z.post(apiUrl, payload, { headers: { 'Content-Type': 'application/json', 'X-Ingest-Secret': ingestSecret } }); return response; } catch (error) { console.error(`Error sending file: ${error.message}`); @@ -73549,7 +73721,7 @@ function isExcluded(filePath, excludeFiles) { async function processFile(filePath, apiUrl) { console.log(`Sending file: ${filePath}`); - const response = await sendFileToApi(filePath, apiUrl); + const response = await sendFileToApi(filePath, apiUrl, ingestSecret); if (response.status === 200) { console.log(`Successfully ingested: ${filePath}`); } else { @@ -73601,6 +73773,7 @@ try { const providerProjectId = _actions_core__WEBPACK_IMPORTED_MODULE_2__.getInput('provider_project_id'); const providerDefaultRegion = _actions_core__WEBPACK_IMPORTED_MODULE_2__.getInput('provider_default_region'); const providerDefaultZone = _actions_core__WEBPACK_IMPORTED_MODULE_2__.getInput('provider_default_zone'); + const ingestSecret = uuidv4(); // Check required parameters if (!dirsToScan) { @@ -73691,6 +73864,7 @@ try { OPENAI_API_KEY: openaiApiKey, MODEL_TYPE_INFERENCE: openaiModelTypeInference, MODEL_TYPE_EMBEDDING: openaiModelTypeEmbedding, + INGEST_SECRET: ingestSecret, REPO_NAME: process.env.GITHUB_REPOSITORY, REPO_URL: `https://github.com/${process.env.GITHUB_REPOSITORY}`, MODE: 'api' @@ -73762,7 +73936,7 @@ try { // Wait for settings endpoint to be available const settingsEndpoint = 'https://' + containerEndpoint + '/api/settings'; try { - console.log('Checking settings endpoint:', settingsEndpoint); + console.log('Checking settings endpoint...'); const startTime = Date.now(); const timeout = 30000; // 30 seconds timeout diff --git a/package-lock.json b/package-lock.json index f0e90ba..c7aab23 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,23 +1,24 @@ { "name": "gh-action-repochat", - "version": "0.7.0", + "version": "0.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gh-action-repochat", - "version": "0.7.0", + "version": "0.8.0", "dependencies": { "@actions/core": "^1.10.1", "@scaleway/configuration-loader": "^1.0.5", "@scaleway/sdk": "^2.44.0", - "@vercel/ncc": "^0.38.1" + "@vercel/ncc": "^0.38.1", + "axios": "^1.7.7" }, "devDependencies": { "@types/jest": "^29.5.2", "@typescript-eslint/eslint-plugin": "^5.59.9", - "axios": "^1.7.7", "commander": "^12.1.0", + "concurrently": "^8.2.2", "eslint": "^8.42.0", "eslint-plugin-jest": "^27.2.1", "lint-staged": "^13.2.2", @@ -152,6 +153,18 @@ "node": ">=4" } }, + "node_modules/@babel/runtime": { + "version": "7.25.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.25.6.tgz", + "integrity": "sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ==", + "dev": true, + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -827,14 +840,12 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/axios": { "version": "1.7.7", "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", - "dev": true, "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -1002,6 +1013,20 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1030,7 +1055,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -1053,6 +1077,33 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "node_modules/concurrently": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.2.tgz", + "integrity": "sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.2", + "date-fns": "^2.30.0", + "lodash": "^4.17.21", + "rxjs": "^7.8.1", + "shell-quote": "^1.8.1", + "spawn-command": "0.0.2", + "supports-color": "^8.1.1", + "tree-kill": "^1.2.2", + "yargs": "^17.7.2" + }, + "bin": { + "conc": "dist/bin/concurrently.js", + "concurrently": "dist/bin/concurrently.js" + }, + "engines": { + "node": "^14.13.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" + } + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -1073,6 +1124,22 @@ "node": ">= 8" } }, + "node_modules/date-fns": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", + "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.21.0" + }, + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -1100,7 +1167,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -1153,6 +1219,21 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "dev": true }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -1536,7 +1617,6 @@ "version": "1.15.9", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", - "dev": true, "funding": [ { "type": "individual", @@ -1556,7 +1636,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -1572,6 +1651,15 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -1742,6 +1830,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -2136,6 +2233,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -2278,7 +2381,6 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, "engines": { "node": ">= 0.6" } @@ -2287,7 +2389,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, "dependencies": { "mime-db": "1.52.0" }, @@ -2566,8 +2667,7 @@ "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "node_modules/punycode": { "version": "2.3.1", @@ -2604,6 +2704,21 @@ "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "dev": true + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -2708,6 +2823,15 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "dev": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -2741,6 +2865,15 @@ "node": ">=8" } }, + "node_modules/shell-quote": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", + "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -2796,6 +2929,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/spawn-command": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz", + "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==", + "dev": true + }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -2826,6 +2965,20 @@ "node": ">=0.6.19" } }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -2862,6 +3015,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -2880,6 +3048,15 @@ "node": ">=8.0" } }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, + "bin": { + "tree-kill": "cli.js" + } + }, "node_modules/ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", @@ -2923,6 +3100,12 @@ } } }, + "node_modules/tslib": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", + "dev": true + }, "node_modules/tsutils": { "version": "3.21.0", "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz", @@ -3053,12 +3236,38 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/yaml": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz", @@ -3068,6 +3277,33 @@ "node": ">= 14" } }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/package.json b/package.json index 2833054..1c6ae4b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gh-action-repochat", - "version": "0.8.0", + "version": "0.9.0", "type": "module", "description": "GitHub Action to run repochat", "main": "index.js", @@ -16,10 +16,11 @@ "prettier": "^2.8.8", "ts-node": "^10.9.1", "typescript": "^5.1.3", - "axios": "^1.7.7", - "commander": "^12.1.0" + "commander": "^12.1.0", + "concurrently": "^8.2.2" }, "dependencies": { + "axios": "^1.7.7", "@actions/core": "^1.10.1", "@scaleway/configuration-loader": "^1.0.5", "@scaleway/sdk": "^2.44.0", diff --git a/scripts/ingest-docs-api.js b/scripts/ingest-docs-api.js index 5587d5a..13214eb 100644 --- a/scripts/ingest-docs-api.js +++ b/scripts/ingest-docs-api.js @@ -7,14 +7,19 @@ function isValidFile(filePath) { return fs.statSync(filePath).isFile() && !path.basename(filePath).startsWith('.'); } -async function sendFileToApi(filePath, apiUrl) { +async function sendFileToApi(filePath, apiUrl, ingestSecret) { const content = fs.readFileSync(filePath, { encoding: 'base64' }); const metadata = { 'source': path.basename(filePath) }; const payload = { 'content': content, 'metadata': metadata }; try { console.log(`Sending: ${filePath} to ${apiUrl}`); - const response = await axios.post(apiUrl, payload, { headers: { 'Content-Type': 'application/json' } }); + const response = await axios.post(apiUrl, payload, { + headers: { + 'Content-Type': 'application/json', + 'X-Ingest-Secret': ingestSecret + } + }); return response; } catch (error) { console.error(`Error sending file: ${error.message}`); @@ -29,9 +34,9 @@ function isExcluded(filePath, excludeFiles) { }); } -async function processFile(filePath, apiUrl) { +async function processFile(filePath, apiUrl, ingestSecret) { console.log(`Sending file: ${filePath}`); - const response = await sendFileToApi(filePath, apiUrl); + const response = await sendFileToApi(filePath, apiUrl, ingestSecret); if (response.status === 200) { console.log(`Successfully ingested: ${filePath}`); } else { @@ -39,7 +44,7 @@ async function processFile(filePath, apiUrl) { } } -async function ingestFiles(directoryPath, apiUrl, excludeFiles = []) { +async function ingestFiles(directoryPath, apiUrl, ingestSecret, excludeFiles = []) { if (!fs.existsSync(directoryPath)) { console.error(`Error: ${directoryPath} does not exist.`); return; @@ -48,7 +53,7 @@ async function ingestFiles(directoryPath, apiUrl, excludeFiles = []) { const stats = fs.statSync(directoryPath); if (stats.isFile()) { if (isValidFile(directoryPath) && !isExcluded(directoryPath, excludeFiles)) { - await processFile(directoryPath, apiUrl); + await processFile(directoryPath, apiUrl, ingestSecret); } return; } @@ -63,9 +68,9 @@ async function ingestFiles(directoryPath, apiUrl, excludeFiles = []) { for (const file of files) { const filePath = path.join(directoryPath, file.name); if (file.isDirectory()) { - await ingestFiles(filePath, apiUrl, excludeFiles); + await ingestFiles(filePath, apiUrl, ingestSecret, excludeFiles); } else if (isValidFile(filePath) && !isExcluded(filePath, excludeFiles)) { - await processFile(filePath, apiUrl); + await processFile(filePath, apiUrl, ingestSecret); } else { console.log(`Skipping invalid, hidden, or excluded file: ${filePath}`); } @@ -75,12 +80,13 @@ async function ingestFiles(directoryPath, apiUrl, excludeFiles = []) { program .description('Ingest files from a directory to the API') .argument('', 'Path to the directory containing files to ingest') + .requiredOption('--ingest-secret ', 'Ingest secret for API authentication') .option('--endpoint ', 'API endpoint URL', 'http://localhost:5328') .option('--exclude ', 'Comma-separated list of files or patterns to exclude', 'node_modules,.git,.env,package-lock.json') .action(async (directoryPath, options) => { const apiUrl = `${options.endpoint}/api/ingest`; const excludeFiles = options.exclude.split(',').map(item => item.trim()); - await ingestFiles(directoryPath, apiUrl, excludeFiles); + await ingestFiles(directoryPath, apiUrl, options.ingestSecret, excludeFiles); }); -program.parse(process.argv); +program.parse(process.argv); \ No newline at end of file diff --git a/scripts/ingest-docs-api.py b/scripts/ingest-docs-api.py index 6e52245..b04c6bd 100644 --- a/scripts/ingest-docs-api.py +++ b/scripts/ingest-docs-api.py @@ -1,4 +1,4 @@ -# Usage: python3 ingest_files.py /path/to/your/directory +# Usage: python3 ingest_files.py /path/to/your/directory_or_file --ingest-secret YOUR_SECRET import os import requests @@ -7,9 +7,9 @@ from pathlib import Path def is_valid_file(file_path): - return os.path.isfile(file_path) and not file_path.startswith('.') + return os.path.isfile(file_path) and not os.path.basename(file_path).startswith('.') -def send_file_to_api(file_path, api_url): +def send_file_to_api(file_path, api_url, ingest_secret): with open(file_path, 'rb') as file: content = base64.b64encode(file.read()).decode('utf-8') @@ -20,32 +20,40 @@ def send_file_to_api(file_path, api_url): "metadata": metadata } - response = requests.post(api_url, json=payload) + headers = { + "X-Ingest-Secret": ingest_secret + } + + response = requests.post(api_url, json=payload, headers=headers) return response -def main(directory_path, api_url): - directory = Path(directory_path) - if not directory.is_dir(): - print(f"Error: {directory_path} is not a valid directory.") - return - - for root, _, files in os.walk(directory): - for file in files: - file_path = os.path.join(root, file) - if is_valid_file(file_path): - print(f"Sending file: {file_path}") - response = send_file_to_api(file_path, api_url) - if response.status_code == 200: - print(f"Successfully ingested: {file_path}") - else: - print(f"Failed to ingest {file_path}. Status code: {response.status_code}") +def process_path(path, api_url, ingest_secret): + if os.path.isfile(path): + if is_valid_file(path): + print(f"Sending file: {path}") + response = send_file_to_api(path, api_url, ingest_secret) + if response.status_code == 200: + print(f"Successfully ingested: {path}") else: - print(f"Skipping invalid or hidden file: {file_path}") + print(f"Failed to ingest {path}. Status code: {response.status_code}") + else: + print(f"Skipping invalid or hidden file: {path}") + elif os.path.isdir(path): + for root, _, files in os.walk(path): + for file in files: + file_path = os.path.join(root, file) + process_path(file_path, api_url, ingest_secret) + else: + print(f"Error: {path} is not a valid file or directory.") + +def main(path, api_url, ingest_secret): + process_path(path, api_url, ingest_secret) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Ingest files from a directory to the API") - parser.add_argument("path", help="Path to the directory containing files to ingest") + parser = argparse.ArgumentParser(description="Ingest files from a directory or a single file to the API") + parser.add_argument("path", help="Path to the directory or file to ingest") parser.add_argument("--endpoint", default="http://localhost:5328", help="API endpoint URL") + parser.add_argument("--ingest-secret", required=True, help="Secret key for API authentication") args = parser.parse_args() - main(args.path, f"{args.endpoint}/api/ingest") + main(args.path, f"{args.endpoint}/api/ingest", args.ingest_secret)