Skip to content

Commit

Permalink
Add support for fetching single document in documents folder (#607)
Browse files Browse the repository at this point in the history
  • Loading branch information
timothycarambat authored Jan 16, 2024
1 parent e973c1e commit c61cbd1
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,6 @@ function CitationDetailModal({ source, onClose }) {
);
}


function truncateMiddle(title) {
if (title.length <= 18) return title;

Expand Down
60 changes: 59 additions & 1 deletion server/endpoints/api/document/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ const {
acceptedFileTypes,
processDocument,
} = require("../../../utils/files/documentProcessor");
const { viewLocalFiles } = require("../../../utils/files");
const {
viewLocalFiles,
findDocumentInDocuments,
} = require("../../../utils/files");
const { handleUploads } = setupMulter();

function apiDocumentEndpoints(app) {
Expand Down Expand Up @@ -133,6 +136,61 @@ function apiDocumentEndpoints(app) {
}
});

app.get("/v1/document/:docName", [validApiKey], async (request, response) => {
/*
#swagger.tags = ['Documents']
#swagger.description = 'Get a single document by its unique AnythingLLM document name'
#swagger.parameters['docName'] = {
in: 'path',
description: 'Unique document name to find (name in /documents)',
required: true,
type: 'string'
}
#swagger.responses[200] = {
content: {
"application/json": {
schema: {
type: 'object',
example: {
"localFiles": {
"name": "documents",
"type": "folder",
items: [
{
"name": "my-stored-document.txt-uuid1234.json",
"type": "file",
"id": "bb07c334-4dab-4419-9462-9d00065a49a1",
"url": "file://my-stored-document.txt",
"title": "my-stored-document.txt",
"cached": false
},
]
}
}
}
}
}
}
#swagger.responses[403] = {
schema: {
"$ref": "#/definitions/InvalidAPIKey"
}
}
*/
try {
const { docName } = request.params;
const document = await findDocumentInDocuments(docName);
if (!document) {
response.sendStatus(404).end();
return;
}
response.status(200).json({ document });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
});

app.get(
"/v1/document/accepted-file-types",
[validApiKey],
Expand Down
75 changes: 75 additions & 0 deletions server/swagger/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,81 @@
}
}
},
"/v1/document/{docName}": {
"get": {
"tags": [
"Documents"
],
"description": "Get a single document by its unique AnythingLLM document name",
"parameters": [
{
"name": "docName",
"in": "path",
"required": true,
"schema": {
"type": "string"
},
"description": "Unique document name to find (name in /documents)"
},
{
"name": "Authorization",
"in": "header",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"example": {
"localFiles": {
"name": "documents",
"type": "folder",
"items": [
{
"name": "my-stored-document.txt-uuid1234.json",
"type": "file",
"id": "bb07c334-4dab-4419-9462-9d00065a49a1",
"url": "file://my-stored-document.txt",
"title": "my-stored-document.txt",
"cached": false
}
]
}
}
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/InvalidAPIKey"
}
},
"application/xml": {
"schema": {
"$ref": "#/components/schemas/InvalidAPIKey"
}
}
}
},
"404": {
"description": "Not Found"
},
"500": {
"description": "Internal Server Error"
}
}
}
},
"/v1/document/accepted-file-types": {
"get": {
"tags": [
Expand Down
38 changes: 38 additions & 0 deletions server/utils/files/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,49 @@ async function purgeVectorCache(filename = null) {
return;
}

// Search for a specific document by its unique name in the entire `documents`
// folder via iteration of all folders and checking if the expected file exists.
async function findDocumentInDocuments(documentName = null) {
if (!documentName) return null;
const documentsFolder =
process.env.NODE_ENV === "development"
? path.resolve(__dirname, `../../storage/documents`)
: path.resolve(process.env.STORAGE_DIR, `documents`);

for (const folder of fs.readdirSync(documentsFolder)) {
const isFolder = fs
.lstatSync(path.join(documentsFolder, folder))
.isDirectory();
if (!isFolder) continue;

const targetFilename = normalizePath(documentName);
const targetFileLocation = path.join(
documentsFolder,
folder,
targetFilename
);
if (!fs.existsSync(targetFileLocation)) continue;

const fileData = fs.readFileSync(targetFileLocation, "utf8");
const cachefilename = `${folder}/${targetFilename}`;
const { pageContent, ...metadata } = JSON.parse(fileData);
return {
name: targetFilename,
type: "file",
...metadata,
cached: await cachedVectorInformation(cachefilename, true),
};
}

return null;
}

function normalizePath(filepath = "") {
return path.normalize(filepath).replace(/^(\.\.(\/|\\|$))+/, "");
}

module.exports = {
findDocumentInDocuments,
cachedVectorInformation,
viewLocalFiles,
purgeSourceDocument,
Expand Down

0 comments on commit c61cbd1

Please sign in to comment.