fix: fetch large files via blob request (#23)

Author: jurijat

CHANGELOG.md

@@ -4,6 +4,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 
+## [0.8.1] - 2025-04-14
+
+### Changed
+
+- Improved GitHub file fetching, by adding a fallback to the `blobs` API, when file content is too large
+
 ## [0.8.0] - 2025-04-08
 
 ### Added

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@open-resource-discovery/provider-server",
-  "version": "0.8.0",
+  "version": "0.8.1",
   "description": "A CLI application or server that takes multiple ORD documents and other metadata files and exposes them as a ORD Provider implementation (ORD Document API)",
   "engines": {
     "node": ">=22.8.0",

package.json

@@ -1,3 +1,7 @@
+import { config } from "dotenv";
+
+config();
+
 // URL path constants
 export const PATH_CONSTANTS = {
   // Base paths

src/constant.ts

@@ -1,6 +1,6 @@
 import { ORDDocument } from "@open-resource-discovery/specification";
 import { DocumentRepository } from "./interfaces/documentRepository.js";
-import { GithubOpts, GitHubFileResponse, GitHubInstance } from "../model/github.js";
+import { GithubOpts, GitHubInstance } from "../model/github.js";
 import { fetchGitHubFile, getDirectoryHash, getGithubDirectoryContents } from "../util/github.js";
 import { normalizePath, joinFilePaths } from "../util/pathUtils.js";
 import { PATH_CONSTANTS } from "../constant.js";
@@ -34,8 +34,7 @@ export class GithubDocumentRepository implements DocumentRepository {
   public async getDocument(relativePath: string): Promise<ORDDocument | null> {
     const githubPath = this.getFullGithubPath(relativePath);
     try {
-      const response = await fetchGitHubFile<GitHubFileResponse>(this.githubInstance, githubPath, this.githubToken);
-      const content = Buffer.from(response.content, "base64").toString("utf-8");
+      const content = await fetchGitHubFile(this.githubInstance, githubPath, this.githubToken);
       const jsonData = JSON.parse(content);
 
       // Basic validation to ensure it's an ORD document
@@ -119,8 +118,7 @@ export class GithubDocumentRepository implements DocumentRepository {
   public async getFileContent(relativePath: string): Promise<string | Buffer | null> {
     const githubPath = this.getFullGithubPath(relativePath);
     try {
-      const response = await fetchGitHubFile<GitHubFileResponse>(this.githubInstance, githubPath, this.githubToken);
-      return Buffer.from(response.content, "base64").toString("utf-8");
+      return await fetchGitHubFile(this.githubInstance, githubPath, this.githubToken);
     } catch (error) {
       log.error(`Error fetching file content from GitHub path ${githubPath}: ${error}`);
       return null;

src/repositories/githubDocumentRepository.ts

@@ -1,9 +1,10 @@
 import { describe, it, expect, jest, beforeAll, afterAll, beforeEach } from "@jest/globals";
-import { fetchGitHubFile } from "src/util/github.js";
-import { GitHubFileResponse, GitHubInstance } from "src/model/github.js";
+import { fetchGitHubFile, GitHubContentItem } from "src/util/github.js";
+import { GitHubInstance } from "src/model/github.js";
 import { GitHubFileNotFoundError, GitHubNetworkError, GitHubAccessError } from "src/model/error/GithubErrors.js";
+import { Buffer } from "buffer";
 
-describe("GitHub", () => {
+describe("GitHub Util", () => {
   beforeAll(() => {
     const mockFetch = jest.fn() as jest.MockedFunction<typeof fetch>;
     global.fetch = mockFetch;
@@ -23,114 +24,172 @@ describe("GitHub", () => {
     branch: "main",
   };
 
-  it("should fetch GitHub file successfully", async () => {
-    const mockFileContent: GitHubFileResponse = {
+  const mockToken = "test-token";
+  const fileContent = "Decoded file content";
+  const fileContentBase64 = Buffer.from(fileContent).toString("base64");
+  const filePath = "path/to/test.json";
+  const fileSha = "abc123def456";
+  const contentsUrl = `https://api.github.com/repos/owner/repo/contents/${filePath}?ref=main`;
+  const blobUrl = `https://api.github.com/repos/owner/repo/git/blobs/${fileSha}`;
+
+  it("should fetch small GitHub file successfully (content in first response)", async () => {
+    const mockMetadataResponse: GitHubContentItem = {
       name: "test.json",
-      path: "test.json",
-      sha: "abc123",
+      path: filePath,
+      sha: fileSha,
       size: 100,
-      url: "https://api.github.com/repos/owner/repo/contents/test.json",
-      html_url: "https://github.com/owner/repo/blob/main/test.json",
-      git_url: "https://api.github.com/repos/owner/repo/git/blobs/abc123",
-      download_url: "https://raw.githubusercontent.com/owner/repo/main/test.json",
       type: "file",
-      content: "base64encodedcontent",
+      content: fileContentBase64,
       encoding: "base64",
     };
-    const mockResponse: Partial<Response> = {
+    const mockFetchResponse: Partial<Response> = {
       ok: true,
       status: 200,
-      statusText: "OK",
-      json: jest.fn<() => Promise<unknown>>().mockResolvedValue(mockFileContent),
+      json: jest.fn<() => Promise<unknown>>().mockResolvedValue(mockMetadataResponse),
+    };
+
+    const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
+    mockFetch.mockResolvedValueOnce(mockFetchResponse as Response);
+
+    const result = await fetchGitHubFile(mockInstance, filePath, mockToken);
+
+    expect(result).toEqual(fileContent);
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith(contentsUrl, {
+      headers: { Authorization: `Token ${mockToken}` },
+    });
+  });
+
+  it("should fetch large GitHub file successfully (content via blob API)", async () => {
+    const mockMetadataResponse: GitHubContentItem = {
+      name: "test.json",
+      path: filePath,
+      sha: fileSha,
+      size: 2 * 1024 * 1024,
+      type: "file",
+    };
+    const mockBlobResponseData = {
+      sha: fileSha,
+      size: 2 * 1024 * 1024,
+      content: fileContentBase64,
+      encoding: "base64",
+    };
+
+    const mockFetchMetadataResponse: Partial<Response> = {
+      ok: true,
+      status: 200,
+      json: jest.fn<() => Promise<unknown>>().mockResolvedValue(mockMetadataResponse),
+    };
+    const mockFetchBlobResponse: Partial<Response> = {
+      ok: true,
+      status: 200,
+      json: jest.fn<() => Promise<unknown>>().mockResolvedValue(mockBlobResponseData),
     };
 
     const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
-    mockFetch.mockResolvedValue(mockResponse as Response);
+    mockFetch
+      .mockResolvedValueOnce(mockFetchMetadataResponse as Response)
+      .mockResolvedValueOnce(mockFetchBlobResponse as Response);
 
-    const result = await fetchGitHubFile<GitHubFileResponse>(mockInstance, "test.json", "token");
+    const result = await fetchGitHubFile(mockInstance, filePath, mockToken);
 
-    expect(result).toEqual(mockFileContent);
-    expect(mockFetch).toHaveBeenCalledWith("https://api.github.com/repos/owner/repo/contents/test.json?ref=main", {
-      headers: { Authorization: "Token token" },
+    expect(result).toEqual(fileContent);
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(mockFetch).toHaveBeenNthCalledWith(1, contentsUrl, {
+      headers: { Authorization: `Token ${mockToken}` },
+    });
+    expect(mockFetch).toHaveBeenNthCalledWith(2, blobUrl, {
+      headers: { Authorization: `Token ${mockToken}` },
     });
   });
 
-  it("should throw GitHubFileNotFoundError when file is not found", async () => {
+  it("should throw GitHubFileNotFoundError when file is not found (404 on metadata fetch)", async () => {
     const mockResponse: Partial<Response> = {
       ok: false,
       status: 404,
       statusText: "Not Found",
     };
     const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
-    mockFetch.mockResolvedValue(mockResponse as Response);
-    await expect(fetchGitHubFile<GitHubFileResponse>(mockInstance, "nonexistent.json", "token")).rejects.toThrow(
-      GitHubFileNotFoundError,
+    mockFetch.mockResolvedValueOnce(mockResponse as Response);
+
+    await expect(fetchGitHubFile(mockInstance, "nonexistent.json", mockToken)).rejects.toThrow(GitHubFileNotFoundError);
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith(
+      "https://api.github.com/repos/owner/repo/contents/nonexistent.json?ref=main",
+      expect.any(Object),
     );
   });
 
-  it("should throw GitHubNetworkError on network errors", async () => {
+  it("should throw GitHubNetworkError on network errors during metadata fetch", async () => {
     const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
-    mockFetch.mockRejectedValue(new TypeError("Failed to fetch"));
-    await expect(fetchGitHubFile<GitHubFileResponse>(mockInstance, "test.json", "token")).rejects.toThrow(
-      GitHubNetworkError,
-    );
+    mockFetch.mockRejectedValueOnce(new TypeError("Failed to fetch"));
+
+    await expect(fetchGitHubFile(mockInstance, filePath, mockToken)).rejects.toThrow(GitHubNetworkError);
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith(contentsUrl, expect.any(Object));
   });
 
-  it("should throw GitHubAccessError on invalid JSON response", async () => {
+  it("should throw GitHubAccessError on invalid JSON response during metadata fetch", async () => {
     const mockResponse: Partial<Response> = {
       ok: true,
       status: 200,
       statusText: "OK",
-      json: jest.fn<() => Promise<unknown>>().mockRejectedValue(new Error("Invalid JSON")),
+      json: jest.fn<() => Promise<unknown>>().mockRejectedValue(new SyntaxError("Invalid JSON")),
     };
     const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
-    mockFetch.mockResolvedValue(mockResponse as Response);
-    await expect(fetchGitHubFile<GitHubFileResponse>(mockInstance, "test.json", "token")).rejects.toThrow(
-      GitHubAccessError,
-    );
+    mockFetch.mockResolvedValueOnce(mockResponse as Response);
+
+    await expect(fetchGitHubFile(mockInstance, filePath, mockToken)).rejects.toThrow(GitHubAccessError);
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith(contentsUrl, expect.any(Object));
   });
 
-  it("should throw GitHubAccessError on empty response", async () => {
+  it("should throw GitHubAccessError on empty response during metadata fetch", async () => {
     const mockResponse: Partial<Response> = {
       ok: true,
       status: 200,
       statusText: "OK",
       json: jest.fn<() => Promise<unknown>>().mockResolvedValue(null),
     };
     const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
-    mockFetch.mockResolvedValue(mockResponse as Response);
-    await expect(fetchGitHubFile<GitHubFileResponse>(mockInstance, "test.json", "token")).rejects.toThrow(
-      GitHubAccessError,
-    );
+    mockFetch.mockResolvedValueOnce(mockResponse as Response);
+
+    await expect(fetchGitHubFile(mockInstance, filePath, mockToken)).rejects.toThrow(GitHubAccessError);
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith(contentsUrl, expect.any(Object));
   });
 
-  it("should throw GitHubAccessError on unauthorized access", async () => {
+  it("should throw GitHubAccessError on unauthorized access during metadata fetch", async () => {
     const mockResponse: Partial<Response> = {
       ok: false,
       status: 401,
       statusText: "Unauthorized",
     };
     const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
-    mockFetch.mockResolvedValue(mockResponse as Response);
-    await expect(fetchGitHubFile<GitHubFileResponse>(mockInstance, "test.json", "token")).rejects.toThrow(
-      GitHubAccessError,
-    );
+    mockFetch.mockResolvedValueOnce(mockResponse as Response);
+
+    await expect(fetchGitHubFile(mockInstance, filePath, mockToken)).rejects.toThrow(GitHubAccessError);
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith(contentsUrl, expect.any(Object));
   });
 
-  it("should include proper error details in GitHubAccessError", async () => {
+  it("should include proper error details in GitHubAccessError from metadata fetch", async () => {
     const mockResponse: Partial<Response> = {
       ok: false,
       status: 401,
       statusText: "Unauthorized",
     };
     const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
-    mockFetch.mockResolvedValue(mockResponse as Response);
+    mockFetch.mockResolvedValueOnce(mockResponse as Response);
+
     try {
-      await fetchGitHubFile<GitHubFileResponse>(mockInstance, "test.json", "token");
+      await fetchGitHubFile(mockInstance, filePath, mockToken);
+
+      expect(true).toBe(false);
     } catch (error) {
       expect(error).toBeInstanceOf(GitHubAccessError);
       if (error instanceof GitHubAccessError) {
-        expect(error.errorItem.target).toBe("test.json");
+        expect(error.errorItem.target).toBe(filePath);
         expect(error.errorItem.details).toBeDefined();
         expect(error.errorItem.details![0].code).toBe(`HTTP_401`);
       }

src/util/__tests__/github.test.ts

@@ -137,22 +137,25 @@ describe("Options Validation", () => {
         ]),
       };
 
-      const mockResponseFile: Partial<Response> = {
+      const mockResponseFileMetadata: Partial<Response> = {
         ok: true,
         status: 200,
         statusText: "OK",
         json: jest.fn<() => Promise<unknown>>().mockResolvedValue({
           name: "test.json",
           path: "documents/test.json",
           type: "file",
+          sha: "dummySha123",
+          size: 123,
           content: Buffer.from(JSON.stringify({ openResourceDiscovery: {} })).toString("base64"),
+          encoding: "base64",
         }),
       };
 
       const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>;
       mockFetch
         .mockResolvedValueOnce(mockResponseDirectory as Response)
-        .mockResolvedValueOnce(mockResponseFile as Response);
+        .mockResolvedValueOnce(mockResponseFileMetadata as Response);
 
       const options = {
         sourceType: OptSourceType.Github,