Skip to content

Commit d225500

Browse files
clostaoclostao
authored
update DAG indexer + refactor DSN fetcher (#134)
* feat: implement DAG node retrieval in dsnFetcher service * Update dag indexer (#135) * refactor: update dag indexer * chore: add dag-indexer command to package.json for improved workspace management * feat: use updated entity for fetching metadata * chore: update GitHub Actions workflow triggers to simplify event handling * refactor: update ExtendedIPLDMetadata interface to omit 'data' property from IPLDNodeData * chore: update test commands in Makefile to include file-retriever and object-mapping-indexer tests * refactor: simplify getPartial function in dsnFetcher and update related tests for chunk retrieval * refactor: clean up partialDownloads test by removing unused imports * chore: update test command in Makefile to replace object-mapping-indexer with indexer * refactor: enhance dsnFetcher to streamline file reconstruction * fix: correct index increment logic in migrateToFileCache function of dsnFetcher * fix: handle errors in fetchFileAsStream to ensure proper stream closure in dsnFetcher * refactor: construct ipld tree using recursive psql query * fix: ensure proper stream closure in fetchFileAsStream by moving controller.close() to the correct position
1 parent 24bcf36 commit d225500

File tree

11 files changed

+510
-259
lines changed

11 files changed

+510
-259
lines changed

‎.github/workflows/services.yml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,9 @@ on:
44
push:
55
branches:
66
- main
7-
paths:
8-
- 'backend/**' # Only trigger on changes within this folder
9-
- '.github/**'
107
pull_request:
11-
paths:
12-
- 'backend/**' # Only trigger on changes within this folder
13-
- '.github/**'
8+
branches:
9+
- main
1410
workflow_dispatch:
1511

1612
jobs:

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ lint:
1717
yarn lint
1818

1919
test:
20-
yarn test
20+
yarn file-retriever test
21+
yarn indexer test
2122

‎common/models/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
export * from './mapping.js'
2+
export * from './ipldNodeMetadata.js'

‎common/models/src/ipldNodeMetadata.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { IPLDNodeData } from '@autonomys/auto-dag-data'
2+
3+
export interface ExtendedIPLDMetadata extends Omit<IPLDNodeData, 'data'> {
4+
cid: string
5+
blockHeight: number
6+
blockHash: string
7+
extrinsicId: string
8+
extrinsicHash: string
9+
indexInBlock: number
10+
links: string[]
11+
blake3Hash: string
12+
timestamp: Date
13+
}

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"benchmark": "node --loader ts-node/esm benchmarks/taurus.ts",
1313
"models": "yarn workspace @auto-files/models",
1414
"rpc": "yarn workspace @auto-files/rpc-apis",
15+
"dag-indexer": "yarn workspace dag-indexer",
1516
"build": "yarn workspaces foreach --topological --verbose --all run build",
1617
"lint": "yarn workspaces foreach --topological --verbose --all run lint",
1718
"test": "yarn workspaces foreach --topological --verbose --all run test"

‎services/dag-indexer/schema.graphql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
type Node @entity {
22
id: ID!
3+
cid: String! @index
4+
type: String
5+
linkDepth: Int!
6+
name: String
37
blockHeight: BigInt!
48
blockHash: String!
59
extrinsicId: String!

‎services/dag-indexer/src/mappings/handleCall.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ export async function handleCall(_call: SubstrateExtrinsic): Promise<void> {
5656
const size = ipldNodeData?.size ?? BigInt(0)
5757
await Node.create({
5858
id: cid,
59+
cid,
60+
type: ipldNodeData.type,
61+
linkDepth: ipldNodeData.linkDepth,
62+
name: ipldNodeData.name,
5963
blockHeight,
6064
blockHash,
6165
extrinsicId,

‎services/file-retriever/__tests__/partialDownloads.spec.ts‎

Lines changed: 35 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,6 @@
1-
import {
2-
createFileChunkIpldNode,
3-
PBNode,
4-
cidToString,
5-
cidOfNode,
6-
createChunkedFileIpldNode,
7-
processFileToIPLDFormat,
8-
stringToCid,
9-
decodeNode,
10-
NODE_METADATA_SIZE,
11-
} from '@autonomys/auto-dag-data'
1+
import { MetadataType, IPLDNodeData } from '@autonomys/auto-dag-data'
122
import { dsnFetcher } from '../src/services/dsnFetcher.js'
133
import { jest } from '@jest/globals'
14-
import { randomBytes } from 'crypto'
15-
import { MemoryBlockstore } from 'blockstore-core'
164

175
describe('Partial downloads', () => {
186
beforeEach(() => {
@@ -23,93 +11,41 @@ describe('Partial downloads', () => {
2311
})
2412

2513
it('should return the correct chunk', async () => {
26-
const nodeMapByCID: Record<string, PBNode> = {}
27-
const chunks = ['a', 'b', 'c', 'd', 'e']
28-
29-
// Create the tree of nodes
30-
const nodes = chunks.map((chunk) =>
31-
createFileChunkIpldNode(Buffer.from(chunk, 'utf-8')),
14+
const chunks = [
15+
{
16+
cid: 'bafkr6idz7htrqhks6xyntulrqfyevx3k5qrhptbmgoxlmbss65yi3u25ym',
17+
links: [],
18+
type: MetadataType.File,
19+
linkDepth: 0,
20+
blockHeight: 0,
21+
blockHash: '',
22+
extrinsicId: '',
23+
extrinsicHash: '',
24+
indexInBlock: 0,
25+
blake3Hash: '',
26+
timestamp: new Date(),
27+
},
28+
]
29+
jest.spyOn(dsnFetcher, 'getFileChunks').mockResolvedValue(chunks)
30+
31+
const fetchNodeSpy = jest
32+
.spyOn(dsnFetcher, 'fetchNode')
33+
.mockImplementation(async (cid) => ({
34+
Data: IPLDNodeData.encode({
35+
type: MetadataType.File,
36+
data: Buffer.from(cid),
37+
}),
38+
Links: [],
39+
}))
40+
41+
const chunk = await dsnFetcher.getPartial(
42+
'bafkr6idz7htrqhks6xyntulrqfyevx3k5qrhptbmgoxlmbss65yi3u25ym',
43+
0,
3244
)
33-
const headNode = createChunkedFileIpldNode(
34-
nodes.map((node) => cidOfNode(node)),
35-
5n,
36-
1,
45+
expect(chunk).toEqual(Buffer.from(chunks[0].cid))
46+
expect(fetchNodeSpy).toHaveBeenCalledWith(
47+
'bafkr6idz7htrqhks6xyntulrqfyevx3k5qrhptbmgoxlmbss65yi3u25ym',
48+
chunks.map((e) => e.cid),
3749
)
38-
39-
// Create a map of the nodes by CID
40-
nodes.forEach((node) => {
41-
const cid = cidToString(cidOfNode(node))
42-
nodeMapByCID[cid] = node
43-
})
44-
nodeMapByCID[cidToString(cidOfNode(headNode))] = headNode
45-
46-
const cid = cidToString(cidOfNode(headNode))
47-
48-
jest.spyOn(dsnFetcher, 'fetchNode').mockImplementation(async (cid) => {
49-
return nodeMapByCID[cid]
50-
})
51-
52-
// First node (head) should return null
53-
const chunk = await dsnFetcher.getPartial(cid, 0)
54-
expect(chunk).toBeInstanceOf(Buffer)
55-
expect(chunk?.length).toBe(0)
56-
57-
// Chunks 1-5 should return the correct chunk
58-
let i = 0
59-
while (i < chunks.length) {
60-
// i + 1 for offsetting the head node
61-
const chunk = await dsnFetcher.getPartial(cid, i + 1)
62-
expect(chunk).toBeDefined()
63-
expect(chunk?.toString('utf-8')).toBe(chunks[i])
64-
i++
65-
}
66-
67-
// Chunk 6 should return empty buffer because it's past the end of the file
68-
const chunk6 = await dsnFetcher.getPartial(cid, 6)
69-
expect(chunk6).toBeNull()
70-
})
71-
72-
it('should work with a large file', async () => {
73-
const chunksCount = 30
74-
const maxLinkPerNode = 5
75-
const chunkSize = 50
76-
const totalSize = chunksCount * chunkSize
77-
78-
const chunks = Array.from({ length: chunksCount }).map(() =>
79-
randomBytes(chunkSize),
80-
)
81-
82-
const blockstore = new MemoryBlockstore()
83-
84-
const cid = cidToString(
85-
await processFileToIPLDFormat(
86-
blockstore,
87-
chunks,
88-
BigInt(totalSize),
89-
'dag-cbor',
90-
{
91-
maxLinkPerNode,
92-
maxNodeSize: chunkSize + NODE_METADATA_SIZE,
93-
},
94-
),
95-
)
96-
97-
jest.spyOn(dsnFetcher, 'fetchNode').mockImplementation(async (cid) => {
98-
return decodeNode(await blockstore.get(stringToCid(cid)))
99-
})
100-
101-
let index = 0,
102-
received = 0
103-
while (received < chunks.length) {
104-
const chunk = await dsnFetcher.getPartial(cid, index)
105-
index++
106-
if (chunk && chunk?.length > 0) {
107-
expect(chunk.toString('hex')).toBe(chunks[received].toString('hex'))
108-
received++
109-
}
110-
}
111-
112-
const finalResult = await dsnFetcher.getPartial(cid, index)
113-
expect(finalResult).toBeNull()
11450
})
11551
})

‎services/file-retriever/src/http/controllers/file.ts‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { asyncSafeHandler } from '../../utils/express.js'
77
import { uniqueHeaderValue } from '../../utils/http.js'
88
import { HttpError } from '../middlewares/error.js'
99
import { dsnFetcher } from '../../services/dsnFetcher.js'
10-
import { isValidCID, safeIPLDDecode } from '../../utils/dagData.js'
10+
import { isValidCID } from '../../utils/dagData.js'
1111
import { fileCache } from '../../services/cache.js'
1212

1313
const fileRouter = Router()
@@ -22,17 +22,8 @@ fileRouter.get(
2222
throw new HttpError(400, 'Invalid CID')
2323
}
2424

25-
const file = await dsnFetcher.fetchNode(cid, [])
26-
if (file) {
27-
const metadata = safeIPLDDecode(file)
28-
29-
res.status(200).json({
30-
...metadata,
31-
size: metadata?.size?.toString(10),
32-
})
33-
} else {
34-
res.sendStatus(404)
35-
}
25+
const metadata = await dsnFetcher.fetchNodeMetadata(cid)
26+
res.status(200).json(metadata)
3627
}),
3728
)
3829

0 commit comments

Comments
 (0)