Skip to content

Commit 600d024

Browse files
ffarallffarall
authored
fix: 🐛 Account for replication target in batchStorageRequests testing helper (#662)
- Make batchStorageRequests replica-aware by switching from `bspApi` → `bspApis[]`, validating `replicationTarget`, waiting for the expected number of volunteer/confirm events, and verifying storage/forest presence per BSP replica. - Harden fisherman/BSP integration tests by adding missing `await`s, waiting for indexing/confirmations where races occurred, and adding extra sanity checks in the deletion catch-up scenarios. - Speed up CI by sharding the “Integration Tests: Fisherman” workflow job into a 2-shard matrix (with fail-fast: false).
1 parent a7b30f7 commit 600d024

17 files changed

+378
-93
lines changed

‎.github/workflows/network.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,10 @@ jobs:
860860
name: "Integration Tests: Fisherman"
861861
runs-on: blacksmith-4vcpu-ubuntu-2404
862862
timeout-minutes: 30
863+
strategy:
864+
fail-fast: false
865+
matrix:
866+
shard: [1, 2]
863867
steps:
864868
- uses: actions/checkout@v4
865869
- uses: ./.github/workflow-templates/setup-pnpm
@@ -904,6 +908,7 @@ jobs:
904908
--test-reporter=spec \
905909
--test-reporter-destination=stdout \
906910
--test-concurrency=1 \
911+
--test-shard=${{ matrix.shard }}/2 \
907912
./suites/integration/fisherman/**.test.ts
908913
909914
- name: Collect Docker logs on failure

‎.github/workflows/sdk.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ jobs:
278278
pnpm dlx create-next-app@latest /tmp/hello_world_nextjs --yes --ts --eslint --app --tailwind false --use-pnpm
279279
- name: Install SDK packages
280280
working-directory: /tmp/hello_world_nextjs
281-
run: pnpm add -w @storagehub-sdk/core @storagehub-sdk/msp-client
281+
run: pnpm add @storagehub-sdk/core @storagehub-sdk/msp-client
282282
- name: Smoke build
283283
working-directory: /tmp/hello_world_nextjs
284284
env:

‎test/scripts/generateFileSystemBenchmarkProofs.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
import { BspNetTestApi, waitFor } from "../util";
2-
import * as ShConsts from "../util/bspNet/consts";
3-
import { NetworkLauncher, type NetLaunchConfig } from "../util/netLaunch";
4-
import * as fs from "node:fs";
51
import { exec } from "node:child_process";
2+
import * as fs from "node:fs";
63
import type { Option } from "@polkadot/types";
74
import type { H256 } from "@polkadot/types/interfaces";
5+
import { BspNetTestApi, waitFor } from "../util";
6+
import * as ShConsts from "../util/bspNet/consts";
87
import type { EnrichedBspApi } from "../util/bspNet/test-api";
8+
import { type NetLaunchConfig, NetworkLauncher } from "../util/netLaunch";
99

1010
//! Configuration options for debugging
1111
const skipProofGeneration = false;
@@ -193,7 +193,7 @@ async function generateBenchmarkProofs() {
193193
const msp1BatchStorageRequestsResult = await userApi.file.batchStorageRequests({
194194
files: filesToUploadBatch1,
195195
owner: userApi.accounts.shUser,
196-
bspApi: bspApi,
196+
bspApis: [bspApi],
197197
mspId: ShConsts.DUMMY_MSP_ID,
198198
mspApi: msp1Api
199199
});
@@ -237,7 +237,7 @@ async function generateBenchmarkProofs() {
237237
const msp2BatchStorageRequestsResult = await userApi.file.batchStorageRequests({
238238
files: filesToUploadBatch2,
239239
owner: userApi.accounts.shUser,
240-
bspApi: bspApi,
240+
bspApis: undefined, // Intentionally skip BSP checks; replicationTarget keeps request incomplete
241241
mspId: ShConsts.DUMMY_MSP_ID_2,
242242
mspApi: msp2Api
243243
});

‎test/suites/integration/bsp/bsp-proof-retry.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import assert from "node:assert";
2-
import { u8aToHex } from "@polkadot/util";
3-
import { decodeAddress } from "@polkadot/util-crypto";
42
import type { ApiPromise } from "@polkadot/api";
53
import type { EventRecord, SignedBlock } from "@polkadot/types/interfaces";
4+
import { u8aToHex } from "@polkadot/util";
5+
import { decodeAddress } from "@polkadot/util-crypto";
66
import { describeMspNet, type EnrichedBspApi, shUser, waitFor } from "../../../util";
77

88
/**
@@ -112,7 +112,7 @@ await describeMspNet(
112112
mspId,
113113
valuePropId,
114114
owner: shUser,
115-
bspApi,
115+
bspApis: [bspApi],
116116
mspApi
117117
});
118118

‎test/suites/integration/bsp/reorg-proof.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ await describeBspNet(
261261
ignoredBlockHash = blockReceipt.blockHash.toString();
262262

263263
// Check that the BSP confirm storing extrinsic is successfully included in the block.
264-
userApi.assert.eventPresent("fileSystem", "BspConfirmedStoring", events);
264+
await userApi.assert.eventPresent("fileSystem", "BspConfirmedStoring", events);
265265

266266
// Check that the BSP root has not changed.
267267
// We check for 3 seconds expecting to have no change, i.e. expecting the check in the
@@ -410,7 +410,7 @@ await describeBspNet(
410410
await userApi.block.seal({ finaliseBlock: false });
411411

412412
// Check that the BSP confirm storing extrinsic is successfully included in the block.
413-
userApi.assert.eventPresent("fileSystem", "BspConfirmedStoring");
413+
await userApi.assert.eventPresent("fileSystem", "BspConfirmedStoring");
414414

415415
// Wait for confirmation line in docker logs.
416416
await bspApi.docker.waitForLog({

‎test/suites/integration/fisherman/batch-file-deletion-catchup.test.ts‎

Lines changed: 116 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ await describeMspNet(
137137
mspId,
138138
valuePropId,
139139
owner: shUser,
140-
bspApi,
140+
bspApis: [bspApi],
141141
mspApi: msp1Api
142142
});
143143

@@ -262,7 +262,7 @@ await describeMspNet(
262262
mspId,
263263
valuePropId,
264264
owner: shUser,
265-
bspApi,
265+
bspApis: [bspApi],
266266
mspApi: msp1Api,
267267
finaliseBlock: false
268268
});
@@ -422,7 +422,11 @@ await describeMspNet(
422422
});
423423

424424
// Verify BspFileDeletionsCompleted event
425-
userApi.assert.eventPresent("fileSystem", "BspFileDeletionsCompleted", deletionResult.events);
425+
await userApi.assert.eventPresent(
426+
"fileSystem",
427+
"BspFileDeletionsCompleted",
428+
deletionResult.events
429+
);
426430

427431
// Verify BucketFileDeletionsCompleted events (one per bucket)
428432
const bucketDeletionEvents = (deletionResult.events || []).filter((record) =>
@@ -619,12 +623,63 @@ await describeMspNet(
619623
mspId,
620624
valuePropId,
621625
owner: shUser,
622-
bspApi,
626+
bspApis: undefined, // Intentionally skip BSP checks; replicationTarget keeps request incomplete
623627
mspApi: msp1Api
624628
});
625629

626630
const { fileKeys, bucketIds } = batchResult;
627631

632+
// Ensure the BSP confirms to store all files before continuing
633+
// Due to race conditions, the BSP confirmations might come in multiple blocks, so we need to wait
634+
// for all confirmations to complete.
635+
const bspAddress = userApi.createType("Address", bspApi.accounts.bspKey.address);
636+
let stillConfirming = true;
637+
while (stillConfirming) {
638+
try {
639+
await userApi.wait.bspStored({
640+
expectedExts: 1,
641+
bspAccount: bspAddress,
642+
timeoutMs: 6000
643+
});
644+
} catch (_) {
645+
stillConfirming = false;
646+
}
647+
}
648+
649+
// Sanity check: this test assumes the (single) BSP is actually storing these files,
650+
// otherwise fisherman won't be able to batch a BSP-side deletion for "Incomplete" requests.
651+
for (const [index, fileKey] of fileKeys.entries()) {
652+
try {
653+
await waitFor({
654+
lambda: async () => {
655+
const bspFileStorageResult =
656+
await bspApi.rpc.storagehubclient.isFileInFileStorage(fileKey);
657+
return bspFileStorageResult.isFileFound;
658+
}
659+
});
660+
} catch (error) {
661+
throw new Error(
662+
`BSP has not stored file in file storage: ${fileKey} at index ${index}: ${error}`
663+
);
664+
}
665+
666+
try {
667+
await waitFor({
668+
lambda: async () => {
669+
const bspForestResult = await bspApi.rpc.storagehubclient.isFileInForest(
670+
null,
671+
fileKey
672+
);
673+
return bspForestResult.isTrue;
674+
}
675+
});
676+
} catch (error) {
677+
throw new Error(
678+
`BSP is not storing file in forest: ${fileKey} at index ${index}: ${error}`
679+
);
680+
}
681+
}
682+
628683
// Store finalized incomplete storage data for verification in test 6
629684
finalizedIncompleteFileKeys = fileKeys;
630685
finalizedIncompleteBucketIds = bucketIds;
@@ -721,14 +776,65 @@ await describeMspNet(
721776
mspId,
722777
valuePropId,
723778
owner: shUser,
724-
bspApi,
779+
bspApis: undefined, // Intentionally skip BSP checks; replicationTarget keeps request incomplete
725780
mspApi: msp1Api,
726781
finaliseBlock: false
727782
});
728783

729784
const { fileKeys: unfinalizedFileKeys, bucketIds: unfinalizedBucketIds } =
730785
unfinalizedBatchResult;
731786

787+
// Ensure the BSP confirms to store all files before continuing
788+
// Due to race conditions, the BSP confirmations might come in multiple blocks, so we need to wait
789+
// for all confirmations to complete.
790+
const bspAddress = userApi.createType("Address", bspApi.accounts.bspKey.address);
791+
let stillConfirming = true;
792+
while (stillConfirming) {
793+
try {
794+
await userApi.wait.bspStored({
795+
expectedExts: 1,
796+
bspAccount: bspAddress,
797+
timeoutMs: 6000
798+
});
799+
} catch (_) {
800+
stillConfirming = false;
801+
}
802+
}
803+
804+
// Sanity check: this test assumes the (single) BSP is actually storing these files,
805+
// otherwise the manual BSP deletion (and later fisherman behaviour) is not meaningful.
806+
for (const [index, fileKey] of unfinalizedFileKeys.entries()) {
807+
try {
808+
await waitFor({
809+
lambda: async () => {
810+
const bspFileStorageResult =
811+
await bspApi.rpc.storagehubclient.isFileInFileStorage(fileKey);
812+
return bspFileStorageResult.isFileFound;
813+
}
814+
});
815+
} catch (error) {
816+
throw new Error(
817+
`BSP has not stored file in file storage: ${fileKey} at index ${index}: ${error}`
818+
);
819+
}
820+
821+
try {
822+
await waitFor({
823+
lambda: async () => {
824+
const bspForestResult = await bspApi.rpc.storagehubclient.isFileInForest(
825+
null,
826+
fileKey
827+
);
828+
return bspForestResult.isTrue;
829+
}
830+
});
831+
} catch (error) {
832+
throw new Error(
833+
`BSP is not storing file in forest: ${fileKey} at index ${index}: ${error}`
834+
);
835+
}
836+
}
837+
732838
// Store unfinalized incomplete storage data for verification in test 6
733839
unfinalizedIncompleteFileKeys = unfinalizedFileKeys;
734840
unfinalizedIncompleteBucketIds = unfinalizedBucketIds;
@@ -799,7 +905,11 @@ await describeMspNet(
799905
});
800906

801907
// Verify BspFileDeletionsCompleted event
802-
userApi.assert.eventPresent("fileSystem", "BspFileDeletionsCompleted", deletionResult.events);
908+
await userApi.assert.eventPresent(
909+
"fileSystem",
910+
"BspFileDeletionsCompleted",
911+
deletionResult.events
912+
);
803913

804914
// Verify BucketFileDeletionsCompleted events (one per bucket)
805915
const bucketDeletionEvents = (deletionResult.events || []).filter((record) =>

‎test/suites/integration/fisherman/batch-file-deletion.test.ts‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
import assert from "node:assert";
2-
import { describeMspNet, type EnrichedBspApi, type SqlClient, shUser } from "../../../util";
2+
import {
3+
describeMspNet,
4+
type EnrichedBspApi,
5+
type SqlClient,
6+
shUser,
7+
waitFor
8+
} from "../../../util";
39

410
/**
511
* FISHERMAN BATCH FILE DELETION INTEGRATION TESTS
@@ -129,7 +135,7 @@ await describeMspNet(
129135
mspId,
130136
valuePropId,
131137
owner: shUser,
132-
bspApi,
138+
bspApis: [bspApi],
133139
mspApi: msp1Api
134140
});
135141

@@ -257,12 +263,63 @@ await describeMspNet(
257263
mspId,
258264
valuePropId,
259265
owner: shUser,
260-
bspApi,
266+
bspApis: undefined, // Intentionally skip BSP checks; replicationTarget keeps request incomplete
261267
mspApi: msp1Api
262268
});
263269

264270
const { fileKeys } = batchResult;
265271

272+
// Ensure the BSP confirms to store all files before continuing
273+
// Due to race conditions, the BSP confirmations might come in multiple blocks, so we need to wait
274+
// for all confirmations to complete.
275+
const bspAddress = userApi.createType("Address", bspApi.accounts.bspKey.address);
276+
let stillConfirming = true;
277+
while (stillConfirming) {
278+
try {
279+
await userApi.wait.bspStored({
280+
expectedExts: 1,
281+
bspAccount: bspAddress,
282+
timeoutMs: 6000
283+
});
284+
} catch (_) {
285+
stillConfirming = false;
286+
}
287+
}
288+
289+
// Sanity check: this test assumes the (single) BSP is actually storing these files,
290+
// otherwise fisherman won't be able to batch a BSP-side deletion for "Incomplete" requests.
291+
for (const [index, fileKey] of fileKeys.entries()) {
292+
try {
293+
await waitFor({
294+
lambda: async () => {
295+
const bspFileStorageResult =
296+
await bspApi.rpc.storagehubclient.isFileInFileStorage(fileKey);
297+
return bspFileStorageResult.isFileFound;
298+
}
299+
});
300+
} catch (error) {
301+
throw new Error(
302+
`BSP has not stored file in file storage: ${fileKey} at index ${index}: ${error}`
303+
);
304+
}
305+
306+
try {
307+
await waitFor({
308+
lambda: async () => {
309+
const bspForestResult = await bspApi.rpc.storagehubclient.isFileInForest(
310+
null,
311+
fileKey
312+
);
313+
return bspForestResult.isTrue;
314+
}
315+
});
316+
} catch (error) {
317+
throw new Error(
318+
`BSP is not storing file in forest: ${fileKey} at index ${index}: ${error}`
319+
);
320+
}
321+
}
322+
266323
await indexerApi.indexer.waitForIndexing({ producerApi: userApi, sql });
267324

268325
// Build all revocation calls

‎test/suites/integration/fisherman/files-deleted-based-on-is-in-bucket.test.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
import assert from "node:assert";
22
import {
3+
bspTwoKey,
34
describeMspNet,
45
type EnrichedBspApi,
5-
type SqlClient,
6-
shUser,
7-
mspKey,
86
hexToBuffer,
9-
bspTwoKey,
10-
ShConsts
7+
mspKey,
8+
ShConsts,
9+
type SqlClient,
10+
shUser
1111
} from "../../../util";
1212

1313
/**
@@ -141,7 +141,7 @@ await describeMspNet(
141141
mspId,
142142
valuePropId,
143143
owner: shUser,
144-
bspApi,
144+
bspApis: [bspApi],
145145
mspApi: msp1Api
146146
});
147147

@@ -337,7 +337,7 @@ await describeMspNet(
337337
mspId,
338338
valuePropId,
339339
owner: shUser,
340-
bspApi,
340+
bspApis: [bspApi],
341341
// mspApi is intentionally not provided - MSP checks will be skipped
342342
maxAttempts: 5
343343
});

0 commit comments

Comments
 (0)