Skip to content

Commit a56bc7e

Browse files
clostaocursoragentcarlos
authored
feat: add s3 support for auto-drive (#446)
* feat: add s3 schema migrations * feat: map aws s3 sdk auth * feat: implement S3 object mappings repository * refactor: update S3 object mappings to use uppercase schema and improve migration scripts * feat: add S3 package with DTOs * fix: improve API key extraction regex in S3 authentication handler * chore: add S3 object mappings export to repository index * feat: create S3 controller and core functionality * feat: add integration tests for AWS S3 SDK functionality * chore: update DEBUG_LEVEL in test environment from debug to info * Create S3 object mapping after completing multipart upload Co-authored-by: carlos <[email protected]> * test: add integration test for downloading an object from AWS S3 * refactor: remove unnecessary logging from S3 request handler * refactor: simplify S3 handler configuration and add HEAD method support * feat: add support for upload options via metadata field * fix: correct syntax in Content-Disposition header for download response * fix: update logging to reference mapping.key instead of mapping in S3 index * feat: add @auto-drive/s3 dependency to package.json * feat: add S3 build target to Makefile and update common dependencies * chore: update dependencies in package.json and yarn.lock to include latest AWS SDK packages * feat: add S3 controller route to download API and remove from frontend API * fix: update import path in S3 SDK integration test to use download API --------- Co-authored-by: Cursor Agent <[email protected]> Co-authored-by: carlos <[email protected]>
1 parent f23fed5 commit a56bc7e

File tree

29 files changed

+1766
-25
lines changed

29 files changed

+1766
-25
lines changed

Makefile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ backend: common
55
yarn backend build
66
models:
77
yarn models build
8+
s3:
9+
yarn s3 build
810
frontend:
911
yarn frontend build
1012
gateway: install
@@ -21,7 +23,7 @@ submodules:
2123
yarn auto-files-gateway install
2224
yarn auto-files-gateway build
2325

24-
common: install submodules models
26+
common: install submodules models s3
2527

2628
test: install
2729
yarn backend test
@@ -32,4 +34,4 @@ lint: install
3234
yarn auth lint
3335
yarn frontend lint
3436

35-
all: submodules models frontend gateway backend
37+
all: submodules models s3 frontend gateway backend

ansible/auto-drive-deployment.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141

4242
- name: Update image to backend container key BACKEND_IMAGE in folder '/{{ matched_group }}' image to {{ image_tag }}
4343
ansible.builtin.shell: |
44-
infisical secrets --projectId {{ infisical_project_id }} --path /{{ target_machines }} --env prod --token {{ login_cmd.stdout }} set BACKEND_IMAGE={{ image_tag }}
44+
infisical secrets --projectId {{ infisical_project_id }} --path /{{ matched_group }} --env prod --token {{ login_cmd.stdout }} set BACKEND_IMAGE={{ image_tag }}
4545
when: image_tag is defined and image_tag != ''
4646
register: hello_output
4747

backend/.env.test

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ AUTH_SERVICE_API_KEY=mock_value
66
AUTH_SERVICE_URL=https://example.com
77
CACHE_DIR='.test-cache'
88
FORBIDDEN_EXTENSIONS='exe,bin'
9-
DEBUG_LEVEL=error
9+
DEBUG_LEVEL=info
Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
import { dbMigration } from '../../utils/dbMigrate.js'
2+
import {
3+
CompleteMultipartUploadCommand,
4+
CreateMultipartUploadCommand,
5+
GetObjectCommand,
6+
HeadObjectCommand,
7+
PutObjectCommand,
8+
S3Client,
9+
UploadPartCommand,
10+
} from '@aws-sdk/client-s3'
11+
import {
12+
createMockUser,
13+
mockRabbitPublish,
14+
unmockMethods,
15+
} from '../../utils/mocks.js'
16+
import { jest } from '@jest/globals'
17+
import { AuthManager } from '../../../src/infrastructure/services/auth/index.js'
18+
import { config } from '../../../src/config.js'
19+
import { SubscriptionsUseCases } from '../../../dist/core/users/subscriptions.js'
20+
21+
describe('AWS S3 - SDK', () => {
22+
let s3Client: S3Client
23+
const user = createMockUser()
24+
const BASE_PATH = `http://localhost:${config.express.port}`
25+
const Bucket = `${BASE_PATH}/s3`
26+
27+
beforeAll(async () => {
28+
await dbMigration.up()
29+
30+
// Mock warnings to clean test logs
31+
const consoleWarn = global.console.warn
32+
global.console.warn = () => {}
33+
// Start the frontend server
34+
await import('../../../src/app/apis/download.js')
35+
global.console.warn = consoleWarn
36+
// Wait for the server to start
37+
await new Promise((resolve) => setTimeout(resolve, 500))
38+
39+
mockRabbitPublish()
40+
// onboard the user
41+
await SubscriptionsUseCases.getOrCreateSubscription(user)
42+
43+
// mock auth manager returning the mock user
44+
jest.spyOn(AuthManager, 'getUserFromAccessToken').mockResolvedValue(user)
45+
jest.spyOn(AuthManager, 'getUserFromPublicId').mockResolvedValue(user)
46+
47+
s3Client = new S3Client({
48+
region: 'us-east-1',
49+
credentials: {
50+
accessKeyId: 'e046e71c8dc3459c8da189e62418203a',
51+
secretAccessKey: '',
52+
},
53+
bucketEndpoint: true,
54+
})
55+
})
56+
57+
afterAll(async () => {
58+
unmockMethods()
59+
await dbMigration.down()
60+
})
61+
62+
it('should be healthy the server', async () => {
63+
const response = await fetch(
64+
`http://localhost:${config.express.port}/health`,
65+
)
66+
expect(response.status).toBe(204)
67+
})
68+
69+
const Key = 'test.txt'
70+
const Body = Buffer.from('Hello, world!')
71+
72+
it('should upload an object', async () => {
73+
const command = new PutObjectCommand({
74+
Bucket,
75+
Key: 'test.txt',
76+
Body,
77+
})
78+
const result = await s3Client.send(command)
79+
expect(result).toMatchObject({
80+
ETag: expect.any(String),
81+
})
82+
})
83+
84+
it('should download the object', async () => {
85+
const command = new GetObjectCommand({
86+
Bucket,
87+
Key,
88+
})
89+
90+
const result = await s3Client.send(command)
91+
92+
expect(result.Body).toBeDefined()
93+
expect(Buffer.from(await result.Body!.transformToByteArray())).toEqual(Body)
94+
})
95+
96+
it('should be able download first 10 bytes of the object', async () => {
97+
const command = new GetObjectCommand({
98+
Bucket,
99+
Key,
100+
Range: 'bytes 0-9',
101+
})
102+
103+
const result = await s3Client.send(command)
104+
105+
expect(result.Body).toBeDefined()
106+
expect(result.ContentRange).toBe('bytes 0-9/13')
107+
const body = await result.Body!.transformToByteArray()
108+
expect(body.length).toBe(10)
109+
expect(body).toMatchObject(Body.subarray(0, 10).buffer)
110+
})
111+
112+
const SecondKey = 'test2.txt'
113+
const SecondBody = Buffer.from('Hello, world!')
114+
115+
it('should be able to upload an object with multipart upload', async () => {
116+
const createCommand = new CreateMultipartUploadCommand({
117+
Bucket,
118+
Key: SecondKey,
119+
})
120+
121+
const result = await s3Client.send(createCommand)
122+
expect(result).toMatchObject({
123+
UploadId: expect.any(String),
124+
})
125+
126+
const uploadId = result.UploadId!
127+
128+
const uploadPartCommand = new UploadPartCommand({
129+
Bucket,
130+
Key: SecondKey,
131+
UploadId: uploadId,
132+
PartNumber: 1,
133+
Body: SecondBody,
134+
})
135+
136+
const partUploadResult = await s3Client.send(uploadPartCommand)
137+
expect(partUploadResult).toMatchObject({
138+
ETag: expect.any(String),
139+
})
140+
141+
const completeCommand = new CompleteMultipartUploadCommand({
142+
Bucket,
143+
Key: SecondKey,
144+
UploadId: uploadId,
145+
MultipartUpload: {
146+
Parts: [
147+
{
148+
ETag: partUploadResult.ETag!,
149+
PartNumber: 1,
150+
},
151+
],
152+
},
153+
})
154+
155+
const completeResult = await s3Client.send(completeCommand)
156+
expect(completeResult).toMatchObject({
157+
ETag: expect.any(String),
158+
})
159+
})
160+
161+
it('should be able to download the object', async () => {
162+
const command = new GetObjectCommand({
163+
Bucket,
164+
Key: SecondKey,
165+
})
166+
167+
const result = await s3Client.send(command)
168+
169+
expect(result.Body).toBeDefined()
170+
expect(Buffer.from(await result.Body!.transformToByteArray())).toEqual(
171+
SecondBody,
172+
)
173+
})
174+
175+
describe('Metadata handled correctly', () => {
176+
const ThirdKey = 'test3.txt'
177+
178+
it('should be able to upload an object with compression and encryption', async () => {
179+
const command = new PutObjectCommand({
180+
Bucket,
181+
Key: ThirdKey,
182+
Body,
183+
Metadata: {
184+
compression: 'ZLIB',
185+
encryption: 'AES_256_GCM',
186+
},
187+
})
188+
189+
const result = await s3Client.send(command)
190+
expect(result).toMatchObject({
191+
ETag: expect.any(String),
192+
})
193+
})
194+
195+
it('should be able to download the object with compression and encryption', async () => {
196+
const command = new HeadObjectCommand({
197+
Bucket,
198+
Key: ThirdKey,
199+
})
200+
201+
const result = await s3Client.send(command)
202+
203+
expect(result.Metadata?.compression).toBe('ZLIB')
204+
expect(result.Metadata?.encryption).toBe('AES_256_GCM')
205+
})
206+
})
207+
})
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
'use strict'
2+
3+
var dbm
4+
var type
5+
var seed
6+
var fs = require('fs')
7+
var path = require('path')
8+
var Promise
9+
10+
/**
11+
* We receive the dbmigrate dependency from dbmigrate initially.
12+
* This enables us to not have to rely on NODE_PATH.
13+
*/
14+
exports.setup = function (options, seedLink) {
15+
dbm = options.dbmigrate
16+
type = dbm.dataType
17+
seed = seedLink
18+
Promise = options.Promise
19+
}
20+
21+
exports.up = function (db) {
22+
var filePath = path.join(__dirname, 'sqls', '20250730145300-s3-layer-up.sql')
23+
return new Promise(function (resolve, reject) {
24+
fs.readFile(filePath, { encoding: 'utf-8' }, function (err, data) {
25+
if (err) return reject(err)
26+
27+
resolve(data)
28+
})
29+
}).then(function (data) {
30+
return db.runSql(data)
31+
})
32+
}
33+
34+
exports.down = function (db) {
35+
var filePath = path.join(
36+
__dirname,
37+
'sqls',
38+
'20250730145300-s3-layer-down.sql',
39+
)
40+
return new Promise(function (resolve, reject) {
41+
fs.readFile(filePath, { encoding: 'utf-8' }, function (err, data) {
42+
if (err) return reject(err)
43+
44+
resolve(data)
45+
})
46+
}).then(function (data) {
47+
return db.runSql(data)
48+
})
49+
}
50+
51+
exports._meta = {
52+
version: 1,
53+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
drop trigger if exists set_timestamp on "S3".object_mappings;
2+
drop table if exists "S3".object_mappings;
3+
drop schema if exists "S3";
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
CREATE SCHEMA IF NOT EXISTS "S3";
2+
3+
CREATE TABLE "S3".object_mappings (
4+
"key" text NOT NULL PRIMARY KEY,
5+
cid text NOT NULL,
6+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
7+
updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP
8+
);
9+
10+
CREATE INDEX idx_s3_mappings_cid ON "S3".object_mappings (cid);
11+
12+
CREATE TRIGGER set_timestamp
13+
BEFORE UPDATE
14+
ON "S3".object_mappings
15+
FOR EACH ROW
16+
EXECUTE FUNCTION trigger_set_timestamp();

backend/package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
},
2020
"dependencies": {
2121
"@auto-drive/models": "workspace:*",
22+
"@auto-drive/s3": "workspace:*",
2223
"@auto-files/rpc-apis": "workspace:*",
2324
"@autonomys/asynchronous": "^1.5.12",
2425
"@autonomys/auto-dag-data": "^1.5.12",
@@ -41,6 +42,7 @@
4142
"debug-level": "^4.1.1",
4243
"dotenv": "^16.4.5",
4344
"express": "^4.19.2",
45+
"js2xmlparser": "^5.0.0",
4446
"jsonwebtoken": "^9.0.2",
4547
"keyv": "^5.3.1",
4648
"lru-cache": "^11.0.2",
@@ -57,6 +59,7 @@
5759
"zod": "^3.23.8"
5860
},
5961
"devDependencies": {
62+
"@aws-sdk/client-s3": "^3.859.0",
6063
"@faker-js/faker": "^9.2.0",
6164
"@ninoseki/eslint-plugin-neverthrow": "^0.0.1",
6265
"@swc/core": "^1.9.2",

backend/src/app/apis/download.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { handleAuth } from '../../infrastructure/services/auth/express.js'
66
import { config } from '../../config.js'
77
import { createLogger } from '../../infrastructure/drivers/logger.js'
88
import { downloadController } from '../controllers/download.js'
9+
import { s3Controller } from '../controllers/s3/http.js'
910

1011
const logger = createLogger('api:download')
1112

@@ -49,6 +50,8 @@ const createServer = async () => {
4950
}
5051

5152
app.use('/downloads', downloadController)
53+
app.use('/s3', s3Controller)
54+
5255
logger.debug('Download controller mounted at /downloads')
5356

5457
app.get('/health', (_req: Request, res: Response) => {

0 commit comments

Comments
 (0)