S3/ Minio integration

.env.example

@@ -30,7 +30,7 @@ CLOUDFLARE_REGION="auto"
 #EMAIL_FROM_NAME=""
 #DISABLE_REGISTRATION=false
 
-# Where will social media icons be saved - local or cloudflare.
+# Where will social media icons be saved - local, cloudflare, or s3.
 STORAGE_PROVIDER="local"
 
 # Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
@@ -39,6 +39,13 @@ STORAGE_PROVIDER="local"
 # Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
 #NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY=""
 
+# S3/MinIO Storage (use STORAGE_PROVIDER="s3")
+#S3_ENDPOINT=""                    # Only for MinIO (e.g., "https://minio.example.com")
+#S3_ACCESS_KEY=""
+#S3_SECRET_KEY=""
+#S3_BUCKET=""
+#S3_REGION="us-east-1"
+
 # Social Media API Settings
 X_API_KEY=""
 X_API_SECRET=""
@@ -117,4 +124,4 @@ POSTIZ_OAUTH_CLIENT_SECRET=""
 
 # LINK_DRIP_API_KEY="" # Your LinkDrip API key
 # LINK_DRIP_API_ENDPOINT="https://api.linkdrip.com/v1/" # Your self-hosted LinkDrip API endpoint
-# LINK_DRIP_SHORT_LINK_DOMAIN="dripl.ink" # Your self-hosted LinkDrip domain
+# LINK_DRIP_SHORT_LINK_DOMAIN="dripl.ink" # Your self-hosted LinkDrip domain

apps/backend/src/api/routes/media.controller.ts

@@ -18,22 +18,34 @@ import { Organization } from '@prisma/client';
 import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';
 import { ApiTags } from '@nestjs/swagger';
 import handleR2Upload from '@gitroom/nestjs-libraries/upload/r2.uploader';
+import {
+  S3Uploader,
+  createS3Uploader,
+} from '@gitroom/nestjs-libraries/upload/s3.uploader';
 import { FileInterceptor } from '@nestjs/platform-express';
 import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation';
 import { SubscriptionService } from '@gitroom/nestjs-libraries/database/prisma/subscriptions/subscription.service';
 import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 import { SaveMediaInformationDto } from '@gitroom/nestjs-libraries/dtos/media/save.media.information.dto';
 import { VideoDto } from '@gitroom/nestjs-libraries/dtos/videos/video.dto';
 import { VideoFunctionDto } from '@gitroom/nestjs-libraries/dtos/videos/video.function.dto';
+import { getCurrentBucketUrl } from '@gitroom/nestjs-libraries/upload/s3.utils';
 
 @ApiTags('Media')
 @Controller('/media')
 export class MediaController {
   private storage = UploadFactory.createStorage();
+  private s3Uploader: S3Uploader | null = null;
+
   constructor(
     private _mediaService: MediaService,
     private _subscriptionService: SubscriptionService
-  ) {}
+  ) {
+    // Initialize S3 uploader if using S3 provider
+    if (process.env.STORAGE_PROVIDER === 's3') {
+      this.s3Uploader = createS3Uploader();
+    }
+  }
 
   @Delete('/:id')
   deleteMedia(@GetOrgFromRequest() org: Organization, @Param('id') id: string) {
@@ -108,11 +120,9 @@ export class MediaController {
     if (!name) {
       return false;
     }
-    return this._mediaService.saveFile(
-      org.id,
-      name,
-      process.env.CLOUDFLARE_BUCKET_URL + '/' + name
-    );
+    // Use appropriate bucket URL based on storage provider
+    const bucketUrl = getCurrentBucketUrl();
+    return this._mediaService.saveFile(org.id, name, bucketUrl + '/' + name);
   }
 
   @Post('/information')
@@ -151,11 +161,27 @@ export class MediaController {
     @Res() res: Response,
     @Param('endpoint') endpoint: string
   ) {
-    const upload = await handleR2Upload(endpoint, req, res);
+    // Route to appropriate handler based on storage provider
+    let upload;
+    if (process.env.STORAGE_PROVIDER === 's3' && this.s3Uploader) {
+      upload = await this.s3Uploader.handleUpload(endpoint, req, res);
+    } else {
+      upload = await handleR2Upload(endpoint, req, res);
+    }
+
     if (endpoint !== 'complete-multipart-upload') {
       return upload;
     }
 
+    // Check if response was already sent (e.g., error handler sent it)
+    if (res.headersSent) {
+      return;
+    }
+
+    // @ts-ignore - upload is CompleteMultipartUploadCommandOutput here
+    if (!upload?.Location) {
+      return res.status(500).json({ error: 'Upload failed - no location returned' });
+    }
     // @ts-ignore
     const name = upload.Location.split('/').pop();
 

apps/frontend/src/app/(app)/layout.tsx

@@ -47,7 +47,7 @@ export default async function AppLayout({ children }: { children: ReactNode }) {
       >
         <VariableContextComponent
           storageProvider={
-            process.env.STORAGE_PROVIDER! as 'local' | 'cloudflare'
+            process.env.STORAGE_PROVIDER! as 'local' | 'cloudflare' | 's3'
           }
           environment={process.env.NODE_ENV!}
           backendUrl={process.env.NEXT_PUBLIC_BACKEND_URL!}

apps/frontend/src/app/(extension)/layout.tsx

@@ -25,7 +25,7 @@ export default async function AppLayout({ children }: { children: ReactNode }) {
         <VariableContextComponent
           language="en"
           storageProvider={
-            process.env.STORAGE_PROVIDER! as 'local' | 'cloudflare'
+            process.env.STORAGE_PROVIDER! as 'local' | 'cloudflare' | 's3'
           }
           environment={process.env.NODE_ENV!}
           backendUrl={process.env.NEXT_PUBLIC_BACKEND_URL!}

apps/frontend/src/components/media/new.uploader.tsx

@@ -192,8 +192,11 @@ export function useUppyUploader(props: {
     uppy2.on('file-added', (file) => {
       setLocked(true);
       uppy2.setFileMeta(file.id, {
-        useCloudflare: storageProvider === 'cloudflare' ? 'true' : 'false', // Example of adding a custom field
-        // Add more fields as needed
+        // Both cloudflare and s3 use multipart upload
+        useCloudflare:
+          storageProvider === 'cloudflare' || storageProvider === 's3'
+            ? 'true'
+            : 'false',
       });
     });
     uppy2.on('error', (result) => {

libraries/nestjs-libraries/src/database/prisma/integrations/integration.repository.ts

@@ -6,6 +6,7 @@ import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
 import { IntegrationTimeDto } from '@gitroom/nestjs-libraries/dtos/integrations/integration.time.dto';
 import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 import { PlugDto } from '@gitroom/nestjs-libraries/dtos/plugs/plug.dto';
+import { isStorageUrl } from '@gitroom/nestjs-libraries/upload/s3.utils';
 
 @Injectable()
 export class IntegrationRepository {
@@ -143,11 +144,7 @@ export class IntegrationRepository {
   }
 
   async updateIntegration(id: string, params: Partial<Integration>) {
-    if (
-      params.picture &&
-      (params.picture.indexOf(process.env.CLOUDFLARE_BUCKET_URL!) === -1 ||
-        params.picture.indexOf(process.env.FRONTEND_URL!) === -1)
-    ) {
+    if (params.picture && !isStorageUrl(params.picture)) {
       params.picture = await this.storage.uploadSimple(params.picture);
     }
 

libraries/nestjs-libraries/src/database/prisma/media/media.service.ts

@@ -25,6 +25,19 @@ export class MediaService {
   ) {}
 
   async deleteMedia(org: string, id: string) {
+    // Get the media record to get the file path
+    const media = await this._mediaRepository.getMediaById(id);
+
+    // Delete from storage if file path exists
+    if (media?.path) {
+      try {
+        await this.storage.removeFile(media.path);
+      } catch (err) {
+        console.error('Error removing file from storage:', err);
+        // Continue with soft delete even if storage removal fails
+      }
+    }
+
     return this._mediaRepository.deleteMedia(org, id);
   }
 

libraries/nestjs-libraries/src/upload/cloudflare.storage.ts

@@ -1,4 +1,4 @@
-import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
+import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
 import 'multer';
 import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
 import mime from 'mime-types';
@@ -9,15 +9,19 @@ import axios from 'axios';
 
 class CloudflareStorage implements IUploadProvider {
   private _client: S3Client;
+  private _uploadUrl: string;
 
   constructor(
     accountID: string,
     accessKey: string,
     secretKey: string,
     private region: string,
     private _bucketName: string,
-    private _uploadUrl: string
+    uploadUrl: string
   ) {
+    // Normalize URL to remove trailing slash
+    this._uploadUrl = uploadUrl.replace(/\/$/, '');
+
     this._client = new S3Client({
       endpoint: `https://${accountID}.r2.cloudflarestorage.com`,
       region,
@@ -62,7 +66,8 @@ class CloudflareStorage implements IUploadProvider {
     const contentType =
       loadImage?.headers?.get('content-type') ||
       loadImage?.headers?.get('Content-Type');
-    const extension = getExtension(contentType)!;
+    // Fallback to 'bin' if MIME type is unrecognized, or try to extract from URL
+    const extension = getExtension(contentType) || path.split('.').pop()?.split('?')[0] || 'bin';
     const id = makeId(10);
 
     const params = {
@@ -112,14 +117,15 @@ class CloudflareStorage implements IUploadProvider {
     }
   }
 
-  // Implement the removeFile method from IUploadProvider
   async removeFile(filePath: string): Promise<void> {
-    // const fileName = filePath.split('/').pop(); // Extract the filename from the path
-    // const command = new DeleteObjectCommand({
-    //   Bucket: this._bucketName,
-    //   Key: fileName,
-    // });
-    // await this._client.send(command);
+    const fileName = filePath.split('/').pop();
+    if (!fileName) return;
+
+    const command = new DeleteObjectCommand({
+      Bucket: this._bucketName,
+      Key: fileName,
+    });
+    await this._client.send(command);
   }
 }
 

libraries/nestjs-libraries/src/upload/local.storage.ts

@@ -13,7 +13,8 @@ export class LocalStorage implements IUploadProvider {
     const contentType =
       loadImage?.headers?.['content-type'] ||
       loadImage?.headers?.['Content-Type'];
-    const findExtension = mime.getExtension(contentType)!;
+    // Fallback to 'bin' if MIME type is unrecognized, or try to extract from URL
+    const findExtension = mime.getExtension(contentType) || path.split('.').pop()?.split('?')[0] || 'bin';
 
     const now = new Date();
     const year = now.getFullYear();
@@ -74,15 +75,44 @@ export class LocalStorage implements IUploadProvider {
   }
 
   async removeFile(filePath: string): Promise<void> {
-    // Logic to remove the file from the filesystem goes here
-    return new Promise((resolve, reject) => {
-      unlink(filePath, (err) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve();
-        }
+    // Convert URL to filesystem path
+    // Input: http://localhost:4200/uploads/2025/01/15/abc123.png
+    // Output: /upload-directory/2025/01/15/abc123.png
+    try {
+      const url = new URL(filePath);
+      const pathAfterUploads = url.pathname.replace(/^\/uploads/, '');
+      const fsPath = `${this.uploadDirectory}${pathAfterUploads}`;
+
+      return new Promise((resolve, reject) => {
+        unlink(fsPath, (err) => {
+          if (err) {
+            // Don't reject if file doesn't exist
+            if (err.code === 'ENOENT') {
+              console.log(`File not found (already deleted?): ${fsPath}`);
+              resolve();
+            } else {
+              console.error(`Error deleting file from local storage: ${fsPath}`, err);
+              reject(err);
+            }
+          } else {
+            console.log(`Successfully deleted file: ${fsPath}`);
+            resolve();
+          }
+        });
       });
-    });
+    } catch (parseError) {
+      console.error(`Error parsing file URL: ${filePath}`, parseError);
+      // If filePath is not a valid URL, try it as a direct path
+      return new Promise((resolve, reject) => {
+        unlink(filePath, (err) => {
+          if (err && err.code !== 'ENOENT') {
+            console.error(`Error deleting file: ${filePath}`, err);
+            reject(err);
+          } else {
+            resolve();
+          }
+        });
+      });
+    }
   }
 }

libraries/nestjs-libraries/src/upload/r2.uploader.ts

@@ -75,7 +75,8 @@ export async function simpleUpload(
   const command = new PutObjectCommand({ ...params });
   await R2.send(command);
 
-  return CLOUDFLARE_BUCKET_URL + '/' + randomFilename;
+  const bucketUrl = (CLOUDFLARE_BUCKET_URL || '').replace(/\/$/, '');
+  return `${bucketUrl}/${randomFilename}`;
 }
 
 export async function createMultipartUpload(req: Request, res: Response) {
@@ -159,24 +160,16 @@ export async function completeMultipartUpload(req: Request, res: Response) {
   const { key, uploadId, parts } = req.body;
 
   try {
-    const params = {
-      Bucket: CLOUDFLARE_BUCKETNAME,
-      Key: key,
-      UploadId: uploadId,
-      MultipartUpload: { Parts: parts },
-    };
-
     const command = new CompleteMultipartUploadCommand({
       Bucket: CLOUDFLARE_BUCKETNAME,
       Key: key,
       UploadId: uploadId,
       MultipartUpload: { Parts: parts },
     });
     const response = await R2.send(command);
-    response.Location =
-      process.env.CLOUDFLARE_BUCKET_URL +
-      '/' +
-      response?.Location?.split('/').at(-1);
+    // Use the key from request (reliable) instead of parsing Location (fragile)
+    const bucketUrl = (process.env.CLOUDFLARE_BUCKET_URL || '').replace(/\/$/, '');
+    response.Location = `${bucketUrl}/${key}`;
     return response;
   } catch (err) {
     console.log('Error', err);