[NAE-1946] Remote file connector to S3

.github/workflows/master-build.yml

@@ -52,6 +52,13 @@ jobs:
           - 9300:9300
         options: -e="discovery.type=single-node" -e="xpack.security.enabled=false" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
+      minio:
+        image: docker.io/bitnami/minio:2022
+        ports:
+          - 9000:9000
+          - 9001:9001
+        options: -e="MINIO_ROOT_USER=root" -e="MINIO_ROOT_PASSWORD=password" -e="MINIO_DEFAULT_BUCKETS=default"
+
     steps:
       - name: Test Database
         env:

.github/workflows/pr-build.yml

@@ -51,6 +51,13 @@ jobs:
           - 9300:9300
         options: -e="discovery.type=single-node" -e="xpack.security.enabled=false" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
+      minio:
+        image: docker.io/bitnami/minio:2022
+        ports:
+          - 9000:9000
+          - 9001:9001
+        options: -e="MINIO_ROOT_USER=root" -e="MINIO_ROOT_PASSWORD=password" -e="MINIO_DEFAULT_BUCKETS=default"
+
     steps:
       - name: Test Database
         env:

.github/workflows/release-build.yml

@@ -63,6 +63,13 @@ jobs:
           - 9300:9300
         options: -e="discovery.type=single-node" -e="xpack.security.enabled=false" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
+      minio:
+        image: docker.io/bitnami/minio:2022
+        ports:
+          - 9000:9000
+          - 9001:9001
+        options: -e="MINIO_ROOT_USER=root" -e="MINIO_ROOT_PASSWORD=password" -e="MINIO_DEFAULT_BUCKETS=default"
+
     steps:
       - name: Test Database
         env:

docker-compose.yml

@@ -38,6 +38,26 @@ services:
     image: redis:6
     ports:
       - "6379:6379"
+  minio:
+    image: docker.io/bitnami/minio:2022
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    networks:
+      - minionetwork
+    volumes:
+      - 'minio_data:/data'
+    environment:
+      - MINIO_ROOT_USER=root
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DEFAULT_BUCKETS=default
+networks:
+  minionetwork:
+    driver: bridge
+
+volumes:
+  minio_data:
+    driver: local
 
 #  kibana:
 #    image: docker.elastic.co/kibana/kibana:7.17.4

pom.xml

@@ -526,6 +526,11 @@
             <artifactId>jackson-module-jsonSchema</artifactId>
             <version>2.13.2</version>
         </dependency>
+        <dependency>
+            <groupId>io.minio</groupId>
+            <artifactId>minio</artifactId>
+            <version>8.5.8</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileField.groovy

@@ -1,12 +1,12 @@
 package com.netgrif.application.engine.petrinet.domain.dataset
 
-
+import com.netgrif.application.engine.files.StorageType
 import org.springframework.data.mongodb.core.mapping.Document
 
 @Document
 class FileField extends Field<FileFieldValue> {
 
-    private Boolean remote
+    private StorageType storageType
 
     FileField() {
         super()
@@ -42,39 +42,19 @@ class FileField extends Field<FileFieldValue> {
         this.setDefaultValue(FileFieldValue.fromString(defaultValue))
     }
 
-    /**
-     * Get complete file path to the file
-     * Path is generated as follow:
-     * - if file is remote, path is field value / remote URI
-     * - if file is local
-     *    - saved file name consists of Case id, field import id and original file name separated by dash
-     * @param caseId
-     * @return path to the saved file
-     */
-    String getFilePath(String caseId) {
-        if (this.remote)
-            return this.getValue().getPath()
-        return this.getValue().getPath(caseId, getStringId())
-    }
-
-    String getFilePreviewPath(String caseId) {
-        return this.getValue().getPreviewPath(caseId, getStringId())
+    StorageType getStorageType() {
+        return this.storageType
     }
 
-    boolean isRemote() {
-        return this.remote
-    }
-
-    void setRemote(boolean remote) {
-        this.remote = remote
+    void setStorageType(StorageType storageType) {
+        this.storageType = storageType
     }
 
     @Override
     Field clone() {
         FileField clone = new FileField()
         super.clone(clone)
-        clone.remote = this.remote
-
+        clone.storageType = this.storageType
         return clone
     }
 }

src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileFieldValue.groovy

@@ -1,16 +1,15 @@
 package com.netgrif.application.engine.petrinet.domain.dataset
 
-import com.netgrif.application.engine.configuration.ApplicationContextProvider
-import com.netgrif.application.engine.workflow.domain.FileStorageConfiguration
-
 class FileFieldValue implements Serializable {
 
-    private static final long serialVersionUID = 1299918326436821185L;
+    private static final long serialVersionUID = 1299918326436821185L
 
     private String name
 
     private String path
 
+    private String previewPath
+
     FileFieldValue() {
     }
 
@@ -19,6 +18,12 @@ class FileFieldValue implements Serializable {
         this.path = path
     }
 
+    FileFieldValue(String name, String path, String previewPath) {
+        this.name = name
+        this.path = path
+        this.previewPath = previewPath
+    }
+
     static FileFieldValue fromString(String value) {
         if (!value.contains(":"))
             return new FileFieldValue(value, null)
@@ -39,21 +44,18 @@ class FileFieldValue implements Serializable {
         return path
     }
 
-    String getPath(String caseId, String fieldId) {
-        FileStorageConfiguration fileStorageConfiguration = ApplicationContextProvider.getBean("fileStorageConfiguration") as FileStorageConfiguration
-        return "${fileStorageConfiguration.getStoragePath()}/${caseId}-${fieldId}-${name}"
+    void setPath(String path) {
+        this.path = path
     }
 
-    String getPreviewPath(String caseId, String fieldId) {
-        FileStorageConfiguration fileStorageConfiguration = ApplicationContextProvider.getBean("fileStorageConfiguration") as FileStorageConfiguration
-        return "${fileStorageConfiguration.getStoragePath()}/file_preview/${caseId}-${fieldId}-${name}"
+    String getPreviewPath() {
+        return previewPath
     }
 
-    void setPath(String path) {
-        this.path = path
+    void setPreviewPath(String previewPath) {
+        this.previewPath = previewPath
     }
 
-
     @Override
     String toString() {
         return path

src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileListField.groovy

@@ -1,7 +1,9 @@
 package com.netgrif.application.engine.petrinet.domain.dataset
 
+import com.netgrif.application.engine.files.StorageType
+
 class FileListField extends Field<FileListFieldValue> {
-    private Boolean remote
+    private StorageType storageType
 
     FileListField() {
         super()
@@ -48,37 +50,20 @@ class FileListField extends Field<FileListFieldValue> {
         this.getValue().getNamesPaths().add(new FileFieldValue(fileName, path))
     }
 
-    /**
-     * Get complete file path to the file
-     * Path is generated as follow:
-     * - if file is remote, path is field value / remote URI
-     * - if file is local
-     *    - saved file path consists of Case id, slash field import id, slash original file name
-     * @param caseId
-     * @param name
-     * @return path to the saved file
-     */
-    String getFilePath(String caseId, String name) {
-        if (this.remote) {
-            FileFieldValue first = this.getValue().getNamesPaths().find({ namePath -> namePath.name == name })
-            return first != null ? first.path : null
-        }
-        return FileListFieldValue.getPath(caseId, getStringId(), name)
+    StorageType getStorageType() {
+        return storageType
     }
 
-    boolean isRemote() {
-        return this.remote
+    void setStorageType(StorageType storageType) {
+        this.storageType = storageType
     }
 
-    void setRemote(boolean remote) {
-        this.remote = remote
-    }
 
     @Override
     Field clone() {
         FileListField clone = new FileListField()
         super.clone(clone)
-        clone.remote = this.remote
+        clone.storageType = this.storageType
 
         return clone
     }

src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileListFieldValue.groovy

@@ -1,8 +1,5 @@
 package com.netgrif.application.engine.petrinet.domain.dataset
 
-import com.netgrif.application.engine.configuration.ApplicationContextProvider
-import com.netgrif.application.engine.workflow.domain.FileStorageConfiguration
-
 class FileListFieldValue implements Serializable {
 
     private static final long serialVersionUID = 5299918326436821185L;
@@ -39,11 +36,6 @@ class FileListFieldValue implements Serializable {
         return newVal
     }
 
-    static String getPath(String caseId, String fieldId, String name) {
-        FileStorageConfiguration fileStorageConfiguration = ApplicationContextProvider.getBean("fileStorageConfiguration")
-        return "${fileStorageConfiguration.getStoragePath()}/${caseId}/${fieldId}/${name}"
-    }
-
     @Override
     String toString() {
         return namesPaths.toString()

src/main/java/com/netgrif/application/engine/files/StorageResolverService.java

@@ -0,0 +1,36 @@
+package com.netgrif.application.engine.files;
+
+import com.netgrif.application.engine.files.interfaces.IStorageService;
+import com.netgrif.application.engine.files.throwable.StorageNotFoundException;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class StorageResolverService {
+
+    private List<IStorageService> storageServices;
+
+    @Autowired
+    private void setStorageServices(List<IStorageService> storageServices) {
+        this.storageServices = storageServices;
+    }
+
+    public IStorageService resolve(StorageType type) {
+        if (storageServices == null) {
+            log.error("Storage services with interface IStorageService not found.");
+            throw new StorageNotFoundException("Remote Storage not available.");
+        }
+        Optional<IStorageService> storageService = storageServices.stream().filter(service -> service.getType().equals(type)).collect(Collectors.toList()).stream().findFirst();
+        if (storageService.isPresent()) {
+            return storageService.get();
+        }
+        throw new StorageNotFoundException("Storage Service with type: " + type + " not available.");
+
+    }
+}

src/main/java/com/netgrif/application/engine/files/StorageType.java

@@ -0,0 +1,27 @@
+package com.netgrif.application.engine.files;
+
+import lombok.Getter;
+
+import java.util.Arrays;
+import java.util.Optional;
+
+@Getter
+public enum StorageType {
+
+    MINIO("MINIO"),
+    LOCAL("LOCAL");
+
+    private final String type;
+
+    StorageType(String type) {
+        this.type = type;
+    }
+
+    public static StorageType fromString(String value) {
+        Optional<StorageType> storageType = Arrays.stream(StorageType.values()).filter(it -> it.getType().equals(value)).findFirst();
+        if (storageType.isPresent()) {
+            return storageType.get();
+        }
+        throw new NullPointerException("Storage type with value: "+ value + " not found.");
+    }
+}

src/main/java/com/netgrif/application/engine/files/interfaces/IStorageService.java

@@ -0,0 +1,25 @@
+package com.netgrif.application.engine.files.interfaces;
+
+import com.netgrif.application.engine.files.StorageType;
+import com.netgrif.application.engine.files.throwable.BadRequestException;
+import com.netgrif.application.engine.files.throwable.ServiceErrorException;
+import com.netgrif.application.engine.files.throwable.StorageException;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.InputStream;
+
+public interface IStorageService {
+    StorageType getType();
+
+    InputStream get(String path) throws BadRequestException, ServiceErrorException;
+
+    boolean save(String path, MultipartFile file) throws StorageException;
+
+    boolean save(String path, InputStream stream) throws StorageException;
+
+    void delete(String path) throws StorageException;
+
+    String getPreviewPath(String caseId, String fieldId, String name);
+
+    String getPath(String caseId, String fieldId, String name);
+}