Merge pull request #243 from netgrif/NAE-1946

[NAE-1946] Remote file connector to S3
netgrif · Sep 25, 2024 · 457776a · 457776a
2 parents d12eefa + e613d76
commit 457776a
Show file tree

Hide file tree

Showing 40 changed files with 952 additions and 329 deletions.
diff --git a/.github/workflows/master-build.yml b/.github/workflows/master-build.yml
@@ -52,6 +52,13 @@ jobs:
           - 9300:9300
         options: -e="discovery.type=single-node" -e="xpack.security.enabled=false" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
+      minio:
+        image: docker.io/bitnami/minio:2022
+        ports:
+          - 9000:9000
+          - 9001:9001
+        options: -e="MINIO_ROOT_USER=root" -e="MINIO_ROOT_PASSWORD=password" -e="MINIO_DEFAULT_BUCKETS=default"
+
     steps:
       - name: Test Database
         env:

diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml
@@ -51,6 +51,13 @@ jobs:
           - 9300:9300
         options: -e="discovery.type=single-node" -e="xpack.security.enabled=false" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
+      minio:
+        image: docker.io/bitnami/minio:2022
+        ports:
+          - 9000:9000
+          - 9001:9001
+        options: -e="MINIO_ROOT_USER=root" -e="MINIO_ROOT_PASSWORD=password" -e="MINIO_DEFAULT_BUCKETS=default"
+
     steps:
       - name: Test Database
         env:

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
@@ -63,6 +63,13 @@ jobs:
           - 9300:9300
         options: -e="discovery.type=single-node" -e="xpack.security.enabled=false" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
+      minio:
+        image: docker.io/bitnami/minio:2022
+        ports:
+          - 9000:9000
+          - 9001:9001
+        options: -e="MINIO_ROOT_USER=root" -e="MINIO_ROOT_PASSWORD=password" -e="MINIO_DEFAULT_BUCKETS=default"
+
     steps:
       - name: Test Database
         env:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -38,6 +38,26 @@ services:
     image: redis:6
     ports:
       - "6379:6379"
+  minio:
+    image: docker.io/bitnami/minio:2022
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    networks:
+      - minionetwork
+    volumes:
+      - 'minio_data:/data'
+    environment:
+      - MINIO_ROOT_USER=root
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DEFAULT_BUCKETS=default
+networks:
+  minionetwork:
+    driver: bridge
+
+volumes:
+  minio_data:
+    driver: local
 
 #  kibana:
 #    image: docker.elastic.co/kibana/kibana:7.17.4

diff --git a/pom.xml b/pom.xml
@@ -525,6 +525,11 @@
             <artifactId>jackson-module-jsonSchema</artifactId>
             <version>2.15.0-rc1</version>
         </dependency>
+        <dependency>
+            <groupId>io.minio</groupId>
+            <artifactId>minio</artifactId>
+            <version>8.5.12</version>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileField.groovy b/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileField.groovy
@@ -4,9 +4,7 @@ package com.netgrif.application.engine.petrinet.domain.dataset
 import org.springframework.data.mongodb.core.mapping.Document
 
 @Document
-class FileField extends Field<FileFieldValue> {
-
-    private Boolean remote
+class FileField extends StorageField<FileFieldValue> {
 
     FileField() {
         super()
@@ -42,39 +40,11 @@ class FileField extends Field<FileFieldValue> {
         this.setDefaultValue(FileFieldValue.fromString(defaultValue))
     }
 
-    /**
-     * Get complete file path to the file
-     * Path is generated as follow:
-     * - if file is remote, path is field value / remote URI
-     * - if file is local
-     *    - saved file name consists of Case id, field import id and original file name separated by dash
-     * @param caseId
-     * @return path to the saved file
-     */
-    String getFilePath(String caseId) {
-        if (this.remote)
-            return this.getValue().getPath()
-        return this.getValue().getPath(caseId, getStringId())
-    }
-
-    String getFilePreviewPath(String caseId) {
-        return this.getValue().getPreviewPath(caseId, getStringId())
-    }
-
-    boolean isRemote() {
-        return this.remote
-    }
-
-    void setRemote(boolean remote) {
-        this.remote = remote
-    }
-
     @Override
     Field clone() {
         FileField clone = new FileField()
         super.clone(clone)
-        clone.remote = this.remote
-
+        clone.storage = this.storage
         return clone
     }
 }
diff --git a/...in/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileFieldDataType.groovy b/...in/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileFieldDataType.groovy
@@ -18,5 +18,11 @@ enum FileFieldDataType {
                 return item
             }
         }
+        return null
+    }
+
+    static FileFieldDataType resolveTypeFromName(String name) {
+        int dot = name.lastIndexOf(".")
+        return resolveType((dot == -1) ? "" : name.substring(dot + 1))
     }
 }
diff --git a/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileFieldValue.groovy b/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileFieldValue.groovy
@@ -1,16 +1,15 @@
 package com.netgrif.application.engine.petrinet.domain.dataset
 
-import com.netgrif.application.engine.configuration.ApplicationContextProvider
-import com.netgrif.application.engine.workflow.domain.FileStorageConfiguration
-
 class FileFieldValue implements Serializable {
 
-    private static final long serialVersionUID = 1299918326436821185L;
+    private static final long serialVersionUID = 1299918326436821185L
 
     private String name
 
     private String path
 
+    private String previewPath
+
     FileFieldValue() {
     }
 
@@ -19,6 +18,12 @@ class FileFieldValue implements Serializable {
         this.path = path
     }
 
+    FileFieldValue(String name, String path, String previewPath) {
+        this.name = name
+        this.path = path
+        this.previewPath = previewPath
+    }
+
     static FileFieldValue fromString(String value) {
         if (!value.contains(":"))
             return new FileFieldValue(value, null)
@@ -39,21 +44,18 @@ class FileFieldValue implements Serializable {
         return path
     }
 
-    String getPath(String caseId, String fieldId) {
-        FileStorageConfiguration fileStorageConfiguration = ApplicationContextProvider.getBean("fileStorageConfiguration") as FileStorageConfiguration
-        return "${fileStorageConfiguration.getStoragePath()}/${caseId}-${fieldId}-${name}"
+    void setPath(String path) {
+        this.path = path
     }
 
-    String getPreviewPath(String caseId, String fieldId) {
-        FileStorageConfiguration fileStorageConfiguration = ApplicationContextProvider.getBean("fileStorageConfiguration") as FileStorageConfiguration
-        return "${fileStorageConfiguration.getStoragePath()}/file_preview/${caseId}-${fieldId}-${name}"
+    String getPreviewPath() {
+        return previewPath
     }
 
-    void setPath(String path) {
-        this.path = path
+    void setPreviewPath(String previewPath) {
+        this.previewPath = previewPath
     }
 
-
     @Override
     String toString() {
         return path

diff --git a/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileListField.groovy b/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileListField.groovy
@@ -1,7 +1,6 @@
 package com.netgrif.application.engine.petrinet.domain.dataset
 
-class FileListField extends Field<FileListFieldValue> {
-    private Boolean remote
+class FileListField extends StorageField<FileListFieldValue> {
 
     FileListField() {
         super()
@@ -48,38 +47,11 @@ class FileListField extends Field<FileListFieldValue> {
         this.getValue().getNamesPaths().add(new FileFieldValue(fileName, path))
     }
 
-    /**
-     * Get complete file path to the file
-     * Path is generated as follow:
-     * - if file is remote, path is field value / remote URI
-     * - if file is local
-     *    - saved file path consists of Case id, slash field import id, slash original file name
-     * @param caseId
-     * @param name
-     * @return path to the saved file
-     */
-    String getFilePath(String caseId, String name) {
-        if (this.remote) {
-            FileFieldValue first = this.getValue().getNamesPaths().find({ namePath -> namePath.name == name })
-            return first != null ? first.path : null
-        }
-        return FileListFieldValue.getPath(caseId, getStringId(), name)
-    }
-
-    boolean isRemote() {
-        return this.remote
-    }
-
-    void setRemote(boolean remote) {
-        this.remote = remote
-    }
-
     @Override
     Field clone() {
         FileListField clone = new FileListField()
         super.clone(clone)
-        clone.remote = this.remote
-
+        clone.storage = this.storage
         return clone
     }
 }
diff --git a/...n/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileListFieldValue.groovy b/...n/groovy/com/netgrif/application/engine/petrinet/domain/dataset/FileListFieldValue.groovy
@@ -1,8 +1,5 @@
 package com.netgrif.application.engine.petrinet.domain.dataset
 
-import com.netgrif.application.engine.configuration.ApplicationContextProvider
-import com.netgrif.application.engine.workflow.domain.FileStorageConfiguration
-
 class FileListFieldValue implements Serializable {
 
     private static final long serialVersionUID = 5299918326436821185L;
@@ -39,11 +36,6 @@ class FileListFieldValue implements Serializable {
         return newVal
     }
 
-    static String getPath(String caseId, String fieldId, String name) {
-        FileStorageConfiguration fileStorageConfiguration = ApplicationContextProvider.getBean("fileStorageConfiguration")
-        return "${fileStorageConfiguration.getStoragePath()}/${caseId}/${fieldId}/${name}"
-    }
-
     @Override
     String toString() {
         return namesPaths.toString()

diff --git a/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/MinIoStorage.groovy b/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/MinIoStorage.groovy
@@ -0,0 +1,17 @@
+package com.netgrif.application.engine.petrinet.domain.dataset;
+
+class MinIoStorage extends Storage {
+    private String bucket
+
+    MinIoStorage() {
+        super(StorageType.MINIO)
+    }
+
+    String getBucket() {
+        return bucket
+    }
+
+    void setBucket(String bucket) {
+        this.bucket = bucket
+    }
+}
diff --git a/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/Storage.groovy b/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/Storage.groovy
@@ -0,0 +1,45 @@
+package com.netgrif.application.engine.petrinet.domain.dataset
+
+import com.querydsl.core.annotations.PropertyType
+import com.querydsl.core.annotations.QueryType
+
+class Storage {
+    private StorageType type
+    private String host
+
+    Storage() {
+        this.type = StorageType.LOCAL
+    }
+
+    Storage(StorageType type) {
+        this()
+        this.type = type
+    }
+
+    Storage(StorageType type, String host) {
+        this(type)
+        this.host = host
+    }
+
+    StorageType getType() {
+        return type
+    }
+
+    void setType(StorageType type) {
+        this.type = type
+    }
+
+    String getHost() {
+        return host
+    }
+
+    void setHost(String host) {
+        this.host = host
+    }
+
+    @Override
+    @QueryType(PropertyType.NONE)
+    MetaClass getMetaClass() {
+        return this.metaClass
+    }
+}
diff --git a/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/StorageField.groovy b/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/StorageField.groovy
@@ -0,0 +1,29 @@
+package com.netgrif.application.engine.petrinet.domain.dataset
+
+abstract class StorageField<T> extends Field<T> {
+    static final long serialVersionUID = -9172755427378929924L
+    private Storage storage
+
+    StorageField() {
+        super()
+    }
+
+    StorageType getStorageType() {
+        if (storage == null) {
+            return StorageType.LOCAL
+        }
+        return storage.getType()
+    }
+
+    void setStorageType(StorageType storageType) {
+        this.storage.setType(storageType)
+    }
+
+    Storage getStorage() {
+        return storage
+    }
+
+    void setStorage(Storage storage) {
+        this.storage = storage
+    }
+}
diff --git a/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/StorageType.groovy b/src/main/groovy/com/netgrif/application/engine/petrinet/domain/dataset/StorageType.groovy
@@ -0,0 +1,6 @@
+package com.netgrif.application.engine.petrinet.domain.dataset
+
+enum StorageType {
+    LOCAL,
+    MINIO;
+}