Update licenses to 2022-2023, add tests for uploading 1,000 and 100,0…

…00 objects in S3 using MinIO, add back Renovate

.github/workflows/Linting.yaml

@@ -66,6 +66,9 @@ jobs:
       - name: Validate Gradle Wrapper
         uses: gradle/wrapper-validation-action@v1
 
+      - name: Upload Google Credentials in ~/.google-creds.json
+        run: echo "${{secrets.GOOGLE_CREDENTIALS_JSON}}" >> ~/.google-creds.json
+
       - name: Lint code-base with Spotless
         uses: gradle/gradle-build-action@v2
         with:
@@ -75,3 +78,8 @@ jobs:
         uses: gradle/gradle-build-action@v2
         with:
           arguments: compileJava --no-daemon --scan
+
+      - name: Run all unit tests
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: test --no-daemon --scan

bom/build.gradle.kts

@@ -84,7 +84,7 @@ val snapshotRelease: Boolean = run {
 
 publishing {
     publications {
-        create<MavenPublication>("ktor") {
+        create<MavenPublication>("remi") {
             from(components["javaPlatform"])
 
             artifactId = "remi-bom"

build.gradle.kts

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

buildSrc/build.gradle.kts

@@ -34,10 +34,6 @@ repositories {
 
 dependencies {
     implementation("com.diffplug.spotless:spotless-plugin-gradle:6.12.0")
-    implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.7.20")
-    implementation(kotlin("gradle-plugin", version = "1.7.22"))
-    implementation(kotlin("serialization", version = "1.7.22"))
-    implementation("io.kotest:kotest-gradle-plugin:0.3.9")
     implementation("dev.floofy.commons:gradle:2.4.0")
     implementation(gradleApi())
 }

buildSrc/src/main/kotlin/org/noelware/remi/gradle/Extensions.kt

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

buildSrc/src/main/kotlin/org/noelware/remi/gradle/Metadata.kt

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -26,5 +26,5 @@ package org.noelware.remi.gradle
 import org.gradle.api.JavaVersion
 import dev.floofy.utils.gradle.*
 
-val VERSION = Version(0, 5, 0, 0, ReleaseType.Snapshot)
+val VERSION = Version(0, 5, 1, 0, ReleaseType.Beta)
 val JAVA_VERSION = JavaVersion.VERSION_17

buildSrc/src/main/kotlin/remi-module.gradle.kts

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -46,11 +46,16 @@ repositories {
 
 dependencies {
     implementation("org.jetbrains:annotations:23.0.0")
+    implementation("org.slf4j:slf4j-api:2.0.6")
 
     // test deps
     testImplementation("org.junit.jupiter:junit-jupiter-api:5.9.1")
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.9.1")
-    testImplementation("org.slf4j:slf4j-simple:2.0.3")
+    testImplementation("org.slf4j:slf4j-simple:2.0.6")
+
+    if (path.startsWith(":support")) {
+        implementation(project(":core"))
+    }
 }
 
 spotless {

renovate.json

@@ -0,0 +1,7 @@
+{
+  "automerge": true,
+  "extends": ["config:base", "default:timezone(America/Phoenix)", "docker:disableMajor"],
+  "vulnerabilityAlerts": {
+    "labels": ["security"]
+  }
+}

settings.gradle.kts

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,13 +23,94 @@
 
 rootProject.name = "remi"
 
+pluginManagement {
+    repositories {
+        gradlePluginPortal()
+        mavenCentral()
+        mavenLocal()
+    }
+}
+
+plugins {
+    id("com.gradle.enterprise") version "3.12"
+}
+
 include(
     ":bom",
     ":core",
-    ":serialization",
     ":support:azure",
     ":support:fs",
     ":support:gcs",
     ":support:oracle-cloud",
     ":support:s3"
 )
+
+gradle.settingsEvaluated {
+    logger.info("Checking if we can overwrite cache...")
+    val overrideBuildCacheProp: String? = System.getProperty("org.noelware.gradle.overwriteCache")
+    val buildCacheDir = when (val prop = System.getProperty("org.noelware.gradle.cachedir")) {
+        null -> "${System.getProperty("user.dir")}/.caches/gradle"
+        else -> when {
+            prop.startsWith("~/") -> "${System.getProperty("user.home")}${prop.substring(1)}"
+            prop.startsWith("./") -> "${System.getProperty("user.dir")}${prop.substring(1)}"
+            else -> prop
+        }
+    }
+
+    if (overrideBuildCacheProp == null) {
+        logger.info("""
+        |If you wish to override the build cache for this Gradle process, you can use the
+        |-Dorg.noelware.gradle.overwriteCache=<bool> Java property in `~/.gradle/gradle.properties`
+        |to overwrite it in $buildCacheDir!
+        """.trimMargin("|"))
+    } else {
+        logger.info("Setting up build cache in directory [$buildCacheDir]")
+        val file = File(buildCacheDir)
+        if (!file.exists()) file.mkdirs()
+
+        buildCache {
+            local {
+                directory = "$file"
+                removeUnusedEntriesAfterDays = 7
+            }
+        }
+    }
+
+    val disableJavaSanityCheck = when {
+        System.getProperty("org.noelware.gradle.ignoreJavaCheck", "false").matches("^(yes|true|1|si|si*)$".toRegex()) -> true
+        (System.getenv("DISABLE_JAVA_SANITY_CHECK") ?: "false").matches("^(yes|true|1|si|si*)$".toRegex()) -> true
+        else -> false
+    }
+
+    if (disableJavaSanityCheck)
+        return@settingsEvaluated
+
+    val version = JavaVersion.current()
+    if (version.majorVersion.toInt() < 17)
+        throw GradleException("Developing charted-server requires JDK 17 or higher, it is currently set in [${System.getProperty("java.home")}, ${System.getProperty("java.version")}] - You can ignore this check by providing the `-Dorg.noelware.charted.ignoreJavaCheck=true` system property.")
+}
+
+val buildScanServer = System.getProperty("org.noelware.gradle.build-scan-server", "") ?: ""
+gradleEnterprise {
+    buildScan {
+        if (buildScanServer.isNotEmpty()) {
+            server = buildScanServer
+            isCaptureTaskInputFiles = true
+            publishAlways()
+        } else {
+            termsOfServiceUrl = "https://gradle.com/terms-of-service"
+            termsOfServiceAgree = "yes"
+
+            // Always publish if we're on CI.
+            if (System.getenv("CI") != null) {
+                publishAlways()
+            }
+        }
+
+        obfuscation {
+            ipAddresses { listOf("0.0.0.0") }
+            hostname { "[redacted]" }
+            username { "[redacted]" }
+        }
+    }
+}

support/azure/build.gradle.kts

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -31,6 +31,4 @@ dependencies {
     testImplementation("org.testcontainers:junit-jupiter:1.17.6")
 
     api("com.azure:azure-storage-blob:12.20.1")
-    implementation("org.slf4j:slf4j-api:2.0.5")
-    implementation(project(":core"))
 }

support/fs/build.gradle.kts

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -24,8 +24,3 @@
 plugins {
     `remi-module`
 }
-
-dependencies {
-    implementation("org.slf4j:slf4j-api:2.0.5")
-    implementation(project(":core"))
-}

support/fs/src/main/java/org/noelware/remi/support/filesystem/FilesystemStorageService.java

@@ -232,16 +232,15 @@ public List<Blob> blobs() throws IOException {
      * @param request The request options object
      * @throws IOException If any I/O exceptions had occurred while uploading the file.
      */
+    @SuppressWarnings("ResultOfMethodCallIgnored")
     @Override
     public void upload(UploadRequest request) throws IOException {
-        final InputStream stream = request.inputStream();
         final String path = request.path();
-
         final File file = Paths.get(config.directory(), path).toFile();
         Files.createDirectories(Paths.get(file.getParent()));
         file.createNewFile();
 
-        try (stream;
+        try (final InputStream stream = request.inputStream();
                 final FileOutputStream out = new FileOutputStream(file)) {
             byte[] buf = new byte[stream.available()];
             int len;
@@ -338,7 +337,7 @@ public boolean delete(String path) throws IOException {
     public void init() {
         final File directory = new File(config.directory());
         if (!directory.exists()) {
-            LOG.debug("Directory [{}] does not exist on local disk, creating!", normalizePath(config.directory()));
+            LOG.debug("Directory [{}] does not exist on local disk, creating!", config.directory());
             directory.mkdirs();
         }
 

support/gcs/build.gradle.kts

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -27,5 +27,4 @@ plugins {
 
 dependencies {
     api("com.google.cloud:google-cloud-storage:2.16.0")
-    implementation(project(":core"))
 }

support/s3/build.gradle.kts

@@ -1,6 +1,6 @@
 /*
  * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider.
- * Copyright (c) 2022 Noelware <[email protected]>
+ * Copyright (c) 2022-2023 Noelware <[email protected]>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -29,6 +29,5 @@ dependencies {
     testImplementation("org.testcontainers:testcontainers:1.17.6")
     testImplementation("org.testcontainers:junit-jupiter:1.17.6")
 
-    implementation(project(":core"))
-    api("software.amazon.awssdk:s3:2.18.24")
+    api("software.amazon.awssdk:s3:2.18.35")
 }

support/s3/src/main/java/org/noelware/remi/support/s3/AmazonS3StorageService.java

@@ -163,34 +163,12 @@ public List<Blob> blobs(@Nullable ListBlobsRequest request) throws IOException {
             final ListObjectsV2Response resp = client.listObjectsV2(listObjectsV2Request);
             try (final Stream<S3Object> stream = resp.contents().parallelStream()) {
                 final List<Blob> newBlobs = stream.map(obj -> {
-                            if (obj.key().endsWith("/")) return null;
-
-                            byte[] data;
-                            try (final InputStream s = open(obj.key())) {
-                                assert s != null;
-                                data = s.readAllBytes();
-                            } catch (IOException e) {
-                                throw new RuntimeException(e);
-                            }
-
-                            String contentType;
                             try {
-                                contentType = getContentTypeOf(data);
+                                return fromS3Object(obj);
                             } catch (IOException e) {
-                                // fallback
-                                contentType = "application/octet-stream";
+                                // Do we throw or return null? I have no idea to be honest...
+                                throw new RuntimeException(e);
                             }
-
-                            return new Blob(
-                                    obj.lastModified(),
-                                    null,
-                                    contentType,
-                                    new ByteArrayInputStream(data),
-                                    obj.eTag(),
-                                    toPrefixedString(obj.key()),
-                                    "s3",
-                                    format("s3://%s", toPrefixedString(obj.key())),
-                                    obj.size());
                         })
                         .filter(Objects::nonNull)
                         .toList();
@@ -226,6 +204,7 @@ public List<Blob> blobs() throws IOException {
      */
     @Override
     public void upload(UploadRequest request) throws IOException {
+        LOG.debug("Uploading object in path [{}] with content type [{}]", request.path(), request.contentType());
         try (final InputStream stream = request.inputStream()) {
             if (stream.available() == 0)
                 throw new IllegalStateException(
@@ -258,7 +237,11 @@ public boolean exists(String path) {
                 builder.key(toPrefixedString(path));
             });
 
-            return !resp.deleteMarker();
+            final Boolean deleteMarker = resp.deleteMarker();
+            if (deleteMarker != null) return !deleteMarker;
+
+            // assume it is true
+            return true;
         } catch (NoSuchKeyException ignored) {
             return false;
         }
@@ -430,6 +413,8 @@ private String toPrefixedString(String key) {
     @Nullable
     private Blob fromS3Object(S3Object obj) throws IOException {
         if (obj.key().endsWith("/")) return null;
+
+        LOG.debug("");
         return blob(obj.key());
     }
 }