Skip to content

Commit 013825c

Browse files
mi-charmi-char
authored
Merge pull request #186 from avast/fix/gcs_remove_bucket_dependency
GCS - remove bucket dependency
2 parents ff194cb + 9e39456 commit 013825c

File tree

3 files changed

+70
-99
lines changed

3 files changed

+70
-99
lines changed

‎gcs/src/main/resources/reference.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
gcsBackendDefaults {
22
//projectId = "" // REQUIRED
33
//bucketName = "" // REQUIRED
4-
//jsonKeyPath = "" // REQUIRED if using service account authentication (see https://github.com/googleapis/google-cloud-java#using-a-service-account-recommended)
4+
//credentialsFile = "" // REQUIRED if using service account authentication (see https://github.com/googleapis/google-cloud-java#using-a-service-account-recommended)
55
}

‎gcs/src/main/scala/com/avast/clients/storage/gcs/GcsStorageBackend.scala

Lines changed: 51 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package com.avast.clients.storage.gcs
22

33
import better.files.File
4-
import cats.data.EitherT
54
import cats.effect.implicits.catsEffectSyntaxBracket
65
import cats.effect.{Blocker, ContextShift, Resource, Sync}
76
import cats.syntax.all._
@@ -10,7 +9,7 @@ import com.avast.clients.storage.{ConfigurationException, GetResult, HeadResult,
109
import com.avast.scala.hashes.Sha256
1110
import com.google.auth.oauth2.ServiceAccountCredentials
1211
import com.google.cloud.ServiceOptions
13-
import com.google.cloud.storage.{Blob, Bucket, Storage, StorageOptions, StorageException => GcStorageException}
12+
import com.google.cloud.storage.{Blob, BlobId, Storage, StorageOptions, StorageException => GcStorageException}
1413
import com.typesafe.config.{Config, ConfigFactory}
1514
import com.typesafe.scalalogging.StrictLogging
1615
import pureconfig.error.ConfigReaderException
@@ -23,7 +22,9 @@ import java.nio.charset.StandardCharsets
2322
import java.nio.file.StandardOpenOption
2423
import java.security.{DigestOutputStream, MessageDigest}
2524

26-
class GcsStorageBackend[F[_]: Sync: ContextShift](bucket: Bucket)(blocker: Blocker) extends StorageBackend[F] with StrictLogging {
25+
class GcsStorageBackend[F[_]: Sync: ContextShift](storageClient: Storage, bucketName: String)(blocker: Blocker)
26+
extends StorageBackend[F]
27+
with StrictLogging {
2728
private val FileStreamOpenOptions = Seq(StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)
2829

2930
override def head(sha256: Sha256): F[Either[StorageException, HeadResult]] = {
@@ -74,7 +75,7 @@ class GcsStorageBackend[F[_]: Sync: ContextShift](bucket: Bucket)(blocker: Block
7475
for {
7576
objectPath <- Sync[F].delay(composeBlobPath(sha256))
7677
result <- blocker.delay {
77-
Option(bucket.get(objectPath))
78+
Option(storageClient.get(BlobId.of(bucketName, objectPath)))
7879
}
7980
} yield result
8081
}
@@ -117,34 +118,28 @@ object GcsStorageBackend {
117118
private val DefaultConfig = ConfigFactory.defaultReference().getConfig("gcsBackendDefaults")
118119

119120
def fromConfig[F[_]: Sync: ContextShift](config: Config,
120-
blocker: Blocker): EitherT[F, ConfigurationException, Resource[F, GcsStorageBackend[F]]] = {
121-
122-
def composeConfig: EitherT[F, ConfigurationException, GcsBackendConfiguration] = EitherT {
123-
Sync[F].delay {
124-
pureconfig.ConfigSource
125-
.fromConfig(config.withFallback(DefaultConfig))
126-
.load[GcsBackendConfiguration]
127-
.leftMap { failures =>
128-
ConfigurationException("Could not load config", new ConfigReaderException[GcsBackendConfiguration](failures))
129-
}
130-
}
121+
blocker: Blocker): Either[ConfigurationException, Resource[F, GcsStorageBackend[F]]] = {
122+
123+
def composeConfig: Either[ConfigurationException, GcsBackendConfiguration] = {
124+
pureconfig.ConfigSource
125+
.fromConfig(config.withFallback(DefaultConfig))
126+
.load[GcsBackendConfiguration]
127+
.leftMap { failures =>
128+
ConfigurationException("Could not load config", new ConfigReaderException[GcsBackendConfiguration](failures))
129+
}
131130
}
132131

133-
{
134-
for {
135-
conf <- composeConfig
136-
storageClient <- prepareStorageClient(conf, blocker)
137-
bucket <- getBucket(conf, storageClient, blocker)
138-
} yield (storageClient, bucket)
139-
}.map {
140-
case (storage, bucket) =>
141-
Resource
142-
.fromAutoCloseable {
143-
Sync[F].pure(storage)
144-
}
145-
.map { _ =>
146-
new GcsStorageBackend[F](bucket)(blocker)
147-
}
132+
for {
133+
conf <- composeConfig
134+
storageClient <- prepareStorageClient(conf, blocker)
135+
} yield {
136+
Resource
137+
.fromAutoCloseable {
138+
Sync[F].pure(storageClient)
139+
}
140+
.map { storageClient =>
141+
new GcsStorageBackend[F](storageClient, conf.bucketName)(blocker)
142+
}
148143
}
149144
}
150145

@@ -154,66 +149,36 @@ object GcsStorageBackend {
154149
}
155150

156151
def prepareStorageClient[F[_]: Sync: ContextShift](conf: GcsBackendConfiguration,
157-
blocker: Blocker): EitherT[F, ConfigurationException, Storage] = {
158-
EitherT {
159-
blocker.delay {
160-
Either
161-
.catchNonFatal {
162-
val credentialsFileContent = conf.credentialsFile
163-
.map { credentialsFilePath =>
164-
new FileInputStream(credentialsFilePath)
165-
}
166-
.orElse {
167-
sys.env.get("GOOGLE_APPLICATION_CREDENTIALS_RAW").map { credentialFileRaw =>
168-
new ByteArrayInputStream(credentialFileRaw.getBytes(StandardCharsets.UTF_8))
169-
}
170-
}
171-
172-
val builder = credentialsFileContent match {
173-
case Some(inputStream) =>
174-
StorageOptions.newBuilder
175-
.setCredentials(ServiceAccountCredentials.fromStream(inputStream))
176-
case None =>
177-
StorageOptions.getDefaultInstance.toBuilder
178-
}
179-
180-
builder
181-
.setProjectId(conf.projectId)
182-
.setRetrySettings(ServiceOptions.getNoRetrySettings)
183-
184-
builder.build.getService
152+
blocker: Blocker): Either[ConfigurationException, Storage] = {
153+
Either
154+
.catchNonFatal {
155+
val credentialsFileContent = conf.credentialsFile
156+
.map { credentialsFilePath =>
157+
new FileInputStream(credentialsFilePath)
185158
}
186-
.leftMap { e =>
187-
ConfigurationException("Could not create GCS client", e)
188-
}
189-
}
190-
}
191-
}
192-
193-
def getBucket[F[_]: Sync: ContextShift](conf: GcsBackendConfiguration,
194-
storageClient: Storage,
195-
blocker: Blocker): EitherT[F, ConfigurationException, Bucket] = {
196-
EitherT {
197-
blocker
198-
.delay {
199-
Either
200-
.catchNonFatal {
201-
Option(storageClient.get(conf.bucketName, Storage.BucketGetOption.userProject(conf.projectId)))
159+
.orElse {
160+
sys.env.get("GOOGLE_APPLICATION_CREDENTIALS_RAW").map { credentialFileRaw =>
161+
new ByteArrayInputStream(credentialFileRaw.getBytes(StandardCharsets.UTF_8))
202162
}
203-
}
204-
.map {
205-
_.leftMap { e =>
206-
ConfigurationException(s"Attempt to get bucket ${conf.bucketName} failed", e)
207-
}.flatMap {
208-
case Some(bucket) =>
209-
Right(bucket)
210-
case None =>
211-
Left {
212-
ConfigurationException(s"Bucket ${conf.bucketName} does not exist")
213-
}
214163
}
164+
165+
val builder = credentialsFileContent match {
166+
case Some(inputStream) =>
167+
StorageOptions.newBuilder
168+
.setCredentials(ServiceAccountCredentials.fromStream(inputStream))
169+
case None =>
170+
StorageOptions.getDefaultInstance.toBuilder
215171
}
216-
}
172+
173+
builder
174+
.setProjectId(conf.projectId)
175+
.setRetrySettings(ServiceOptions.getNoRetrySettings)
176+
177+
builder.build.getService
178+
}
179+
.leftMap { e =>
180+
ConfigurationException("Could not create GCS client", e)
181+
}
217182
}
218183
}
219184

‎gcs/src/test/scala/com/avast/clients/storage/gcs/GcsStorageBackendTest.scala

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ package com.avast.clients.storage.gcs
22

33
import better.files.File
44
import cats.effect.Blocker
5-
import com.avast.clients.storage.gcs.TestImplicits.{randomString, StringOps}
5+
import com.avast.clients.storage.gcs.TestImplicits.{StringOps, randomString}
66
import com.avast.clients.storage.{GetResult, HeadResult}
77
import com.avast.scala.hashes.Sha256
8-
import com.google.cloud.storage.{Blob, Bucket}
8+
import com.google.cloud.storage.{Blob, BlobId, Storage}
99
import monix.eval.Task
1010
import monix.execution.Scheduler.Implicits.global
1111
import org.junit.runner.RunWith
@@ -26,13 +26,16 @@ class GcsStorageBackendTest extends FunSuite with ScalaFutures with MockitoSugar
2626
val content = randomString(fileSize)
2727
val sha = content.sha256
2828
val shaStr = sha.toString()
29+
val bucketName = "bucket-tst"
2930

3031
val blob = mock[Blob]
3132
when(blob.getSize).thenReturn(fileSize.toLong)
3233

33-
val bucket = mock[Bucket]
34-
when(bucket.get(any[String]())).thenAnswer { call =>
35-
val blobPath = call.getArgument[String](0)
34+
val storageClient = mock[Storage]
35+
when(storageClient.get(any[BlobId]())).thenAnswer { call =>
36+
val blobId = call.getArgument[BlobId](0)
37+
val blobPath = blobId.getName
38+
assertResult(bucketName)(blobId.getBucket)
3639
assertResult {
3740
List(
3841
shaStr.substring(0, 2),
@@ -44,7 +47,7 @@ class GcsStorageBackendTest extends FunSuite with ScalaFutures with MockitoSugar
4447
blob
4548
}
4649

47-
val result = composeTestBackend(bucket).head(sha).runSyncUnsafe(10.seconds)
50+
val result = composeTestBackend(storageClient, bucketName).head(sha).runSyncUnsafe(10.seconds)
4851

4952
assertResult(Right(HeadResult.Exists(fileSize)))(result)
5053
}
@@ -54,6 +57,7 @@ class GcsStorageBackendTest extends FunSuite with ScalaFutures with MockitoSugar
5457
val content = randomString(fileSize)
5558
val sha = content.sha256
5659
val shaStr = sha.toString()
60+
val bucketName = "bucket-tst"
5761

5862
val blob = mock[Blob]
5963
when(blob.getSize).thenReturn(fileSize.toLong)
@@ -62,9 +66,11 @@ class GcsStorageBackendTest extends FunSuite with ScalaFutures with MockitoSugar
6266
outputStream.write(content.getBytes())
6367
}
6468

65-
val bucket = mock[Bucket]
66-
when(bucket.get(any[String]())).thenAnswer { call =>
67-
val blobPath = call.getArgument[String](0)
69+
val storageClient = mock[Storage]
70+
when(storageClient.get(any[BlobId]())).thenAnswer { call =>
71+
val blobId = call.getArgument[BlobId](0)
72+
val blobPath = blobId.getName
73+
assertResult(bucketName)(blobId.getBucket)
6874
assertResult {
6975
List(
7076
shaStr.substring(0, 2),
@@ -77,7 +83,7 @@ class GcsStorageBackendTest extends FunSuite with ScalaFutures with MockitoSugar
7783
}
7884

7985
File.usingTemporaryFile() { file =>
80-
val result = composeTestBackend(bucket).get(sha, file).runSyncUnsafe(10.seconds)
86+
val result = composeTestBackend(storageClient, bucketName).get(sha, file).runSyncUnsafe(10.seconds)
8187
assertResult(Right(GetResult.Downloaded(file, fileSize)))(result)
8288
assertResult(sha.toString.toLowerCase)(file.sha256.toLowerCase)
8389
assertResult(fileSize)(file.size)
@@ -89,8 +95,8 @@ class GcsStorageBackendTest extends FunSuite with ScalaFutures with MockitoSugar
8995
assertResult("d0/5a/f9/d05af9a8494696906e8eec79843ca1e4bf408c280616a121ed92f9e92e2de831")(GcsStorageBackend.composeBlobPath(sha))
9096
}
9197

92-
private def composeTestBackend(bucket: Bucket): GcsStorageBackend[Task] = {
98+
private def composeTestBackend(storageClient: Storage, bucketName: String): GcsStorageBackend[Task] = {
9399
val blocker = Blocker.liftExecutionContext(monix.execution.Scheduler.io())
94-
new GcsStorageBackend[Task](bucket)(blocker)
100+
new GcsStorageBackend[Task](storageClient, bucketName)(blocker)
95101
}
96102
}

0 commit comments

Comments
 (0)