Merge pull request #4 from wallarm/dev

Allow to use StringIO as a download dest

Author: yamax2

.rubocop.yml

@@ -36,3 +36,6 @@ RSpec/ExampleLength:
 
 RSpec/NestedGroups:
   Max: 4
+
+Lint/SuppressedException:
+  Enabled: false

README.md

@@ -85,6 +85,11 @@ cli = CloudStorage::Client.new(
 > f = cli.upload_file(key: 'test.txt', file: File.open('test.txt', 'rb'))
 > f.name
 => 'test.txt'
+
+> stream = StringIO.new('test')
+> f = cli.upload_file(key: 'test.txt', file: stream)
+> f.name
+=> 'test.txt'
 ```
 
 ### signed url:
@@ -107,6 +112,11 @@ cli = CloudStorage::Client.new(
 ```ruby
 > cli.find('test.txt').download.read
 => "This is a test download\n"
+
+s = StringIO.new
+> cli.find('test.txt').download(s)
+> s.read
+=> "This is a test download\n"
 ```
 
 ```ruby

dip.yml

@@ -37,6 +37,7 @@ provision:
   - ./docker/prepare_env.sh
   - mkdir -p uploads/s3/$S3_BUCKET
   - rm -rf uploads/gcs/$GCS_BUCKET
+  - rm -rf Gemfile.lock
   - mkdir -p uploads/gcs/$GCS_BUCKET
   - docker volume create --name bundler_data
   - dip bundle install

lib/cloud_storage/objects/base.rb

@@ -14,10 +14,7 @@ def size
       end
 
       def download(local_file = Tempfile.new)
-        internal_download(local_file.path)
-
-        local_file.rewind
-        local_file
+        internal_download(local_file).tap(&:rewind)
       end
 
       def signed_url(**opts)
@@ -30,7 +27,7 @@ def delete!
 
       private
 
-      def internal_download(path)
+      def internal_download(local_file)
         raise 'not implemented'
       end
     end

lib/cloud_storage/objects/gcs.rb

@@ -32,8 +32,8 @@ def delete!
 
       private
 
-      def internal_download(path)
-        @internal.download(path)
+      def internal_download(local_file)
+        @internal.download(local_file)
       end
     end
   end

lib/cloud_storage/objects/s3.rb

@@ -37,8 +37,10 @@ def delete!
 
       private
 
-      def internal_download(path)
-        @resource.bucket(bucket_name).object(key).download_file(path)
+      def internal_download(local_file)
+        @resource.bucket(bucket_name).object(key).download_file(local_file)
+
+        local_file
       end
     end
   end

lib/cloud_storage/wrappers/gcs.rb

@@ -52,7 +52,7 @@ def exist?(key)
 
       def upload_file(key:, file:, **opts)
         Objects::Gcs.new \
-          bucket.create_file(file.path, key, **opts),
+          bucket.create_file(file, key, **opts),
           uri: uri
       end
 
@@ -96,6 +96,7 @@ def bucket
       def build_bucket
         bucket = storage.bucket(@bucket_name)
         raise ObjectNotFound, @bucket_name unless bucket
+
         bucket
       end
     end

lib/cloud_storage/wrappers/s3.rb

@@ -34,8 +34,7 @@ def each
               resource: @resource,
               client: @client
           end
-
-          rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound
+        rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound
         end
       end
 
@@ -50,7 +49,7 @@ def exist?(key)
       def upload_file(key:, file:, **opts)
         obj = resource.bucket(@bucket_name).object(key)
 
-        return unless obj.upload_file(file.path, **opts)
+        return unless upload_file_or_io(obj, file, **opts)
 
         Objects::S3.new \
           obj,
@@ -74,12 +73,13 @@ def find(key)
       end
 
       def delete_files(keys)
-        resource.bucket(@bucket_name).delete_objects({
-          delete: {
-            objects: keys.map { |key| {key: key} },
-            quiet: true
-          }
-        })
+        resource
+          .bucket(@bucket_name)
+          .delete_objects \
+            delete: {
+              objects: keys.map { |key| { key: key } },
+              quiet: true
+            }
       rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound
       end
 
@@ -92,6 +92,14 @@ def client
       def resource
         @resource ||= Aws::S3::Resource.new(@options)
       end
+
+      def upload_file_or_io(obj, file_or_io, **opts)
+        if file_or_io.respond_to?(:path)
+          obj.upload_file(file_or_io.path, **opts)
+        else
+          obj.upload_stream(**opts) { |write_stream| IO.copy_stream(file_or_io, write_stream) }
+        end
+      end
     end
   end
 end

spec/cloud_storage/objects/gcs_spec.rb

@@ -5,6 +5,8 @@
   let(:file) { File.open('spec/fixtures/test.txt', 'rb') }
   let!(:obj) { cli.upload_file(key: 'test_1.txt', file: file) }
 
+  after { file.close }
+
   describe '#signed_url' do
     after { obj.delete! }
 
@@ -45,16 +47,50 @@
     after { obj.delete! }
 
     context 'when default' do
+      subject(:content) { tmp.read }
+
+      let(:tmp) { obj.download }
+
+      after do
+        tmp.close
+        tmp.unlink
+      end
+
       it do
-        expect(obj.download.read).to eq("This is a test upload\n")
+        expect { content }.to change { opened_tmp_files_count }.by(1)
+
+        expect(content).to eq("This is a test upload\n")
       end
     end
 
     context 'when download to a custom tmp' do
+      subject(:content) { obj.download(tmp) }
+
       let(:tmp) { Tempfile.new }
 
+      after do
+        tmp.close
+        tmp.unlink
+      end
+
+      it do
+        tmp
+
+        expect { content }
+          .to change(tmp, :read)
+          .from('')
+          .to("This is a test upload\n")
+          .and change { opened_tmp_files_count }.by(0)
+      end
+    end
+
+    context 'when download to a string io' do
+      subject(:content) { obj.download(StringIO.new).read }
+
       it do
-        expect { obj.download(tmp) }.to change(tmp, :read).from('').to("This is a test upload\n")
+        expect { content }.not_to(change { opened_tmp_files_count })
+
+        expect(content).to eq("This is a test upload\n")
       end
     end
   end

spec/cloud_storage/objects/s3_spec.rb

@@ -5,6 +5,8 @@
   let(:file) { File.open('spec/fixtures/test.txt', 'rb') }
   let!(:obj) { cli.upload_file(key: 'test_1.txt', file: file) }
 
+  after { file.close }
+
   describe '#signed_url' do
     subject(:url) { obj.signed_url(expires_in: 30) }
 
@@ -36,16 +38,50 @@
     after { obj.delete! }
 
     context 'when default' do
+      subject(:content) { tmp.read }
+
+      let(:tmp) { obj.download }
+
+      after do
+        tmp.close
+        tmp.unlink
+      end
+
+      it do
+        expect { content }.to change { opened_tmp_files_count }.by(1)
+
+        expect(content).to eq("This is a test upload\n")
+      end
+    end
+
+    context 'when download to a string io' do
+      subject(:content) { obj.download(StringIO.new).read }
+
       it do
-        expect(obj.download.read).to eq("This is a test upload\n")
+        expect { content }.not_to(change { opened_tmp_files_count })
+
+        expect(content).to eq("This is a test upload\n")
       end
     end
 
     context 'when download to a custom tmp' do
+      subject(:content) { obj.download(tmp) }
+
       let(:tmp) { Tempfile.new }
 
+      after do
+        tmp.close
+        tmp.unlink
+      end
+
       it do
-        expect { obj.download(tmp) }.to change(tmp, :read).from('').to("This is a test upload\n")
+        tmp
+
+        expect { content }
+          .to change(tmp, :read)
+          .from('')
+          .to("This is a test upload\n")
+          .and change { opened_tmp_files_count }.by(0)
       end
     end
   end