Merge pull request #21 from Freika/fix/import-files-rework

Fix imports uploading

.app_version

@@ -1 +1 @@
-0.1.8.1
+0.1.9

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.1.9] — 2024-04-25
+
+### Added
+
+- A test for CheckAppVersion service class
+
+### Changed
+
+- Replaced ActiveStorage with Shrine for file uploads
+
+### Fixed
+
+- `ActiveStorage::FileNotFoundError` error when uploading export files
+
+
 ## [0.1.8.1] — 2024-04-21
 
 ### Changed

Gemfile

@@ -5,37 +5,38 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
 ruby '3.2.3'
 gem 'bootsnap', require: false
+gem 'chartkick'
 gem 'devise'
+gem 'geocoder'
+gem 'importmap-rails'
 gem 'pg'
 gem 'puma'
 gem 'pundit'
 gem 'rails'
+gem 'shrine', '~> 3.5'
+gem 'sidekiq'
+gem 'sidekiq-cron'
 gem 'sprockets-rails'
 gem 'stimulus-rails'
 gem 'tailwindcss-rails'
 gem 'turbo-rails'
 gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]
-gem 'importmap-rails'
-gem 'chartkick'
-gem 'geocoder'
-gem 'sidekiq'
-gem 'sidekiq-cron'
-
 
 group :development, :test do
   gem 'debug', platforms: %i[mri mingw x64_mingw]
+  gem 'dotenv-rails'
   gem 'factory_bot_rails'
   gem 'ffaker'
-  gem 'rspec-rails'
-  gem 'dotenv-rails'
   gem 'pry-byebug'
   gem 'pry-rails'
+  gem 'rspec-rails'
 end
 
 group :test do
   gem 'shoulda-matchers'
   gem 'simplecov'
   gem 'super_diff'
+  gem 'webmock'
 end
 
 group :development do

Gemfile.lock

@@ -75,6 +75,8 @@ GEM
       minitest (>= 5.1)
       mutex_m
       tzinfo (~> 2.0)
+    addressable (2.8.6)
+      public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     attr_extras (7.1.0)
     base64 (0.2.0)
@@ -88,6 +90,10 @@ GEM
     coderay (1.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
+    content_disposition (1.0.0)
+    crack (1.0.0)
+      bigdecimal
+      rexml
     crass (1.0.6)
     date (3.3.4)
     debug (1.9.2)
@@ -105,6 +111,8 @@ GEM
     dotenv-rails (3.1.0)
       dotenv (= 3.1.0)
       railties (>= 6.1)
+    down (5.4.2)
+      addressable (~> 2.8)
     drb (2.2.1)
     erubi (1.12.0)
     et-orbi (1.2.11)
@@ -122,6 +130,7 @@ GEM
     geocoder (1.8.2)
     globalid (1.2.1)
       activesupport (>= 6.1)
+    hashdiff (1.1.0)
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
     importmap-rails (2.0.1)
@@ -189,6 +198,7 @@ GEM
       pry (>= 0.10.4)
     psych (5.1.2)
       stringio
+    public_suffix (5.0.5)
     puma (6.4.2)
       nio4r (~> 2.0)
     pundit (2.3.1)
@@ -285,6 +295,9 @@ GEM
     ruby-progressbar (1.13.0)
     shoulda-matchers (6.2.0)
       activesupport (>= 5.2.0)
+    shrine (3.5.0)
+      content_disposition (~> 1.0)
+      down (~> 5.1)
     sidekiq (7.2.2)
       concurrent-ruby (< 2)
       connection_pool (>= 2.3.0)
@@ -337,6 +350,10 @@ GEM
     unicode-display_width (2.5.0)
     warden (1.2.9)
       rack (>= 2.0.9)
+    webmock (3.23.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.8.1)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
@@ -372,6 +389,7 @@ DEPENDENCIES
   rspec-rails
   rubocop-rails
   shoulda-matchers
+  shrine (~> 3.5)
   sidekiq
   sidekiq-cron
   simplecov
@@ -381,6 +399,7 @@ DEPENDENCIES
   tailwindcss-rails
   turbo-rails
   tzinfo-data
+  webmock
 
 RUBY VERSION
    ruby 3.2.3p157

app/controllers/imports_controller.rb

@@ -15,19 +15,18 @@ def new
 
   def create
     files = import_params[:files].reject(&:blank?)
+
     import_ids = files.map do |file|
       import = current_user.imports.create(
         name: file.original_filename,
         source: params[:import][:source]
       )
 
-      import.file.attach(file)
+      import.update(raw_data: JSON.parse(File.read(file)))
       import.id
     end
 
-    import_ids.each do |import_id|
-      ImportJob.set(wait: 5.seconds).perform_later(current_user.id, import_id)
-    end
+    import_ids.each { ImportJob.perform_later(current_user.id, _1) }
 
     redirect_to imports_url, notice: "#{files.size} files are queued to be imported in background", status: :see_other
   rescue StandardError => e

app/jobs/import_job.rb

@@ -1,12 +1,11 @@
+# frozen_string_literal: true
+
 class ImportJob < ApplicationJob
   queue_as :default
 
   def perform(user_id, import_id)
     user = User.find(user_id)
     import = user.imports.find(import_id)
-    file = import.file
-
-    sleep 3 # It takes time to process uploaded file
 
     result = parser(import.source).new(import).call
 

app/models/import.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 class Import < ApplicationRecord
   belongs_to :user
   has_many :points, dependent: :destroy
 
-  has_one_attached :file
+  include ImportUploader::Attachment(:raw)
 
   enum source: { google: 0, owntracks: 1 }
 end

app/services/check_app_version.rb

@@ -9,7 +9,7 @@ def initialize
   def call
     begin
       latest_version = JSON.parse(Net::HTTP.get(URI.parse(@repo_url)))[0]['name']
-    rescue
+    rescue StandardError
       return false
     end
 

app/services/google_maps/timeline_parser.rb

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
 class GoogleMaps::TimelineParser
-  attr_reader :import, :json
+  attr_reader :import
 
   def initialize(import)
     @import = import
-    @json = JSON.parse(import.file.download)
   end
 
   def call
@@ -38,7 +37,7 @@ def call
   private
 
   def parse_json
-    json['timelineObjects'].flat_map do |timeline_object|
+    import.raw_data['timelineObjects'].flat_map do |timeline_object|
       if timeline_object['activitySegment'].present?
         if timeline_object['activitySegment']['startLocation'].blank?
           next if timeline_object['activitySegment']['waypointPath'].blank?
@@ -61,7 +60,7 @@ def parse_json
         end
       elsif timeline_object['placeVisit'].present?
         if timeline_object['placeVisit']['location']['latitudeE7'].present? &&
-            timeline_object['placeVisit']['location']['longitudeE7'].present?
+           timeline_object['placeVisit']['location']['longitudeE7'].present?
           {
             latitude: timeline_object['placeVisit']['location']['latitudeE7'].to_f / 10**7,
             longitude: timeline_object['placeVisit']['location']['longitudeE7'].to_f / 10**7,

app/services/own_tracks/export_parser.rb

@@ -5,7 +5,7 @@ class OwnTracks::ExportParser
 
   def initialize(import)
     @import = import
-    @json = JSON.parse(import.file.download)
+    @json = import.raw_data
   end
 
   def call
@@ -32,16 +32,16 @@ def call
     doubles = points_data.size - points
     processed = points + doubles
 
-    { raw_points: points_data.size, points: points, doubles: doubles, processed: processed }
+    { raw_points: points_data.size, points:, doubles:, processed: }
   end
 
   private
 
   def parse_json
     points = []
 
-    json.keys.each do |user|
-      json[user].keys.each do |devise|
+    json.each_key do |user|
+      json[user].each_key do |devise|
         json[user][devise].each { |point| points << OwnTracks::Params.new(point).call }
       end
     end

app/uploaders/import_uploader.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ImportUploader < Shrine
+  # plugins and uploading logic
+end

config/shrine.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'shrine'
+require 'shrine/storage/file_system'
+
+Shrine.storages = {
+  cache: Shrine::Storage::FileSystem.new('public', prefix: 'uploads/cache'), # temporary
+  store: Shrine::Storage::FileSystem.new('public', prefix: 'uploads')        # permanent
+}
+
+Shrine.plugin :activerecord           # loads Active Record integration
+Shrine.plugin :cached_attachment_data # enables retaining cached file across form redisplays
+Shrine.plugin :restore_cached_data    # extracts metadata for assigned cached files

db/migrate/20240425200155_add_raw_data_to_imports.rb

@@ -0,0 +1,5 @@
+class AddRawDataToImports < ActiveRecord::Migration[7.1]
+  def change
+    add_column :imports, :raw_data, :jsonb
+  end
+end

spec/factories/imports.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 FactoryBot.define do
   factory :import do
     user
     name { 'APRIL_2013.json' }
     source { 1 }
+    raw_data { JSON.parse(File.read('spec/fixtures/files/owntracks/export.json')) }
   end
 end

spec/jobs/import_job_spec.rb

@@ -4,10 +4,8 @@
   describe '#perform' do
     subject(:perform) { described_class.new.perform(user.id, import.id) }
 
-    let(:file_path) { 'spec/fixtures/files/owntracks/export.json' }
-    let(:file) { fixture_file_upload(file_path) }
     let(:user) { create(:user) }
-    let(:import) { create(:import, user: user, file: file, name: File.basename(file.path)) }
+    let(:import) { create(:import, user:, name: 'owntracks_export.json') }
 
     it 'creates points' do
       expect { perform }.to change { Point.count }.by(8)

spec/rails_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'spec_helper'
 ENV['RAILS_ENV'] ||= 'test'
 require_relative '../config/environment'
@@ -6,7 +8,7 @@
 require 'rspec/rails'
 # Add additional requires below this line. Rails is not loaded until this point!
 
-Dir[Rails.root.join('spec', 'support', '**', '*.rb')].sort.each { |f| require f }
+Dir[Rails.root.join('spec/support/**/*.rb')].sort.each { |f| require f }
 
 # Checks for pending migrations and applies them before tests are run.
 # If you are not using ActiveRecord, you can remove these lines.

spec/requests/export_spec.rb

@@ -1,13 +1,18 @@
+# frozen_string_literal: true
+
 require 'rails_helper'
 
-RSpec.describe "Exports", type: :request do
-  describe "GET /create" do
+RSpec.describe 'Exports', type: :request do
+  describe 'GET /create' do
     before do
+      stub_request(:any, 'https://api.github.com/repos/Freika/dawarich/tags')
+        .to_return(status: 200, body: '[{"name": "1.0.0"}]', headers: {})
+
       sign_in create(:user)
     end
 
-    it "returns http success" do
-      get "/export"
+    it 'returns http success' do
+      get '/export'
       expect(response).to have_http_status(:success)
     end
   end

spec/requests/home_spec.rb

@@ -1,9 +1,14 @@
 require 'rails_helper'
 
-RSpec.describe "Homes", type: :request do
-  describe "GET /" do
-    it "returns http success" do
-      get "/"
+RSpec.describe 'Homes', type: :request do
+  describe 'GET /' do
+    before do
+      stub_request(:any, 'https://api.github.com/repos/Freika/dawarich/tags')
+        .to_return(status: 200, body: '[{"name": "1.0.0"}]', headers: {})
+    end
+
+    it 'returns http success' do
+      get '/'
       expect(response).to have_http_status(:success)
     end
   end