From c79a8e3eb4adb300789e352b75d7357be1d660c8 Mon Sep 17 00:00:00 2001 From: Jaymee Hyppolite <54775395+JaymeeH@users.noreply.github.com> Date: Thu, 30 May 2024 19:14:44 -0400 Subject: [PATCH] 542 cleaning exported files (#750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Creating a cron job to delete old exported files Adding whenever gem to tigerdata Automated cron job deletes exported files that are more than one week old. * rubocop 🚨 * updating cron job * deleting inventory request record a week after it is created, and after the rake task deletes the file --- Gemfile | 1 + Gemfile.lock | 4 +++ app/jobs/delete_user_job.rb | 21 ++++++++++++ app/jobs/list_project_contents_job.rb | 5 +++ config/schedule.rb | 25 +++++++++++++++ lib/tasks/exports.rake | 4 +-- spec/jobs/delete_user_job_spec.rb | 46 +++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 app/jobs/delete_user_job.rb create mode 100644 config/schedule.rb create mode 100644 spec/jobs/delete_user_job_spec.rb diff --git a/Gemfile b/Gemfile index 102ed285..4c9dd243 100644 --- a/Gemfile +++ b/Gemfile @@ -65,6 +65,7 @@ gem "mailcatcher" gem "net-http-persistent" gem "sidekiq" +gem "whenever", require: false group :staging, :production do gem "ddtrace", require: "ddtrace/auto_instrument" end diff --git a/Gemfile.lock b/Gemfile.lock index e7d104be..acf9d79a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -136,6 +136,7 @@ GEM rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) + chronic (0.10.2) coderay (1.1.3) coercible (1.0.0) descendants_tracker (~> 0.0.1) @@ -495,6 +496,8 @@ GEM websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) + whenever (1.0.0) + chronic (>= 0.6.3) xpath (3.2.0) nokogiri (~> 1.8) yard (0.9.36) @@ -557,6 +560,7 @@ DEPENDENCIES vite_rails web-console webmock + whenever yard RUBY VERSION diff --git a/app/jobs/delete_user_job.rb b/app/jobs/delete_user_job.rb new file mode 100644 index 00000000..875b9a37 --- /dev/null +++ b/app/jobs/delete_user_job.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +class DeleteUserJob < ApplicationJob + queue_as :default + + def perform(job_id:, user_id:) + user = User.find(user_id) + job = UserJob.find(job_id) + job.delete + + mark_user_job_as_complete(job_id: job_id, user: user) + end + + private + + def mark_user_job_as_complete(job_id:, user:) + user_job = UserJob.create_and_link_to_user(job_id: job_id, user: user, job_title: "Deleting user job with id: #{job_id}") + user_job.completed_at = Time.current.in_time_zone("America/New_York").iso8601 + user_job.save! + user_job.reload + end +end diff --git a/app/jobs/list_project_contents_job.rb b/app/jobs/list_project_contents_job.rb index 90ee3c7f..152823b3 100644 --- a/app/jobs/list_project_contents_job.rb +++ b/app/jobs/list_project_contents_job.rb @@ -1,5 +1,10 @@ # frozen_string_literal: true class ListProjectContentsJob < ApplicationJob + after_perform do |job| + uid = job.arguments.first[:user_id] + DeleteUserJob.set(wait: 1.week).perform_later(job_id: job_id, user_id: uid) + end + def perform(user_id:, project_id:) project = Project.find(project_id) raise "Invalid project id #{project_id} for job #{job_id}" if project.nil? diff --git a/config/schedule.rb b/config/schedule.rb new file mode 100644 index 00000000..1de6d885 --- /dev/null +++ b/config/schedule.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true +# Use this file to easily define all of your cron jobs. +# +# It's helpful, but not entirely necessary to understand cron before proceeding. +# http://en.wikipedia.org/wiki/Cron + +# Example: +# +# set :output, "/path/to/my/cron_log.log" +# +# every 2.hours do +# command "/usr/bin/some_great_command" +# runner "MyModel.some_method" +# rake "some:great:rake:task" +# end +# +# every 4.days do +# runner "AnotherModel.prune_old_records" +# end + +# Learn more: http://github.com/javan/whenever + +every 1.day do + rake "exports:delete_old" +end diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake index 035f32c9..195c0824 100644 --- a/lib/tasks/exports.rake +++ b/lib/tasks/exports.rake @@ -9,9 +9,9 @@ namespace :exports do end end - desc "Deletes files that are too old (default to 30 days)" + desc "Deletes files that are too old (default to 7 days)" task :delete_old, [:days] => [:environment] do |_, args| - days = (args[:days] || "30").to_i + days = (args[:days] || "7").to_i pathname = Pathname.new(Rails.configuration.mediaflux["shared_files_location"]) scan_directory(pathname.join("*.csv")).each do |file| if file[:age] > days diff --git a/spec/jobs/delete_user_job_spec.rb b/spec/jobs/delete_user_job_spec.rb new file mode 100644 index 00000000..52df4cd5 --- /dev/null +++ b/spec/jobs/delete_user_job_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true +require "rails_helper" + +RSpec.describe DeleteUserJob, stub_mediaflux: true do + let(:user) { FactoryBot.create(:user) } + let(:sponsor_user) { FactoryBot.create(:project_sponsor, uid: "pul123") } + let(:metadata) do + { + data_sponsor: sponsor_user.uid, + data_manager: sponsor_user.uid, + project_directory: "project-123", + title: "project 123", + departments: ["RDSS"], + description: "hello world", + status: ::Project::PENDING_STATUS + } + end + + let(:project_in_mediaflux) { FactoryBot.create(:project, mediaflux_id: 8888, metadata: metadata) } + + before do + stub_request(:post, "http://mediaflux.example.com:8888/__mflux_svc__") + .with(body: //) + .to_return(status: 200, body: fixture_file("files/query_response.xml")) + + stub_request(:post, "http://mediaflux.example.com:8888/__mflux_svc__") + .with(body: //) + .to_return(status: 200, body: fixture_file("files/iterator_response_get_values.xml")) + + stub_request(:post, "http://mediaflux.example.com:8888/__mflux_svc__") + .with(body: //) + .to_return(status: 200, body: "") + end + + describe "#perform_now" do + it "deletes the user job that requested file inventory" do + # Request inventory + job = ListProjectContentsJob.perform_now(user_id: user.id, project_id: project_in_mediaflux.id) + + # Delete the inventory request record after a week + uid = user.id.to_s + described_class.perform_now(user_id: uid, job_id: job.id) + expect(user.user_jobs.where("id=#{job.id}").empty?).to be_truthy + end + end +end