542 cleaning exported files (#750)

* Creating a cron job to delete old exported files
Adding whenever gem to tigerdata
Automated cron job deletes exported files that are more than one week old.

* rubocop 🚨

* updating cron job

* deleting inventory request record a week after it is created, and after the rake task deletes the file

Author: JaymeeH

Gemfile

@@ -65,6 +65,7 @@ gem "mailcatcher"
 gem "net-http-persistent"
 gem "sidekiq"
 
+gem "whenever", require: false
 group :staging, :production do
   gem "ddtrace", require: "ddtrace/auto_instrument"
 end

Gemfile.lock

@@ -136,6 +136,7 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
+    chronic (0.10.2)
     coderay (1.1.3)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
@@ -495,6 +496,8 @@ GEM
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
+    whenever (1.0.0)
+      chronic (>= 0.6.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
     yard (0.9.36)
@@ -557,6 +560,7 @@ DEPENDENCIES
   vite_rails
   web-console
   webmock
+  whenever
   yard
 
 RUBY VERSION

app/jobs/delete_user_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+class DeleteUserJob < ApplicationJob
+  queue_as :default
+
+  def perform(job_id:, user_id:)
+    user = User.find(user_id)
+    job = UserJob.find(job_id)
+    job.delete
+
+    mark_user_job_as_complete(job_id: job_id, user: user)
+  end
+
+  private
+
+    def mark_user_job_as_complete(job_id:, user:)
+      user_job = UserJob.create_and_link_to_user(job_id: job_id, user: user, job_title: "Deleting user job with id: #{job_id}")
+      user_job.completed_at = Time.current.in_time_zone("America/New_York").iso8601
+      user_job.save!
+      user_job.reload
+    end
+end

app/jobs/list_project_contents_job.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 class ListProjectContentsJob < ApplicationJob
+  after_perform do |job|
+    uid = job.arguments.first[:user_id]
+    DeleteUserJob.set(wait: 1.week).perform_later(job_id: job_id, user_id: uid)
+  end
+
   def perform(user_id:, project_id:)
     project = Project.find(project_id)
     raise "Invalid project id #{project_id} for job #{job_id}" if project.nil?

config/schedule.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+#   command "/usr/bin/some_great_command"
+#   runner "MyModel.some_method"
+#   rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+#   runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+
+every 1.day do
+  rake "exports:delete_old"
+end

lib/tasks/exports.rake

@@ -9,9 +9,9 @@ namespace :exports do
     end
   end
 
-  desc "Deletes files that are too old (default to 30 days)"
+  desc "Deletes files that are too old (default to 7 days)"
   task :delete_old, [:days] => [:environment] do |_, args|
-    days = (args[:days] || "30").to_i
+    days = (args[:days] || "7").to_i
     pathname = Pathname.new(Rails.configuration.mediaflux["shared_files_location"])
     scan_directory(pathname.join("*.csv")).each do |file|
       if file[:age] > days

spec/jobs/delete_user_job_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+require "rails_helper"
+
+RSpec.describe DeleteUserJob, stub_mediaflux: true do
+  let(:user) { FactoryBot.create(:user) }
+  let(:sponsor_user) { FactoryBot.create(:project_sponsor, uid: "pul123") }
+  let(:metadata) do
+    {
+      data_sponsor: sponsor_user.uid,
+      data_manager: sponsor_user.uid,
+      project_directory: "project-123",
+      title: "project 123",
+      departments: ["RDSS"],
+      description: "hello world",
+      status: ::Project::PENDING_STATUS
+    }
+  end
+
+  let(:project_in_mediaflux) { FactoryBot.create(:project, mediaflux_id: 8888, metadata: metadata) }
+
+  before do
+    stub_request(:post, "http://mediaflux.example.com:8888/__mflux_svc__")
+      .with(body: /<service name=\"asset.query\" session=\"test-session-token\">/)
+      .to_return(status: 200, body: fixture_file("files/query_response.xml"))
+
+    stub_request(:post, "http://mediaflux.example.com:8888/__mflux_svc__")
+      .with(body: /<service name=\"asset.query.iterate\" session=\"test-session-token\">/)
+      .to_return(status: 200, body: fixture_file("files/iterator_response_get_values.xml"))
+
+    stub_request(:post, "http://mediaflux.example.com:8888/__mflux_svc__")
+      .with(body: /<service name=\"asset.query.iterator.destroy\" session=\"test-session-token\">/)
+      .to_return(status: 200, body: "")
+  end
+
+  describe "#perform_now" do
+    it "deletes the user job that requested file inventory" do
+      # Request inventory
+      job = ListProjectContentsJob.perform_now(user_id: user.id, project_id: project_in_mediaflux.id)
+
+      # Delete the inventory request record after a week
+      uid = user.id.to_s
+      described_class.perform_now(user_id: uid, job_id: job.id)
+      expect(user.user_jobs.where("id=#{job.id}").empty?).to be_truthy
+    end
+  end
+end