Tigerdata Refactor: User Requests (#780)

* New Model for UserRequests
* refactoring the job id on a user request to be a uuid instead of an integer
* creating file inventory request and activate project requests as child classes of user request
* Rename job to FileInventoryJob to match FileInventoryRequest
* Delete unused rake tasks and refactor the job scheduler to run the inventory cleanup daily

Co-authored-by: Bess Sadler <[email protected]>
Co-authored-by: Carolyn Cole <[email protected]>
Co-authored-by: Jaymee Hyppolite <[email protected]>
Co-authored-by: Robert-Anthony Lee-Faison <[email protected]>
Co-authored-by: James R. Griffin III <[email protected]>

Author: 5 people

app/jobs/activate_project_job.rb

@@ -4,7 +4,8 @@ class ActivateProjectJob < ApplicationJob
   def perform(user:, project_id:)
     project = Project.find(project_id)
     raise "Invalid project id #{project_id} for job #{job_id}" if project.nil?
-    collection_id = project.metadata_json["project_id"]
+    #The id of the project in mediaflux is the collection id, and mediaflux id is what we refer to the collection id in rails
+    collection_id = project.mediaflux_id 
     project_metadata = project.metadata_model
 
     # ACTIVATE THE PROJECT IF THE DOI IN RAILS AND MF MATCH
@@ -25,20 +26,11 @@ def perform(user:, project_id:)
       project_metadata: project.metadata
     }
     Honeybadger.notify(activation_failure_msg, context: honeybadger_context)
-
-    mark_user_job_as_complete(project: project, user: user)
   end
 
   private
     def mediaflux_session
       logon_request = Mediaflux::Http::LogonRequest.new
       logon_request.session_token
     end
-
-    def mark_user_job_as_complete(project:, user:)
-      user_job = UserJob.create_and_link_to_user(job_id: job_id, user: user, job_title: "Project Activation for #{project.title}")
-      user_job.completed_at = Time.current.in_time_zone("America/New_York").iso8601
-      user_job.save!
-      user_job.reload
-    end
 end

app/jobs/delete_user_job.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+class FileInventoryCleanupJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    FileInventoryRequest.where(["completion_time < ?", 7.days.ago]).each do |req|
+      File.delete(req.output_file) if File.exist?(req.output_file)
+      req.state = UserRequest::STALE
+      req.save
+    end
+  end
+end

app/jobs/file_inventory_cleanup_job.rb

@@ -1,23 +1,22 @@
 # frozen_string_literal: true
-class ListProjectContentsJob < ApplicationJob
-  after_perform do |job|
-    uid = job.arguments.first[:user_id]
-    DeleteUserJob.set(wait: 1.week).perform_later(job_id: job_id, user_id: uid)
-  end
-
+class FileInventoryJob < ApplicationJob
   def perform(user_id:, project_id:)
     project = Project.find(project_id)
     raise "Invalid project id #{project_id} for job #{job_id}" if project.nil?
     user = User.find(user_id)
     raise "Invalid user id #{user_id} for job #{job_id}" if user.nil?
+    Rails.logger.debug inspect
 
     # Queries Mediaflux for the file list and saves it to a CSV file.
     filename = filename_for_export
     Rails.logger.info "Exporting file list to #{filename} for project #{project_id}"
     project.file_list_to_file(session_id: mediaflux_session, filename: filename)
     Rails.logger.info "Export file generated #{filename} for project #{project_id}"
 
-    mark_user_job_as_complete(project: project, user: user)
+    # Make the FileInventoryRequest object
+    # TODO: Figure out what request_details should have.
+    FileInventoryRequest.create(user_id: user.id, project_id: project.id, job_id: @job_id, state: UserRequest::PENDING, request_details: { output_file: filename, project_title: project.title },
+                                completion_time: Time.current.in_time_zone("America/New_York"))
   end
 
   private
@@ -32,11 +31,4 @@ def filename_for_export
       pathname = Pathname.new(Rails.configuration.mediaflux["shared_files_location"])
       pathname.join("#{job_id}.csv").to_s
     end
-
-    def mark_user_job_as_complete(project:, user:)
-      user_job = UserJob.create_and_link_to_user(job_id: job_id, user: user, job_title: "File list for #{project.title}")
-      user_job.completed_at = Time.current.in_time_zone("America/New_York").iso8601
-      user_job.save!
-      user_job.reload
-    end
 end

app/jobs/list_project_contents_job.rb renamed to app/jobs/file_inventory_job.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+class ActivateProjectRequest < UserRequest
+end

app/models/activate_project_request.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+class FileInventoryRequest < UserRequest
+  def output_file
+    request_details["output_file"]
+  end
+end

app/models/file_inventory_request.rb

@@ -204,6 +204,7 @@ def file_list_to_file(session_id:, filename:)
     query_req = Mediaflux::Http::QueryRequest.new(session_token: session_id, collection: mediaflux_id, deep_search: true)
     iterator_id = query_req.result
 
+
     File.open(filename, "w") do |file|
       # file header
       file.write("ID, PATH, NAME, COLLECTION?, LAST_MODIFIED, SIZE\r\n")

app/models/project.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+class UserRequest < ApplicationRecord
+  belongs_to :user
+  belongs_to :project
+
+  PENDING = "pending"
+  COMPLETED = "completed"
+  STALE = "stale"
+
+  validates :state, inclusion: { in: [PENDING, COMPLETED, STALE] }
+end

app/models/user_request.rb

@@ -6,7 +6,7 @@ def initialize(project:)
   end
 
   def list_contents_job(user:)
-    job = ListProjectContentsJob.perform_later(user_id: user.id, project_id: @project.id)
+    job = FileInventoryJob.perform_later(user_id: user.id, project_id: @project.id)
     # Log the job id and the Sidekiq JID in case we need to troubleshoot the job
     # https://github.com/sidekiq/sidekiq/wiki/Active-Job#job-id
     Rails.logger.info("Job scheduled, job id: #{job.job_id}, (Sidekiq JID: #{job.provider_job_id || 'nil'})")

app/services/project_job_service.rb

@@ -21,5 +21,5 @@
 # Learn more: http://github.com/javan/whenever
 
 every 1.day do
-  rake "exports:delete_old"
+  FileInventoryCleanupJob.perform_later
 end

config/schedule.rb

@@ -0,0 +1,15 @@
+class CreateUserRequests < ActiveRecord::Migration[7.0]
+  def change
+    create_table :user_requests do |t|
+      t.integer :user_id
+      t.integer :project_id
+      t.uuid :job_id
+      t.datetime :completion_time
+      t.string :state
+      t.string :type
+      t.jsonb :request_details
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20240617174803_create_user_requests.rb

@@ -1,31 +1,20 @@
 # frozen_string_literal: true
 require "rails_helper"
 
-RSpec.describe ActivateProjectJob, type: :job do
+RSpec.describe ActivateProjectJob, connect_to_mediaflux: true, type: :job do
   let(:user) { FactoryBot.create(:user) }
-  let(:sponsor_user) { FactoryBot.create(:project_sponsor, uid: "pul123") }
-  let(:collection_id) { 1170 }
-  let(:approved_project) { FactoryBot.create :project }
-  let(:metadata) do
-    {
-      data_sponsor: sponsor_user.uid,
-      data_manager: sponsor_user.uid,
-      directory: "project-123",
-      title: "project 123",
-      departments: ["RDSS"],
-      description: "hello world",
-      status: ::Project::PENDING_STATUS
-    }
-  end
-  let(:project_in_mediaflux) { FactoryBot.create(:project, mediaflux_id: 8888, metadata: metadata) }
+  let(:project_in_mediaflux) { FactoryBot.create(:project_with_doi, status: Project::APPROVED_STATUS) }
 
+  before do
+    ProjectMediaflux.create!(session_id: user.mediaflux_session, project: project_in_mediaflux)
+  end
 
-  describe "#perform_now", connect_to_mediaflux: true do
-    it "updates the UserJob#completed_at attribute" do
-      job = described_class.perform_now(user:, project_id: project_in_mediaflux.id)
-      user_job = UserJob.where(job_id: job.job_id).first
-      expect(user_job.completed_at).to_not be nil
+  describe "#perform_now" do
+    it "marks the state as active" do
+      expect(project_in_mediaflux.status).to eq(Project::APPROVED_STATUS)
+      described_class.perform_now(user: user, project_id: project_in_mediaflux.id)
+      project_in_mediaflux.reload
+      expect(project_in_mediaflux.status).to eq(Project::ACTIVE_STATUS)
     end
-
   end
 end

db/schema.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+require "rails_helper"
+
+RSpec.describe FileInventoryCleanupJob, connect_to_mediaflux: true, type: :job do
+  let(:user) { FactoryBot.create(:user) }
+  let(:project_in_mediaflux) { FactoryBot.create(:project_with_doi) }
+  let(:eight_days_ago) { Time.current.in_time_zone("America/New_York") - 8.days }
+
+  before do
+    ProjectMediaflux.create!(session_id: user.mediaflux_session, project: project_in_mediaflux)
+  end
+
+  describe "#perform_now" do
+    it "deletes any files older than 7 days" do
+      req = FileInventoryJob.perform_now(user_id: user.id, project_id: project_in_mediaflux.id)
+      req.completion_time = eight_days_ago
+      req.save
+
+      expect(File.exist?(req.output_file)).to be_truthy
+      described_class.perform_now
+      expect(File.exist?(req.output_file)).to be_falsey
+    end
+
+    it "marks the file inventory request stale" do
+      req = FileInventoryJob.perform_now(user_id: user.id, project_id: project_in_mediaflux.id)
+      req.completion_time = eight_days_ago
+      req.save
+
+      expect(req.state).to eq(UserRequest::PENDING)
+      described_class.perform_now
+      req.reload
+      expect(req.state).to eq(UserRequest::STALE)
+    end
+  end
+end