From 344f23cd2016580c584c6cba150e15d46ebbb73f Mon Sep 17 00:00:00 2001 From: Joe Yates Date: Mon, 1 Apr 2024 11:20:33 +0200 Subject: [PATCH] Add the 'copy' command --- lib/imap/backup/cli.rb | 63 +++++++++++ lib/imap/backup/cli/transfer.rb | 13 ++- lib/imap/backup/mirror.rb | 8 +- spec/features/copy_spec.rb | 195 ++++++++++++++++++++++++++++++++ spec/unit/cli/transfer_spec.rb | 29 +++++ 5 files changed, 299 insertions(+), 9 deletions(-) create mode 100644 spec/features/copy_spec.rb diff --git a/lib/imap/backup/cli.rb b/lib/imap/backup/cli.rb index 078ff151..119940ab 100644 --- a/lib/imap/backup/cli.rb +++ b/lib/imap/backup/cli.rb @@ -94,6 +94,69 @@ def backup Backup.new(non_logging_options).run end + desc( + "copy SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]", + "Copies emails from the SOURCE account to the DESTINATION account, avoiding duplicates" + ) + long_desc <<~DESC + This command copies messages from the SOURCE_EMAIL account + to the DESTINATION_EMAIL account. It keeps track of copied + messages and avoids duplicate copies. + + Any other messages that are present on the DESTINATION_EMAIL account + are not affected. + + If a folder list is configured for the SOURCE_EMAIL account, + only the folders indicated by the setting are copied. + + First, it runs the download of the SOURCE_EMAIL account. + + When the copy command is used, for each folder that is processed, + a new file is created alongside the normal backup files (.imap and .mbox) + This file has a '.mirror' extension. This file contains a mapping of + the known UIDs on the source account to those on the destination account. + + Some configuration may be necessary, as follows: + + #{NAMESPACE_CONFIGURATION_DESCRIPTION} + DESC + config_option + quiet_option + verbose_option + method_option( + "automatic-namespaces", + type: :boolean, + desc: "automatically choose delimiters and prefixes" + ) + method_option( + "destination-delimiter", + type: :string, + desc: "the delimiter for destination folder names" + ) + method_option( + "destination-prefix", + type: :string, + desc: "the prefix (namespace) to add to destination folder names", + aliases: ["-d"] + ) + method_option( + "source-delimiter", + type: :string, + desc: "the delimiter for source folder names" + ) + method_option( + "source-prefix", + type: :string, + desc: "the prefix (namespace) to strip from source folder names", + aliases: ["-s"] + ) + # Copies messages from one email account to another + # @return [void] + def copy(source_email, destination_email) + non_logging_options = Imap::Backup::Logger.setup_logging(options) + Transfer.new(:copy, source_email, destination_email, non_logging_options).run + end + desc "local SUBCOMMAND [OPTIONS]", "View local info" subcommand "local", Local diff --git a/lib/imap/backup/cli/transfer.rb b/lib/imap/backup/cli/transfer.rb index b3a03675..7a18829f 100644 --- a/lib/imap/backup/cli/transfer.rb +++ b/lib/imap/backup/cli/transfer.rb @@ -13,7 +13,7 @@ class CLI::Transfer include CLI::Helpers # The possible values for the action parameter - ACTIONS = %i(migrate mirror).freeze + ACTIONS = %i(copy migrate mirror).freeze def initialize(action, source_email, destination_email, options) @action = action @@ -39,14 +39,17 @@ def run raise "Unknown action '#{action}'" if !ACTIONS.include?(action) process_options! - prepare_mirror if action == :mirror + warn_if_source_account_is_not_in_mirror_mode if action == :mirror + run_backup if %i(copy mirror).include?(action) folders.each do |serializer, folder| case action + when :copy + Mirror.new(serializer, folder, reset: false).run when :migrate Migrator.new(serializer, folder, reset: reset).run when :mirror - Mirror.new(serializer, folder).run + Mirror.new(serializer, folder, reset: true).run end end end @@ -123,9 +126,7 @@ def add_prefix_and_delimiter_defaults self.source_prefix ||= "" end - def prepare_mirror - warn_if_source_account_is_not_in_mirror_mode - + def run_backup CLI::Backup.new(config: config_path, accounts: source_email).run end diff --git a/lib/imap/backup/mirror.rb b/lib/imap/backup/mirror.rb index 06b87ff5..49b0bb91 100644 --- a/lib/imap/backup/mirror.rb +++ b/lib/imap/backup/mirror.rb @@ -5,9 +5,10 @@ module Imap; end module Imap::Backup # Synchronises a folder between a source and destination class Mirror - def initialize(serializer, folder) + def initialize(serializer, folder, reset: false) @serializer = serializer @folder = folder + @reset = reset end # If necessary, reates the destination folder, @@ -19,7 +20,7 @@ def initialize(serializer, folder) # @return [void] def run ensure_destination_folder - delete_destination_only_emails + delete_destination_only_emails if reset update_flags append_emails map.save @@ -31,6 +32,7 @@ def run attr_reader :serializer attr_reader :folder + attr_reader :reset def ensure_destination_folder return if folder.exist? @@ -90,7 +92,7 @@ def map destination: folder.uid_validity ) if !map_ok - folder.clear + folder.clear if reset map.reset( source_uid_validity: serializer.uid_validity, destination_uid_validity: folder.uid_validity diff --git a/spec/features/copy_spec.rb b/spec/features/copy_spec.rb new file mode 100644 index 00000000..300b71b3 --- /dev/null +++ b/spec/features/copy_spec.rb @@ -0,0 +1,195 @@ +require "features/helper" + +RSpec.describe "imap-backup copy", :container, type: :aruba do + include_context "message-fixtures" + + let(:source_folder) { "my_folder" } + let(:destination_folder) { "other_public.my_folder" } + let(:mirror_file_path) do + File.join(test_server_connection_parameters[:local_path], "#{source_folder}.mirror") + end + let(:msg1_source_uid) { test_server.folder_uids(source_folder).first } + let(:msg1_destination_id) { other_server.folder_uids(destination_folder).first } + let(:config_options) do + { + accounts: [ + test_server_connection_parameters.merge(folders: [source_folder]), + other_server_connection_parameters + ] + } + end + let(:command) do + "imap-backup copy " \ + "--destination-prefix=other_public " \ + "--destination-delimiter=. " \ + "#{test_server_connection_parameters[:username]} " \ + "#{other_server_connection_parameters[:username]}" + end + + before do + test_server.warn_about_non_default_folders + test_server.create_folder source_folder + test_server.send_email source_folder, **message_one, flags: [:Seen] + create_config(**config_options) + FileUtils.rm_rf test_server_connection_parameters[:local_path] + end + + after do + test_server.delete_folder source_folder + test_server.disconnect + other_server.delete_folder destination_folder + other_server.disconnect + end + + it "backs up the source account" do + run_command_and_stop command + + content = mbox_content(test_server_connection_parameters[:username], source_folder) + + expect(content).to eq(to_mbox_entry(**message_one)) + end + + it "creates the destination folder" do + run_command_and_stop command + + expect(other_server.folder_exists?(destination_folder)).to be true + end + + it "appends all emails" do + run_command_and_stop command + + messages = other_server.folder_messages(destination_folder).map do |m| + server_message_to_body(m) + end + expect(messages).to eq([message_as_server_message(**message_one)]) + end + + it "sets flags" do + run_command_and_stop command + + flags = other_server.folder_messages(destination_folder).first["FLAGS"] + flags.reject! { |f| f == :Recent } + expect(flags).to eq([:Seen]) + end + + it "saves the .mirror file" do + run_command_and_stop command + + content = JSON.parse(File.read(mirror_file_path)) + map = content.dig(other_server_connection_parameters[:username], "map") + + expect(map).to eq({msg1_source_uid.to_s => msg1_destination_id}) + end + + context "when there are emails on the destination server" do + before do + other_server.create_folder destination_folder + other_server.send_email destination_folder, **message_two + end + + it "keeps them" do + run_command_and_stop command + + messages = other_server.folder_messages(destination_folder).map do |m| + server_message_to_body(m) + end + expect(messages).to eq([message_as_server_message(**message_two), + message_as_server_message(**message_one)]) + end + end + + context "when a mirror file exists" do + let(:mirror_contents) do + { + other_server_connection_parameters[:username] => { + "source_uid_validity" => test_server.folder_uid_validity(source_folder), + "destination_uid_validity" => other_server.folder_uid_validity(destination_folder), + "map" => { + msg1_source_uid => msg1_destination_id + } + } + } + end + let(:mirror_file) do + FileUtils.mkdir_p test_server_connection_parameters[:local_path] + File.write(mirror_file_path, mirror_contents.to_json) + end + + before do + # msg1 on both, msg2 on source + other_server.create_folder destination_folder + other_server.send_email destination_folder, **message_one + test_server.send_email source_folder, **message_two + mirror_file + end + + it "appends missing emails" do + run_command_and_stop command + + messages = other_server.folder_messages(destination_folder).map do |m| + server_message_to_body(m) + end + expect(messages).to eq([message_as_server_message(**message_one), + message_as_server_message(**message_two)]) + end + + context "when flags have changed" do + before do + other_server.create_folder destination_folder + other_server.send_email destination_folder, **message_one, flags: [:Draft] + mirror_file + end + + it "updates them" do + run_command_and_stop command + + flags = other_server.folder_messages(destination_folder).first["FLAGS"] + flags.reject! { |f| f == :Recent } + expect(flags).to eq([:Seen]) + end + end + + context "when there are emails on the destination server that are not on the source server" do + before do + other_server.send_email destination_folder, **message_three + end + + it "keeps them" do + run_command_and_stop command + + messages = other_server.folder_messages(destination_folder).map do |m| + server_message_to_body(m) + end + expect(messages).to eq( + [ + message_as_server_message(**message_one), + message_as_server_message(**message_three), + message_as_server_message(**message_two) + ] + ) + end + end + end + + context "when a config path is supplied" do + let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } + let(:config_options) { super().merge(path: custom_config_path) } + let(:command) do + "imap-backup copy " \ + "--destination-prefix=other_public " \ + "--destination-delimiter=. " \ + "#{test_server_connection_parameters[:username]} " \ + "#{other_server_connection_parameters[:username]} " \ + "--config #{custom_config_path}" + end + + it "copies messages" do + run_command_and_stop command + + messages = other_server.folder_messages(destination_folder).map do |m| + server_message_to_body(m) + end + expect(messages).to eq([message_as_server_message(**message_one)]) + end + end +end diff --git a/spec/unit/cli/transfer_spec.rb b/spec/unit/cli/transfer_spec.rb index a2037403..78cfc62c 100644 --- a/spec/unit/cli/transfer_spec.rb +++ b/spec/unit/cli/transfer_spec.rb @@ -60,6 +60,29 @@ module Imap::Backup end end + context "when in copy mode" do + let(:action) { :copy } + let(:backup) { instance_double(CLI::Backup, "backup_1", run: nil) } + + it "runs backup on the source" do + subject.run + + expect(backup).to have_received(:run) + end + + it "mirrors each folder" do + subject.run + + expect(mirror).to have_received(:run) + end + + it "instructs the mirror class to not reset the destination folder" do + subject.run + + expect(Mirror).to have_received(:new).with(anything, anything, reset: false) { mirror } + end + end + context "when in mirror mode" do let(:action) { :mirror } let(:backup) { instance_double(CLI::Backup, "backup_1", run: nil) } @@ -87,6 +110,12 @@ module Imap::Backup expect(mirror).to have_received(:run) end + + it "instructs the mirror class to reset the destination folder" do + subject.run + + expect(Mirror).to have_received(:new).with(anything, anything, reset: true) { mirror } + end end context "when source and destination emails are the same" do