Refactroging – preparing for #42

Author: Vít Šesták

backup.py

@@ -73,49 +73,16 @@ def action_backup(vm_info, config, session, args):
 	vm_instance = vm_info.vm.instance_if_running()
 	if vm_instance is not None:
 		vm_instance.try_sync()
-	volume_clone = vm.private_volume().clone("v6-qubes-backup-poc-cloned")
-	try:
-		backup_storage_vm = VmInstance(config.get_backup_storage_vm_name())
-		with Dvm() as dvm:
-			dvm.attach("xvdz", volume_clone)  # --ro: 1. is not needed since it is a clone, 2. blocks repair procedures when mounting
-			try:
-				dvm.check_call("sudo mkdir /mnt/clone")
-				dvm.check_call("sudo mount /dev/xvdz /mnt/clone") # TODO: consider -o nosuid,noexec – see issue #16
-				try:
-					backup_backend.upload_agent(dvm)
-					with backup_backend.add_permissions(backup_storage_vm, dvm, vm_keys.encrypted_name):
-						# run the agent
-						with subprocess.Popen(dvm.create_command("/tmp/backup-agent "+shlex.quote(backup_storage_vm.get_name())+" "+shlex.quote(vm_keys.encrypted_name)), stdin = subprocess.PIPE) as proc:
-							proc.stdin.write(vm_keys.key)
-							proc.stdin.close()
-							assert(proc.wait() == 0) # uarrgh, implemented by busy loop
-					# TODO: also copy ~/.v6-qubes-backup-poc/master to the backup in order to make it recoverable without additional data (except password). See issue #12.
-				finally: dvm.check_call("sudo umount /mnt/clone")
-			finally: dvm.detach_all()
-	finally: volume_clone.remove()
+	backup_storage_vm = VmInstance(config.get_backup_storage_vm_name())
+	backup_backend.backup_vm(vm, vm_keys, backup_storage_vm)
 
 def action_restore(restored_vm_info, config, session, args):
 	# Maybe type vm_info is not what I need here…
 	new_name = args.vm_name_template.replace("%", restored_vm_info.vm.get_name())
-	subprocess.check_call("qvm-create "+shlex.quote(new_name)+" "+args.qvm_create_args, shell=True)
-	subprocess.check_call(["qvm-prefs", "-s", new_name, "netvm", "none"]) # Safe approach…
 	new_vm = Vm(new_name)
 	backup_backend = config.get_backup_backend()
 	backup_storage_vm = VmInstance(config.get_backup_storage_vm_name())
-	with Dvm() as dvm:
-		dvm.attach("xvdz", new_vm.private_volume())
-		try:
-			dvm.check_call("sudo mkdir /mnt/clone")
-			dvm.check_call("sudo mount /dev/xvdz /mnt/clone")
-			try:
-				backup_backend.upload_agent(dvm)
-				with backup_backend.add_permissions(backup_storage_vm, dvm, restored_vm_info.vm_keys.encrypted_name):
-					with subprocess.Popen(dvm.create_command("/tmp/restore-agent "+shlex.quote(backup_storage_vm.get_name())+" "+shlex.quote(restored_vm_info.vm_keys.encrypted_name)), stdin = subprocess.PIPE) as proc:
-						proc.stdin.write(restored_vm_info.vm_keys.key)
-						proc.stdin.close()
-						assert(proc.wait() == 0) # uarrgh, implemented by busy loop
-			finally: dvm.check_call("sudo umount /mnt/clone")
-		finally: dvm.detach_all()
+	backup_backend.restore_vm(new_vm, new_name, args.qvm_create_args, restored_vm_info.vm_keys, backup_storage_vm)
 
 def action_show_vm_keys(vm_info, config, session, args):
 	print(vm_info.vm_keys.encrypted_name+": "+base64.b64encode(vm_info.vm_keys.key).decode("ascii"))

backupbackends/basic.py

@@ -0,0 +1,24 @@
+from base64 import b64encode
+import shlex
+
+# If you want to implement another backend, you can, but maybe you should wait a while until the API becomes more stable.
+# TODO: document interface
+class BackupBackend:	
+	
+	# TODO: Move elsewhere. This will be done as part of backupbackend-agnostic backup storage within #37
+	def add_permissions(self, backup_storage_vm, dvm, encrypted_name):
+		permission_file = "/var/run/v6-qubes-backup-poc-permissions/"+b64encode(dvm.get_name().encode("ascii")).decode("ascii")
+		return _DuplicityPermissionsContext(backup_storage_vm, encrypted_name, permission_file)
+
+
+class _DuplicityPermissionsContext:
+	def __init__(self, backup_storage_vm, encrypted_name, permission_file):
+		self.permission_file = permission_file
+		self.backup_storage_vm = backup_storage_vm
+		self.encrypted_name = encrypted_name
+	def __enter__(self):
+		self.backup_storage_vm.check_call("sudo mkdir -p /var/run/v6-qubes-backup-poc-permissions")
+		self.backup_storage_vm.check_call("echo -n "+shlex.quote(self.encrypted_name)+" | sudo tee "+shlex.quote(self.permission_file)+".ip")
+		self.backup_storage_vm.check_call("sudo mv "+shlex.quote(self.permission_file)+".ip "+shlex.quote(self.permission_file)) # This way prevents race condition. I know, this is not the best approach for duraility, but that's not what we need. (Especially if those files don't survive reboot.)
+	def __exit__(self, type, value, traceback):
+		self.backup_storage_vm.check_call("sudo rm "+shlex.quote(self.permission_file)) # remove permissions (not strictly needed for security, just hygiene)

backupbackends/duplicity.py

@@ -3,6 +3,7 @@
 import subprocess
 import shlex
 import base64
+from .dvmbased import DvmBasedBackupBackend
 
 # I know this is duplicated code, but sharing the same code between Python 2 and Python 3 would be probably hell.
 
@@ -44,9 +45,7 @@ def read_safe_filename(inp):
 	return sanitize_filename(read_until_zero(inp, 255).decode("ascii"))
 
 
-# If you want to implement another backend, you can, but maybe you should wait a while until the API becomes more stable.
-# TODO: document interface
-class DuplicityBackupBackend:
+class DuplicityBackupBackend(DvmBasedBackupBackend):
 	base_path = os.path.dirname(os.path.realpath(__file__))+"/duplicity-vm-files/"
 	def upload_agent(self, vm):
 		with open(self.base_path+"vm-backup-agent", "rb") as inp:
@@ -61,7 +60,9 @@ def upload_agent(self, vm):
 			vm.check_call("sudo mkdir /usr/lib/python2.7/dist-packages/duplicity/backends/qubesintervmbackendprivate")
 			vm.check_call("sudo touch /usr/lib/python2.7/dist-packages/duplicity/backends/qubesintervmbackendprivate/__init__.py")
 			vm.check_call("sudo tee /usr/lib/python2.7/dist-packages/duplicity/backends/qubesintervmbackendprivate/common.py", stdin = inp)
-
+	
+	# TODO: Move methods below to backupbackend-agnostic backup storage handler:
+	
 	def install_dom0(self, vm):
 		subprocess.check_call("echo "+shlex.quote("$anyvm  "+vm.get_name()+" allow")+" | sudo tee /etc/qubes-rpc/policy/v6ak.QubesInterVmBackupStorage", shell=True, stdout=subprocess.DEVNULL)
 
@@ -77,10 +78,6 @@ def install_backup_storage_vm(self, vm):
 			# FIXME: Don't be so aggressive!
 			vm.check_call("sudo tee /usr/local/share/v6-qubes-backup-poc/common.py", stdin = inp)
 
-	def add_permissions(self, backup_storage_vm, dvm, encrypted_name):
-		permission_file = "/var/run/v6-qubes-backup-poc-permissions/"+base64.b64encode(dvm.get_name().encode("ascii")).decode("ascii")
-		return _DuplicityPermissionsContext(backup_storage_vm, encrypted_name, permission_file)
-
 	def list_backups(self, backup_storage_vm):
 		vms = []
 		with backup_storage_vm.popen("/usr/local/share/v6-qubes-backup-poc/list-backups.py", stdout = subprocess.PIPE) as proc:
@@ -94,14 +91,3 @@ def list_backups(self, backup_storage_vm):
 				else:
 					raise Exception("Unexpected character #"+str(ord(c)))
 
-class _DuplicityPermissionsContext:
-	def __init__(self, backup_storage_vm, encrypted_name, permission_file):
-		self.permission_file = permission_file
-		self.backup_storage_vm = backup_storage_vm
-		self.encrypted_name = encrypted_name
-	def __enter__(self):
-		self.backup_storage_vm.check_call("sudo mkdir -p /var/run/v6-qubes-backup-poc-permissions")
-		self.backup_storage_vm.check_call("echo -n "+shlex.quote(self.encrypted_name)+" | sudo tee "+shlex.quote(self.permission_file)+".ip")
-		self.backup_storage_vm.check_call("sudo mv "+shlex.quote(self.permission_file)+".ip "+shlex.quote(self.permission_file)) # This way prevents race condition. I know, this is not the best approach for duraility, but that's not what we need. (Especially if those files don't survive reboot.)
-	def __exit__(self, type, value, traceback):
-		self.backup_storage_vm.check_call("sudo rm "+shlex.quote(self.permission_file)) # remove permissions (not strictly needed for security, just hygiene)

backupbackends/dvmbased.py

@@ -0,0 +1,39 @@
+from qubesvmtools import Dvm
+import shlex
+import subprocess
+from .basic import BackupBackend
+
+class DvmBasedBackupBackend(BackupBackend):
+	def backup_vm(self, vm, vm_keys, backup_storage_vm):
+		volume_clone = vm.private_volume().clone("v6-qubes-backup-poc-cloned")
+		try:
+			with Dvm() as dvm:
+				dvm.attach("xvdz", volume_clone)  # --ro: 1. is not needed since it is a clone, 2. blocks repair procedures when mounting
+				try:
+					dvm.check_call("sudo mkdir /mnt/clone")
+					dvm.check_call("sudo mount /dev/xvdz /mnt/clone") # TODO: consider -o nosuid,noexec – see issue #16
+					try:
+						self.upload_agent(dvm)
+						with self.add_permissions(backup_storage_vm, dvm, vm_keys.encrypted_name):
+							dvm.check_call("/tmp/backup-agent "+shlex.quote(backup_storage_vm.get_name())+" "+shlex.quote(vm_keys.encrypted_name), input = vm_keys.key, stdout = None, stderr = None)
+					finally: dvm.check_call("sudo umount /mnt/clone")
+				finally: dvm.detach_all()
+		finally: volume_clone.remove()
+
+	def restore_vm(self, new_vm, new_name, qvm_create_args, vm_keys, backup_storage_vm):
+		subprocess.check_call("qvm-create "+shlex.quote(new_name)+" "+qvm_create_args, shell=True)
+		subprocess.check_call(["qvm-prefs", "-s", new_name, "netvm", "none"]) # Safe approach…
+		with Dvm() as dvm:
+			dvm.attach("xvdz", new_vm.private_volume())
+			try:
+				dvm.check_call("sudo mkdir /mnt/clone")
+				dvm.check_call("sudo mount /dev/xvdz /mnt/clone")
+				try:
+					self.upload_agent(dvm)
+					with self.add_permissions(backup_storage_vm, dvm, vm_keys.encrypted_name):
+						dvm.check_call("/tmp/restore-agent "+shlex.quote(backup_storage_vm.get_name())+" "+shlex.quote(vm_keys.encrypted_name), input = vm_keys.key, stdout = None, stderr = None)
+				finally: dvm.check_call("sudo umount /mnt/clone")
+			finally: dvm.detach_all()
+
+	# abstract def upload_agent(self, dvm)
+

qubesvmtools.py

@@ -1,6 +1,6 @@
 # python3
 import subprocess
-from subprocess import Popen
+from subprocess import Popen, DEVNULL
 import re
 from pathlib import Path
 import os
@@ -10,7 +10,7 @@ class Vm:
 	def __init__(self, name):
 		pattern = re.compile("\\A[a-zA-Z0-9-]+\\Z")
 		if not pattern.match(name):
-			raise Exception("bad name")
+			raise Exception("bad name: "+name)
 		self.name = name
 	def is_running(self):
 		ret = subprocess.call(["qvm-check", "--running", self.name], stdout=subprocess.DEVNULL)
@@ -66,17 +66,18 @@ def try_sync(self):
 			pass
 	def sync(self):
 		self.check_call("sync") # TODO: Sync in more universal way. See #46
-	def check_call(self, command, stdin = None, input = None):
+	def check_call(self, command, stdin = None, input = None, stdout = DEVNULL, stderr = DEVNULL):
 		if stdin == None:
 			stdin_type = subprocess.PIPE
 		elif input == None:
 			stdin_type = stdin
 		else:
 			raise Exception("cannot handle both stdin and input")
 		command_native = self.create_command(command)
-		with Popen(command_native, stdin = stdin_type, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL) as proc:
+		with Popen(command_native, stdin = stdin_type, stdout = stdout, stderr = stderr) as proc:
 			if input is not None:
 				proc.stdin.write(input)
+				proc.stdin.close()
 			ret = proc.wait()
 			if ret != 0:
 				raise subprocess.CalledProcessError(ret, command_native)

upload.sh

@@ -9,6 +9,8 @@ set -o pipefail
 PACK=(
 	backup backup.py
 	install-backup-storage-vm install-backup-storage-vm.py
+	backupbackends/basic.py
+	backupbackends/dvmbased.py
 	backupbackends/duplicity.py
 	backupbackends/duplicity-vm-files/vm-backup-agent
 	backupbackends/duplicity-vm-files/vm-restore-agent
@@ -30,6 +32,7 @@ IGNORE=(
 	__pycache__
 	backupbackends/__pycache__
 	tests/__pycache__
+	backupbackends/qvmbackup.py.notdone
 )
 
 if ! diff <(find "${PACK[@]}" "${IGNORE[@]}" -type f -or -type l | sort) <(find -type f -or -type l | sed 's#^\./##' | sort); then