diff --git a/Makefile b/Makefile index 3309e6a..95507e7 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ endif kubectl --kubeconfig $(KUBECONFIG) delete -f "deploy/$(DB)-$(BACKUP_PROVIDER).yaml" || true # for idempotence kubectl --kubeconfig $(KUBECONFIG) apply -f "deploy/$(DB)-$(BACKUP_PROVIDER).yaml" # tailing - stern --kubeconfig $(KUBECONFIG) '.*' + kubectl stern --kubeconfig $(KUBECONFIG) '.*' .PHONY: kind-cluster-create kind-cluster-create: dockerimage diff --git a/README.md b/README.md index 44f450a..8689124 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ With `--compression-method` you can define how generated backups are compressed - S3 Buckets (tested against Ceph RADOS gateway) - Local +## Encryption + +For all three storage providers AES encryption is supported and can be enabled with `--encryption-key=`. +The key must be 32 bytes (AES-256) long. +The backups are stored at the storage provider with the `.aes` suffix. If the file does not have this suffix, decryption is skipped. + ## How it works ![Sequence Diagram](docs/sequence.drawio.svg) diff --git a/api/v1/backup.pb.go b/api/v1/backup.pb.go index ab51dca..1b5a772 100644 --- a/api/v1/backup.pb.go +++ b/api/v1/backup.pb.go @@ -254,6 +254,100 @@ func (*RestoreBackupResponse) Descriptor() ([]byte, []int) { return file_v1_backup_proto_rawDescGZIP(), []int{4} } +type GetBackupByVersionRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` +} + +func (x *GetBackupByVersionRequest) Reset() { + *x = GetBackupByVersionRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_backup_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetBackupByVersionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBackupByVersionRequest) ProtoMessage() {} + +func (x *GetBackupByVersionRequest) ProtoReflect() protoreflect.Message { + mi := &file_v1_backup_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBackupByVersionRequest.ProtoReflect.Descriptor instead. +func (*GetBackupByVersionRequest) Descriptor() ([]byte, []int) { + return file_v1_backup_proto_rawDescGZIP(), []int{5} +} + +func (x *GetBackupByVersionRequest) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +type GetBackupByVersionResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Backup *Backup `protobuf:"bytes,1,opt,name=backup,proto3" json:"backup,omitempty"` +} + +func (x *GetBackupByVersionResponse) Reset() { + *x = GetBackupByVersionResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_backup_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetBackupByVersionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBackupByVersionResponse) ProtoMessage() {} + +func (x *GetBackupByVersionResponse) ProtoReflect() protoreflect.Message { + mi := &file_v1_backup_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBackupByVersionResponse.ProtoReflect.Descriptor instead. +func (*GetBackupByVersionResponse) Descriptor() ([]byte, []int) { + return file_v1_backup_proto_rawDescGZIP(), []int{6} +} + +func (x *GetBackupByVersionResponse) GetBackup() *Backup { + if x != nil { + return x.Backup + } + return nil +} + var File_v1_backup_proto protoreflect.FileDescriptor var file_v1_backup_proto_rawDesc = []byte{ @@ -277,23 +371,36 @@ var file_v1_backup_proto_rawDesc = []byte{ 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x17, 0x0a, 0x15, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x94, 0x01, 0x0a, 0x0d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x42, - 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x12, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x67, 0x0a, 0x06, - 0x63, 0x6f, 0x6d, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x6c, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x72, - 0x6f, 0x70, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0xa2, - 0x02, 0x03, 0x56, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x56, 0x31, 0xca, 0x02, 0x02, 0x56, 0x31, 0xe2, - 0x02, 0x0e, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0xea, 0x02, 0x02, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x35, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x40, 0x0a, 0x1a, + 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x06, 0x62, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x76, 0x31, 0x2e, + 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x32, 0xe9, + 0x01, 0x0a, 0x0d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x3d, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x12, + 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x44, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x42, 0x79, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x67, 0x0a, 0x06, 0x63, 0x6f, + 0x6d, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x01, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x6d, 0x65, 0x74, 0x61, 0x6c, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x72, 0x6f, 0x70, + 0x74, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0xa2, 0x02, 0x03, + 0x56, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x56, 0x31, 0xca, 0x02, 0x02, 0x56, 0x31, 0xe2, 0x02, 0x0e, + 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, + 0x02, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -308,27 +415,32 @@ func file_v1_backup_proto_rawDescGZIP() []byte { return file_v1_backup_proto_rawDescData } -var file_v1_backup_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_v1_backup_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_v1_backup_proto_goTypes = []any{ - (*ListBackupsRequest)(nil), // 0: v1.ListBackupsRequest - (*BackupListResponse)(nil), // 1: v1.BackupListResponse - (*Backup)(nil), // 2: v1.Backup - (*RestoreBackupRequest)(nil), // 3: v1.RestoreBackupRequest - (*RestoreBackupResponse)(nil), // 4: v1.RestoreBackupResponse - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp + (*ListBackupsRequest)(nil), // 0: v1.ListBackupsRequest + (*BackupListResponse)(nil), // 1: v1.BackupListResponse + (*Backup)(nil), // 2: v1.Backup + (*RestoreBackupRequest)(nil), // 3: v1.RestoreBackupRequest + (*RestoreBackupResponse)(nil), // 4: v1.RestoreBackupResponse + (*GetBackupByVersionRequest)(nil), // 5: v1.GetBackupByVersionRequest + (*GetBackupByVersionResponse)(nil), // 6: v1.GetBackupByVersionResponse + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp } var file_v1_backup_proto_depIdxs = []int32{ 2, // 0: v1.BackupListResponse.backups:type_name -> v1.Backup - 5, // 1: v1.Backup.timestamp:type_name -> google.protobuf.Timestamp - 0, // 2: v1.BackupService.ListBackups:input_type -> v1.ListBackupsRequest - 3, // 3: v1.BackupService.RestoreBackup:input_type -> v1.RestoreBackupRequest - 1, // 4: v1.BackupService.ListBackups:output_type -> v1.BackupListResponse - 4, // 5: v1.BackupService.RestoreBackup:output_type -> v1.RestoreBackupResponse - 4, // [4:6] is the sub-list for method output_type - 2, // [2:4] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 7, // 1: v1.Backup.timestamp:type_name -> google.protobuf.Timestamp + 2, // 2: v1.GetBackupByVersionResponse.backup:type_name -> v1.Backup + 0, // 3: v1.BackupService.ListBackups:input_type -> v1.ListBackupsRequest + 3, // 4: v1.BackupService.RestoreBackup:input_type -> v1.RestoreBackupRequest + 5, // 5: v1.BackupService.GetBackupByVersion:input_type -> v1.GetBackupByVersionRequest + 1, // 6: v1.BackupService.ListBackups:output_type -> v1.BackupListResponse + 4, // 7: v1.BackupService.RestoreBackup:output_type -> v1.RestoreBackupResponse + 6, // 8: v1.BackupService.GetBackupByVersion:output_type -> v1.GetBackupByVersionResponse + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_v1_backup_proto_init() } @@ -397,6 +509,30 @@ func file_v1_backup_proto_init() { return nil } } + file_v1_backup_proto_msgTypes[5].Exporter = func(v any, i int) any { + switch v := v.(*GetBackupByVersionRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v1_backup_proto_msgTypes[6].Exporter = func(v any, i int) any { + switch v := v.(*GetBackupByVersionResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -404,7 +540,7 @@ func file_v1_backup_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_v1_backup_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, diff --git a/api/v1/backup_grpc.pb.go b/api/v1/backup_grpc.pb.go index 36e6361..3d48371 100644 --- a/api/v1/backup_grpc.pb.go +++ b/api/v1/backup_grpc.pb.go @@ -19,8 +19,9 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - BackupService_ListBackups_FullMethodName = "/v1.BackupService/ListBackups" - BackupService_RestoreBackup_FullMethodName = "/v1.BackupService/RestoreBackup" + BackupService_ListBackups_FullMethodName = "/v1.BackupService/ListBackups" + BackupService_RestoreBackup_FullMethodName = "/v1.BackupService/RestoreBackup" + BackupService_GetBackupByVersion_FullMethodName = "/v1.BackupService/GetBackupByVersion" ) // BackupServiceClient is the client API for BackupService service. @@ -29,6 +30,7 @@ const ( type BackupServiceClient interface { ListBackups(ctx context.Context, in *ListBackupsRequest, opts ...grpc.CallOption) (*BackupListResponse, error) RestoreBackup(ctx context.Context, in *RestoreBackupRequest, opts ...grpc.CallOption) (*RestoreBackupResponse, error) + GetBackupByVersion(ctx context.Context, in *GetBackupByVersionRequest, opts ...grpc.CallOption) (*GetBackupByVersionResponse, error) } type backupServiceClient struct { @@ -59,12 +61,23 @@ func (c *backupServiceClient) RestoreBackup(ctx context.Context, in *RestoreBack return out, nil } +func (c *backupServiceClient) GetBackupByVersion(ctx context.Context, in *GetBackupByVersionRequest, opts ...grpc.CallOption) (*GetBackupByVersionResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBackupByVersionResponse) + err := c.cc.Invoke(ctx, BackupService_GetBackupByVersion_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // BackupServiceServer is the server API for BackupService service. // All implementations should embed UnimplementedBackupServiceServer // for forward compatibility. type BackupServiceServer interface { ListBackups(context.Context, *ListBackupsRequest) (*BackupListResponse, error) RestoreBackup(context.Context, *RestoreBackupRequest) (*RestoreBackupResponse, error) + GetBackupByVersion(context.Context, *GetBackupByVersionRequest) (*GetBackupByVersionResponse, error) } // UnimplementedBackupServiceServer should be embedded to have @@ -80,6 +93,9 @@ func (UnimplementedBackupServiceServer) ListBackups(context.Context, *ListBackup func (UnimplementedBackupServiceServer) RestoreBackup(context.Context, *RestoreBackupRequest) (*RestoreBackupResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method RestoreBackup not implemented") } +func (UnimplementedBackupServiceServer) GetBackupByVersion(context.Context, *GetBackupByVersionRequest) (*GetBackupByVersionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetBackupByVersion not implemented") +} func (UnimplementedBackupServiceServer) testEmbeddedByValue() {} // UnsafeBackupServiceServer may be embedded to opt out of forward compatibility for this service. @@ -136,6 +152,24 @@ func _BackupService_RestoreBackup_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _BackupService_GetBackupByVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBackupByVersionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BackupServiceServer).GetBackupByVersion(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BackupService_GetBackupByVersion_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BackupServiceServer).GetBackupByVersion(ctx, req.(*GetBackupByVersionRequest)) + } + return interceptor(ctx, in, info, handler) +} + // BackupService_ServiceDesc is the grpc.ServiceDesc for BackupService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -151,6 +185,10 @@ var BackupService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RestoreBackup", Handler: _BackupService_RestoreBackup_Handler, }, + { + MethodName: "GetBackupByVersion", + Handler: _BackupService_GetBackupByVersion_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "v1/backup.proto", diff --git a/cmd/internal/backup/backup.go b/cmd/internal/backup/backup.go index cc71637..5c33061 100644 --- a/cmd/internal/backup/backup.go +++ b/cmd/internal/backup/backup.go @@ -10,6 +10,7 @@ import ( backuproviders "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/compress" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/encryption" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/metrics" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" cron "github.com/robfig/cron/v3" @@ -23,6 +24,7 @@ type BackuperConfig struct { BackupProvider backuproviders.BackupProvider Metrics *metrics.Metrics Compressor *compress.Compressor + Encrypter *encryption.Encrypter } type Backuper struct { @@ -33,6 +35,7 @@ type Backuper struct { metrics *metrics.Metrics comp *compress.Compressor sem *semaphore.Weighted + encrypter *encryption.Encrypter } func New(config *BackuperConfig) *Backuper { @@ -44,7 +47,8 @@ func New(config *BackuperConfig) *Backuper { metrics: config.Metrics, comp: config.Compressor, // sem guards backups to be taken concurrently - sem: semaphore.NewWeighted(1), + sem: semaphore.NewWeighted(1), + encrypter: config.Encrypter, } } @@ -105,6 +109,15 @@ func (b *Backuper) CreateBackup(ctx context.Context) error { b.log.Info("compressed backup") + if b.encrypter != nil { + filename, err = b.encrypter.Encrypt(filename) + if err != nil { + b.metrics.CountError("encrypt") + return fmt.Errorf("error encrypting backup: %w", err) + } + b.log.Info("encrypted backup") + } + err = b.bp.UploadBackup(ctx, filename) if err != nil { b.metrics.CountError("upload") diff --git a/cmd/internal/backup/providers/contract.go b/cmd/internal/backup/providers/contract.go index 30604d7..4356dbd 100644 --- a/cmd/internal/backup/providers/contract.go +++ b/cmd/internal/backup/providers/contract.go @@ -10,7 +10,7 @@ type BackupProvider interface { ListBackups(ctx context.Context) (BackupVersions, error) CleanupBackups(ctx context.Context) error GetNextBackupName(ctx context.Context) string - DownloadBackup(ctx context.Context, version *BackupVersion) error + DownloadBackup(ctx context.Context, version *BackupVersion, outDir string) (string, error) UploadBackup(ctx context.Context, sourcePath string) error } diff --git a/cmd/internal/backup/providers/gcp/gcp.go b/cmd/internal/backup/providers/gcp/gcp.go index 89cae60..cd3e51b 100644 --- a/cmd/internal/backup/providers/gcp/gcp.go +++ b/cmd/internal/backup/providers/gcp/gcp.go @@ -6,7 +6,6 @@ import ( "io" "log/slog" "net/http" - "path" "path/filepath" "strconv" "strings" @@ -147,11 +146,11 @@ func (b *BackupProviderGCP) CleanupBackups(_ context.Context) error { return nil } -// DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *providers.BackupVersion) error { +// DownloadBackup downloads the given backup version to the specified folder +func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outDir string) (string, error) { gen, err := strconv.ParseInt(version.Version, 10, 64) if err != nil { - return err + return "", err } bucket := b.c.Bucket(b.config.BucketName) @@ -160,28 +159,29 @@ func (b *BackupProviderGCP) DownloadBackup(ctx context.Context, version *provide if strings.Contains(downloadFileName, "/") { downloadFileName = filepath.Base(downloadFileName) } - backupFilePath := path.Join(constants.DownloadDir, downloadFileName) + + backupFilePath := filepath.Join(outDir, downloadFileName) b.log.Info("downloading", "object", version.Name, "gen", gen, "to", backupFilePath) r, err := bucket.Object(version.Name).Generation(gen).NewReader(ctx) if err != nil { - return fmt.Errorf("backup not found: %w", err) + return "", fmt.Errorf("backup not found: %w", err) } defer r.Close() f, err := b.fs.Create(backupFilePath) if err != nil { - return err + return "", err } defer f.Close() _, err = io.Copy(f, r) if err != nil { - return fmt.Errorf("error writing file from gcp to filesystem: %w", err) + return "", fmt.Errorf("error writing file from gcp to filesystem: %w", err) } - return nil + return backupFilePath, nil } // UploadBackup uploads a backup to the backup provider diff --git a/cmd/internal/backup/providers/gcp/gcp_integration_test.go b/cmd/internal/backup/providers/gcp/gcp_integration_test.go index ec87167..caf22dc 100644 --- a/cmd/internal/backup/providers/gcp/gcp_integration_test.go +++ b/cmd/internal/backup/providers/gcp/gcp_integration_test.go @@ -153,18 +153,17 @@ func Test_BackupProviderGCP(t *testing.T) { latestVersion := versions.Latest() require.NotNil(t, latestVersion) - err = p.DownloadBackup(ctx, latestVersion) + backupFilePath, err := p.DownloadBackup(ctx, latestVersion, "") require.NoError(t, err) - downloadPath := path.Join(constants.DownloadDir, expectedBackupName) - gotContent, err := afero.ReadFile(fs, downloadPath) + gotContent, err := afero.ReadFile(fs, backupFilePath) require.NoError(t, err) backupContent := fmt.Sprintf("precious data %d", backupAmount-1) require.Equal(t, backupContent, string(gotContent)) // cleaning up after test - err = fs.Remove(downloadPath) + err = fs.Remove(backupFilePath) require.NoError(t, err) }) diff --git a/cmd/internal/backup/providers/local/local.go b/cmd/internal/backup/providers/local/local.go index 8c2a7f4..8174003 100644 --- a/cmd/internal/backup/providers/local/local.go +++ b/cmd/internal/backup/providers/local/local.go @@ -85,19 +85,20 @@ func (b *BackupProviderLocal) CleanupBackups(_ context.Context) error { return nil } -// DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderLocal) DownloadBackup(_ context.Context, version *providers.BackupVersion) error { +// DownloadBackup downloads the given backup version to the specified folder +func (b *BackupProviderLocal) DownloadBackup(_ context.Context, version *providers.BackupVersion, outDir string) (string, error) { b.log.Info("download backup called for provider local") source := filepath.Join(b.config.LocalBackupPath, version.Name) - destination := filepath.Join(constants.DownloadDir, version.Name) - err := utils.Copy(b.fs, source, destination) + backupFilePath := filepath.Join(outDir, version.Name) + + err := utils.Copy(b.fs, source, backupFilePath) if err != nil { - return err + return "", err } - return nil + return backupFilePath, err } // UploadBackup uploads a backup to the backup provider diff --git a/cmd/internal/backup/providers/local/local_test.go b/cmd/internal/backup/providers/local/local_test.go index 44d528d..05286d6 100644 --- a/cmd/internal/backup/providers/local/local_test.go +++ b/cmd/internal/backup/providers/local/local_test.go @@ -130,17 +130,16 @@ func Test_BackupProviderLocal(t *testing.T) { latestVersion := versions.Latest() require.NotNil(t, latestVersion) - err = p.DownloadBackup(ctx, latestVersion) + backupFilePath, err := p.DownloadBackup(ctx, latestVersion, "") require.NoError(t, err) - downloadPath := path.Join(constants.DownloadDir, latestVersion.Name) - gotContent, err := afero.ReadFile(fs, downloadPath) + gotContent, err := afero.ReadFile(fs, backupFilePath) require.NoError(t, err) require.Equal(t, fmt.Sprintf("precious data %d", backupAmount), string(gotContent)) // cleaning up after test - err = fs.Remove(downloadPath) + err = fs.Remove(backupFilePath) require.NoError(t, err) }) diff --git a/cmd/internal/backup/providers/s3/s3.go b/cmd/internal/backup/providers/s3/s3.go index 10200cd..7634a95 100644 --- a/cmd/internal/backup/providers/s3/s3.go +++ b/cmd/internal/backup/providers/s3/s3.go @@ -3,7 +3,6 @@ package s3 import ( "context" "log/slog" - "path" "path/filepath" "strings" @@ -189,8 +188,8 @@ func (b *BackupProviderS3) CleanupBackups(_ context.Context) error { return nil } -// DownloadBackup downloads the given backup version to the restoration folder -func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *providers.BackupVersion) error { +// DownloadBackup downloads the given backup version to the specified folder +func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outDir string) (string, error) { bucket := aws.String(b.config.BucketName) downloadFileName := version.Name @@ -198,11 +197,11 @@ func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *provider downloadFileName = filepath.Base(downloadFileName) } - backupFilePath := path.Join(constants.DownloadDir, downloadFileName) + backupFilePath := filepath.Join(outDir, downloadFileName) f, err := b.fs.Create(backupFilePath) if err != nil { - return err + return "", err } defer f.Close() @@ -217,10 +216,10 @@ func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *provider VersionId: &version.Version, }) if err != nil { - return err + return "", err } - return nil + return backupFilePath, nil } // UploadBackup uploads a backup to the backup provider diff --git a/cmd/internal/backup/providers/s3/s3_integration_test.go b/cmd/internal/backup/providers/s3/s3_integration_test.go index 4efcbaa..20c3e03 100644 --- a/cmd/internal/backup/providers/s3/s3_integration_test.go +++ b/cmd/internal/backup/providers/s3/s3_integration_test.go @@ -147,18 +147,17 @@ func Test_BackupProviderS3(t *testing.T) { latestVersion := versions.Latest() require.NotNil(t, latestVersion) - err = p.DownloadBackup(ctx, latestVersion) + backupFilePath, err := p.DownloadBackup(ctx, latestVersion, "") require.NoError(t, err) - downloadPath := path.Join(constants.DownloadDir, expectedBackupName) - gotContent, err := afero.ReadFile(fs, downloadPath) + gotContent, err := afero.ReadFile(fs, backupFilePath) require.NoError(t, err) backupContent := fmt.Sprintf("precious data %d", backupAmount-1) require.Equal(t, backupContent, string(gotContent)) // cleaning up after test - err = fs.Remove(downloadPath) + err = fs.Remove(backupFilePath) require.NoError(t, err) }) diff --git a/cmd/internal/encryption/encryption.go b/cmd/internal/encryption/encryption.go new file mode 100644 index 0000000..65b0c85 --- /dev/null +++ b/cmd/internal/encryption/encryption.go @@ -0,0 +1,238 @@ +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "unicode" + + "github.com/spf13/afero" +) + +// suffix is appended on encryption and removed on decryption from given input +const suffix = ".aes" + +// Encrypter is used to encrypt/decrypt backups +type Encrypter struct { + fs afero.Fs + key string + log *slog.Logger +} + +type EncrypterConfig struct { + FS afero.Fs + Key string +} + +// New creates a new Encrypter with the given key. +// The key should be 32 bytes (AES-256) +func New(log *slog.Logger, config *EncrypterConfig) (*Encrypter, error) { + if len(config.Key) != 32 { + return nil, fmt.Errorf("key length: %d invalid, must be 32 bytes", len(config.Key)) + } + if !isASCII(config.Key) { + return nil, fmt.Errorf("key must only contain ascii characters") + } + if config.FS == nil { + config.FS = afero.NewOsFs() + } + + return &Encrypter{ + log: log, + key: config.Key, + fs: config.FS, + }, nil + +} + +// Encrypt input file with key and store encrypted result with suffix +func (e *Encrypter) Encrypt(inputPath string) (string, error) { + output := inputPath + suffix + e.log.Debug("encrypt", "input", inputPath, "output", output) + infile, err := e.fs.Open(inputPath) + if err != nil { + return "", err + } + defer infile.Close() + + block, err := e.createCipher() + if err != nil { + return "", err + } + + iv, err := e.generateIV(block) + if err != nil { + return "", err + } + + outfile, err := e.openOutputFile(output) + if err != nil { + return "", err + } + defer outfile.Close() + + if err := e.encryptFile(infile, outfile, block, iv); err != nil { + return "", err + } + + if err := e.fs.Remove(inputPath); err != nil { + e.log.Warn("unable to remove input", "error", err) + } + + return output, nil +} + +// Decrypt input file with key and store decrypted result with suffix removed +// if input does not end with suffix, it is assumed that the file was not encrypted. +func (e *Encrypter) Decrypt(inputPath string) (string, error) { + output := strings.TrimSuffix(inputPath, suffix) + e.log.Debug("decrypt", "input", inputPath, "output", output) + + if !IsEncrypted(inputPath) { + return "", fmt.Errorf("input is not encrypted") + } + + infile, err := e.fs.Open(inputPath) + if err != nil { + return "", err + } + defer infile.Close() + + block, err := e.createCipher() + if err != nil { + return "", err + } + + iv, msgLen, err := e.readIVAndMessageLength(infile, block) + if err != nil { + return "", err + } + + outfile, err := e.openOutputFile(output) + if err != nil { + return "", err + } + + if err := e.decryptFile(infile, outfile, block, iv, msgLen); err != nil { + return "", err + } + + if err := e.fs.Remove(inputPath); err != nil { + e.log.Warn("unable to remove input", "error", err) + } + return output, nil +} + +func isASCII(s string) bool { + for _, c := range s { + if c > unicode.MaxASCII { + return false + } + } + return true +} + +// createCipher() returns new cipher block for encryption/decryption based on encryption-key +func (e *Encrypter) createCipher() (cipher.Block, error) { + key := []byte(e.key) + return aes.NewCipher(key) +} + +func (e *Encrypter) openOutputFile(output string) (afero.File, error) { + return e.fs.OpenFile(output, os.O_RDWR|os.O_CREATE, 0644) +} + +// generateIV() returns unique initalization vector of same size as cipher block for encryption +func (e *Encrypter) generateIV(block cipher.Block) ([]byte, error) { + iv := make([]byte, block.BlockSize()) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return nil, err + } + return iv, nil +} + +// encryptFile() encrypts infile to outfile using CTR mode (cipher and iv) and appends iv for decryption +func (e *Encrypter) encryptFile(infile, outfile afero.File, block cipher.Block, iv []byte) error { + buf := make([]byte, 1024) + stream := cipher.NewCTR(block, iv) + + for { + n, err := infile.Read(buf) + if n > 0 { + stream.XORKeyStream(buf, buf[:n]) + if _, err := outfile.Write(buf[:n]); err != nil { + return err + } + } + + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("error reading from file (%d bytes read): %w", n, err) + } + } + + if _, err := outfile.Write(iv); err != nil { + return fmt.Errorf("could not append iv: %w", err) + } + + return nil +} + +// IsEncrypted() tests if target file is encrypted +func IsEncrypted(path string) bool { + return filepath.Ext(path) == suffix +} + +// readIVAndMessageLength() returns initialization vector and message length for decryption +func (e *Encrypter) readIVAndMessageLength(infile afero.File, block cipher.Block) ([]byte, int64, error) { + fi, err := infile.Stat() + if err != nil { + return nil, 0, err + } + + iv := make([]byte, block.BlockSize()) + msgLen := fi.Size() - int64(len(iv)) + if _, err := infile.ReadAt(iv, msgLen); err != nil { + return nil, 0, err + } + + return iv, msgLen, nil +} + +// decryptFile() decrypts infile to outfile using CTR mode (cipher and iv) +func (e *Encrypter) decryptFile(infile, outfile afero.File, block cipher.Block, iv []byte, msgLen int64) error { + buf := make([]byte, 1024) + stream := cipher.NewCTR(block, iv) + + for { + n, err := infile.Read(buf) + if n > 0 { + if n > int(msgLen) { + n = int(msgLen) + } + msgLen -= int64(n) + stream.XORKeyStream(buf, buf[:n]) + if _, err := outfile.Write(buf[:n]); err != nil { + return err + } + } + + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("error reading from file (%d bytes read): %w", n, err) + } + } + + return nil +} diff --git a/cmd/internal/encryption/encryption_test.go b/cmd/internal/encryption/encryption_test.go new file mode 100644 index 0000000..bbc3823 --- /dev/null +++ b/cmd/internal/encryption/encryption_test.go @@ -0,0 +1,62 @@ +package encryption + +import ( + "log/slog" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestEncrypter(t *testing.T) { + fs := afero.NewMemMapFs() + + // Key too short + _, err := New(slog.Default(), &EncrypterConfig{Key: "tooshortkey", FS: fs}) + require.EqualError(t, err, "key length: 11 invalid, must be 32 bytes") + + // Key too long + _, err = New(slog.Default(), &EncrypterConfig{Key: "toolooooooooooooooooooooooooooooooooongkey", FS: fs}) + require.EqualError(t, err, "key length: 42 invalid, must be 32 bytes") + + _, err = New(slog.Default(), &EncrypterConfig{Key: "äöüäöüäöüäöüäöüä", FS: fs}) + require.EqualError(t, err, "key must only contain ascii characters") + + e, err := New(slog.Default(), &EncrypterConfig{Key: "01234567891234560123456789123456", FS: fs}) + require.NoError(t, err, "") + + input, err := fs.Create("encrypt") + require.NoError(t, err) + + cleartextInput := []byte("This is the content of the file") + err = afero.WriteFile(fs, input.Name(), cleartextInput, 0600) + require.NoError(t, err) + output, err := e.Encrypt(input.Name()) + require.NoError(t, err) + encryptedText, err := afero.ReadFile(fs, output) + require.NoError(t, err) + + require.Equal(t, input.Name()+suffix, output) + require.NotEqual(t, cleartextInput, encryptedText) + + cleartextFile, err := e.Decrypt(output) + require.NoError(t, err) + cleartext, err := afero.ReadFile(fs, cleartextFile) + require.NoError(t, err) + require.Equal(t, cleartextInput, cleartext) + + // Test with 100MB file + bigBuff := make([]byte, 100000000) + err = afero.WriteFile(fs, "bigfile.test", bigBuff, 0600) + require.NoError(t, err) + + bigEncFile, err := e.Encrypt("bigfile.test") + require.NoError(t, err) + _, err = e.Decrypt(bigEncFile) + require.NoError(t, err) + + err = fs.Remove(input.Name()) + require.NoError(t, err) + err = fs.Remove("bigfile.test") + require.NoError(t, err) +} diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index 135645d..2f11fbb 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -15,6 +15,7 @@ import ( "github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/compress" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/encryption" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/metrics" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" @@ -34,9 +35,10 @@ type Initializer struct { comp *compress.Compressor metrics *metrics.Metrics dbDataDir string + encrypter *encryption.Encrypter } -func New(log *slog.Logger, addr string, db database.Database, bp providers.BackupProvider, comp *compress.Compressor, metrics *metrics.Metrics, dbDataDir string) *Initializer { +func New(log *slog.Logger, addr string, db database.Database, bp providers.BackupProvider, comp *compress.Compressor, metrics *metrics.Metrics, dbDataDir string, encrypter *encryption.Encrypter) *Initializer { return &Initializer{ currentStatus: &v1.StatusResponse{ Status: v1.StatusResponse_CHECKING, @@ -49,6 +51,7 @@ func New(log *slog.Logger, addr string, db database.Database, bp providers.Backu comp: comp, dbDataDir: dbDataDir, metrics: metrics, + encrypter: encrypter, } } @@ -136,6 +139,12 @@ func (i *Initializer) initialize(ctx context.Context) error { return fmt.Errorf("unable to ensure backup bucket: %w", err) } + i.log.Info("ensuring default download directory") + err = os.MkdirAll(constants.DownloadDir, 0755) + if err != nil { + return fmt.Errorf("unable to ensure default download directory: %w", err) + } + i.log.Info("checking database") i.currentStatus.Status = v1.StatusResponse_CHECKING i.currentStatus.Message = "checking database" @@ -163,6 +172,12 @@ func (i *Initializer) initialize(ctx context.Context) error { return nil } + if i.encrypter == nil { + if encryption.IsEncrypted(latestBackup.Name) { + return fmt.Errorf("latest backup is encrypted, but no encryption/decryption is configured") + } + } + err = i.Restore(ctx, latestBackup) if err != nil { return fmt.Errorf("unable to restore database: %w", err) @@ -197,11 +212,22 @@ func (i *Initializer) Restore(ctx context.Context, version *providers.BackupVers return fmt.Errorf("could not delete priorly downloaded file: %w", err) } - err := i.bp.DownloadBackup(ctx, version) + backupFilePath, err := i.bp.DownloadBackup(ctx, version, constants.DownloadDir) if err != nil { return fmt.Errorf("unable to download backup: %w", err) } + if i.encrypter != nil { + if encryption.IsEncrypted(backupFilePath) { + backupFilePath, err = i.encrypter.Decrypt(backupFilePath) + if err != nil { + return fmt.Errorf("unable to decrypt backup: %w", err) + } + } else { + i.log.Info("restoring unencrypted backup with configured encryption - skipping decryption...") + } + } + i.currentStatus.Message = "uncompressing backup" err = i.comp.Decompress(backupFilePath) if err != nil { diff --git a/cmd/internal/initializer/service.go b/cmd/internal/initializer/service.go index df8c7b2..fdc70ad 100644 --- a/cmd/internal/initializer/service.go +++ b/cmd/internal/initializer/service.go @@ -84,6 +84,25 @@ func (s *backupService) RestoreBackup(ctx context.Context, req *v1.RestoreBackup return &v1.RestoreBackupResponse{}, nil } +func (s *backupService) GetBackupByVersion(ctx context.Context, req *v1.GetBackupByVersionRequest) (*v1.GetBackupByVersionResponse, error) { + if req.GetVersion() == "" { + return nil, status.Error(codes.InvalidArgument, "version to get must be defined explicitly") + } + + versions, err := s.bp.ListBackups(ctx) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + version, err := versions.Get(req.GetVersion()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return &v1.GetBackupByVersionResponse{Backup: &v1.Backup{Name: version.Name, Version: version.Version, Timestamp: timestamppb.New(version.Date)}}, nil + +} + type databaseService struct { backupFn func() error } diff --git a/cmd/main.go b/cmd/main.go index 25c2df8..74f9414 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -23,6 +23,7 @@ import ( "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/postgres" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/redis" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/rethinkdb" + "github.com/metal-stack/backup-restore-sidecar/cmd/internal/encryption" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/initializer" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/metrics" "github.com/metal-stack/backup-restore-sidecar/cmd/internal/probe" @@ -93,14 +94,19 @@ const ( s3SecretKeyFlg = "s3-secret-key" compressionMethod = "compression-method" + + encryptionKeyFlg = "encryption-key" + + downloadOutputFlg = "output" ) var ( - cfgFile string - logger *slog.Logger - db database.Database - bp providers.BackupProvider - stop context.Context + cfgFile string + logger *slog.Logger + db database.Database + bp providers.BackupProvider + encrypter *encryption.Encrypter + stop context.Context ) var rootCmd = &cobra.Command{ @@ -124,6 +130,9 @@ var startCmd = &cobra.Command{ if err := initDatabase(); err != nil { return err } + if err := initEncrypter(); err != nil { + return err + } return initBackupProvider() }, RunE: func(cmd *cobra.Command, args []string) error { @@ -157,9 +166,10 @@ var startCmd = &cobra.Command{ BackupProvider: bp, Metrics: metrics, Compressor: comp, + Encrypter: encrypter, }) - if err := initializer.New(logger.WithGroup("initializer"), addr, db, bp, comp, metrics, viper.GetString(databaseDatadirFlg)).Start(stop, backuper); err != nil { + if err := initializer.New(logger.WithGroup("initializer"), addr, db, bp, comp, metrics, viper.GetString(databaseDatadirFlg), encrypter).Start(stop, backuper); err != nil { return err } @@ -262,6 +272,54 @@ var waitCmd = &cobra.Command{ }, } +var downloadBackupCmd = &cobra.Command{ + Use: "download ", + Short: "downloads backup without restoring", + PreRunE: func(cm *cobra.Command, args []string) error { + err := initEncrypter() + if err != nil { + return err + } + return initBackupProvider() + }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("no version argument specified") + } + + c, err := client.New(context.Background(), viper.GetString(serverAddrFlg)) + if err != nil { + return fmt.Errorf("error creating client: %w", err) + } + + backup, err := c.BackupServiceClient().GetBackupByVersion(context.Background(), &v1.GetBackupByVersionRequest{Version: args[0]}) + + if err != nil { + return fmt.Errorf("error getting backup by version: %w", err) + } + + output := viper.GetString(downloadOutputFlg) + + destination, err := bp.DownloadBackup(context.Background(), &providers.BackupVersion{Name: backup.GetBackup().GetName()}, output) + + if err != nil { + return fmt.Errorf("failed downloading backup: %w", err) + } + + if encrypter != nil { + if encryption.IsEncrypted(destination) { + _, err = encrypter.Decrypt(destination) + if err != nil { + return fmt.Errorf("unable to decrypt backup: %w", err) + } + } else { + logger.Info("downloading unencrypted backup with configured encryption - skipping decryption...") + } + } + return nil + }, +} + func main() { if err := rootCmd.Execute(); err != nil { if logger == nil { @@ -273,7 +331,7 @@ func main() { } func init() { - rootCmd.AddCommand(startCmd, waitCmd, restoreCmd, createBackupCmd) + rootCmd.AddCommand(startCmd, waitCmd, restoreCmd, createBackupCmd, downloadBackupCmd) rootCmd.PersistentFlags().StringP(logLevelFlg, "", "info", "sets the application log level") rootCmd.PersistentFlags().StringP(databaseFlg, "", "", "the kind of the database [postgres|rethinkdb|etcd|meilisearch|redis|keydb|localfs]") @@ -322,6 +380,8 @@ func init() { startCmd.Flags().StringP(compressionMethod, "", "targz", "the compression method to use to compress the backups (tar|targz|tarlz4)") + startCmd.Flags().StringP(encryptionKeyFlg, "", "", "the encryption key for aes") + err = viper.BindPFlags(startCmd.Flags()) if err != nil { fmt.Printf("unable to construct initializer command: %v", err) @@ -339,6 +399,13 @@ func init() { } restoreCmd.AddCommand(restoreListCmd) + + downloadBackupCmd.Flags().StringP(downloadOutputFlg, "o", constants.DownloadDir, "the target directory for the downloaded backup") + err = viper.BindPFlags(downloadBackupCmd.Flags()) + if err != nil { + fmt.Printf("unable to construct download command: %v", err) + os.Exit(1) + } } func initConfig() { @@ -476,6 +543,19 @@ func initDatabase() error { return nil } +func initEncrypter() error { + var err error + key := viper.GetString(encryptionKeyFlg) + if key != "" { + encrypter, err = encryption.New(logger.WithGroup("encrypter"), &encryption.EncrypterConfig{Key: key}) + if err != nil { + return fmt.Errorf("unable to initialize encrypter: %w", err) + } + logger.Info("initialized encrypter") + } + return nil +} + func initBackupProvider() error { bpString := viper.GetString(backupProviderFlg) var err error diff --git a/deploy/etcd-local.yaml b/deploy/etcd-local.yaml index b9de8a6..a1274d5 100644 --- a/deploy/etcd-local.yaml +++ b/deploy/etcd-local.yaml @@ -145,6 +145,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: etcd-test etcd-endpoints: http://localhost:32379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - etcd --data-dir=/data/etcd --listen-client-urls http://0.0.0.0:32379 --advertise-client-urls http://0.0.0.0:32379 --listen-peer-urls http://0.0.0.0:32380 --initial-advertise-peer-urls http://0.0.0.0:32380 --initial-cluster default=http://0.0.0.0:32380 --listen-metrics-urls http://0.0.0.0:32381 kind: ConfigMap diff --git a/deploy/keydb-local.yaml b/deploy/keydb-local.yaml index 8487f2a..d8288dd 100644 --- a/deploy/keydb-local.yaml +++ b/deploy/keydb-local.yaml @@ -140,6 +140,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: keydb-test redis-addr: localhost:6379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - keydb-server kind: ConfigMap diff --git a/deploy/localfs-local.yaml b/deploy/localfs-local.yaml index 103395e..7174f43 100644 --- a/deploy/localfs-local.yaml +++ b/deploy/localfs-local.yaml @@ -116,6 +116,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: localfs-test redis-addr: localhost:6379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - tail -f /etc/hosts kind: ConfigMap diff --git a/deploy/meilisearch-local.yaml b/deploy/meilisearch-local.yaml index 320a519..72c6c74 100644 --- a/deploy/meilisearch-local.yaml +++ b/deploy/meilisearch-local.yaml @@ -157,6 +157,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: meilisearch-test compression-method: targz + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files kind: ConfigMap diff --git a/deploy/postgres-local.yaml b/deploy/postgres-local.yaml index 12c18bb..443d44e 100644 --- a/deploy/postgres-local.yaml +++ b/deploy/postgres-local.yaml @@ -184,6 +184,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: postgres-test compression-method: tar + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - docker-entrypoint.sh postgres kind: ConfigMap diff --git a/deploy/redis-local.yaml b/deploy/redis-local.yaml index 14b0d9b..101f0bb 100644 --- a/deploy/redis-local.yaml +++ b/deploy/redis-local.yaml @@ -140,6 +140,7 @@ data: backup-cron-schedule: "*/1 * * * *" object-prefix: redis-test redis-addr: localhost:6379 + encryption-key: "01234567891234560123456789123456" post-exec-cmds: - redis-server kind: ConfigMap diff --git a/deploy/rethinkdb-local.yaml b/deploy/rethinkdb-local.yaml index ad1f7bb..8d9465f 100644 --- a/deploy/rethinkdb-local.yaml +++ b/deploy/rethinkdb-local.yaml @@ -141,6 +141,7 @@ data: rethinkdb-passwordfile: /rethinkdb-secret/rethinkdb-password.txt backup-cron-schedule: "*/1 * * * *" object-prefix: rethinkdb-test + encryption-key: "01234567891234560123456789123456" post-exec-cmds: # IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location - rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} diff --git a/docs/sequence.drawio.svg b/docs/sequence.drawio.svg index 6c39927..94003db 100644 --- a/docs/sequence.drawio.svg +++ b/docs/sequence.drawio.svg @@ -1,4 +1,4 @@ - + @@ -73,18 +73,18 @@ -
+
- uncompress backup archive + (decrypt), uncompress backup archive
and restore database
- - uncompress backup archive... + + (decrypt), uncompress backup archive... @@ -226,7 +226,7 @@ -
+
backup version @@ -234,7 +234,7 @@
- + backup version @@ -244,7 +244,7 @@ -
+
download backup version @@ -252,17 +252,17 @@
- + download backup version - - + + -
+
backup archive @@ -270,7 +270,7 @@
- + backup archive @@ -327,7 +327,7 @@ -
+
probe @@ -335,7 +335,7 @@
- + probe @@ -381,17 +381,19 @@ -
+
compress to
- backup archive + backup archive, +
+ (encrypt)
- + compress to... @@ -474,7 +476,7 @@ -
+
status checking @@ -482,7 +484,7 @@
- + status checking @@ -654,4 +656,4 @@ - + \ No newline at end of file diff --git a/go.sum b/go.sum index b76fe99..50d4e78 100644 --- a/go.sum +++ b/go.sum @@ -306,8 +306,6 @@ github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/j github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= -github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= diff --git a/integration/main_test.go b/integration/main_test.go index c29fec9..a9d089e 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -143,6 +143,8 @@ func restoreFlow(t *testing.T, spec *flowSpec) { require.NoError(t, err) require.NotNil(t, backup) + require.True(t, strings.HasSuffix(backup.GetName(), ".aes")) + t.Log("remove sts and delete data volume") err = c.Delete(ctx, spec.sts(ns.Name)) diff --git a/pkg/generate/examples/examples/etcd.go b/pkg/generate/examples/examples/etcd.go index d76cc1c..6c28ea4 100644 --- a/pkg/generate/examples/examples/etcd.go +++ b/pkg/generate/examples/examples/etcd.go @@ -246,6 +246,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: etcd-test etcd-endpoints: http://localhost:32379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - etcd --data-dir=/data/etcd --listen-client-urls http://0.0.0.0:32379 --advertise-client-urls http://0.0.0.0:32379 --listen-peer-urls http://0.0.0.0:32380 --initial-advertise-peer-urls http://0.0.0.0:32380 --initial-cluster default=http://0.0.0.0:32380 --listen-metrics-urls http://0.0.0.0:32381 `, diff --git a/pkg/generate/examples/examples/keydb.go b/pkg/generate/examples/examples/keydb.go index 706768c..dff05b2 100644 --- a/pkg/generate/examples/examples/keydb.go +++ b/pkg/generate/examples/examples/keydb.go @@ -238,6 +238,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: keydb-test redis-addr: localhost:6379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - keydb-server `, diff --git a/pkg/generate/examples/examples/localfs.go b/pkg/generate/examples/examples/localfs.go index 90aa1a6..1af850b 100644 --- a/pkg/generate/examples/examples/localfs.go +++ b/pkg/generate/examples/examples/localfs.go @@ -207,6 +207,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: localfs-test redis-addr: localhost:6379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - tail -f /etc/hosts `, diff --git a/pkg/generate/examples/examples/meilisearch.go b/pkg/generate/examples/examples/meilisearch.go index 39db910..ece9749 100644 --- a/pkg/generate/examples/examples/meilisearch.go +++ b/pkg/generate/examples/examples/meilisearch.go @@ -279,6 +279,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: meilisearch-test compression-method: targz +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - meilisearch --db-path=/data/data.ms/ --dump-dir=/backup/upload/files `, diff --git a/pkg/generate/examples/examples/postgres.go b/pkg/generate/examples/examples/postgres.go index 4be6afc..96149cb 100644 --- a/pkg/generate/examples/examples/postgres.go +++ b/pkg/generate/examples/examples/postgres.go @@ -309,6 +309,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: postgres-test compression-method: tar +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - docker-entrypoint.sh postgres `, diff --git a/pkg/generate/examples/examples/redis.go b/pkg/generate/examples/examples/redis.go index ff8ec12..e21b32f 100644 --- a/pkg/generate/examples/examples/redis.go +++ b/pkg/generate/examples/examples/redis.go @@ -238,6 +238,7 @@ backup-provider: local backup-cron-schedule: "*/1 * * * *" object-prefix: redis-test redis-addr: localhost:6379 +encryption-key: "01234567891234560123456789123456" post-exec-cmds: - redis-server `, diff --git a/pkg/generate/examples/examples/rethinkdb.go b/pkg/generate/examples/examples/rethinkdb.go index dd0305d..a5ac7e9 100644 --- a/pkg/generate/examples/examples/rethinkdb.go +++ b/pkg/generate/examples/examples/rethinkdb.go @@ -261,6 +261,7 @@ backup-provider: local rethinkdb-passwordfile: /rethinkdb-secret/rethinkdb-password.txt backup-cron-schedule: "*/1 * * * *" object-prefix: rethinkdb-test +encryption-key: "01234567891234560123456789123456" post-exec-cmds: # IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location - rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} diff --git a/proto/v1/backup.proto b/proto/v1/backup.proto index 1eb0590..c1e9f99 100644 --- a/proto/v1/backup.proto +++ b/proto/v1/backup.proto @@ -7,6 +7,7 @@ import "google/protobuf/timestamp.proto"; service BackupService { rpc ListBackups(ListBackupsRequest) returns (BackupListResponse); rpc RestoreBackup(RestoreBackupRequest) returns (RestoreBackupResponse); + rpc GetBackupByVersion(GetBackupByVersionRequest) returns (GetBackupByVersionResponse); } message ListBackupsRequest {} @@ -26,3 +27,11 @@ message RestoreBackupRequest { } message RestoreBackupResponse {} + +message GetBackupByVersionRequest { + string version = 1; +} + +message GetBackupByVersionResponse { + Backup backup = 1; +}