From f2440cd33569a5b1af50418697fec2d9ac465ced Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 5 Mar 2025 10:43:12 +0100 Subject: [PATCH 01/78] feat(template-manager): build logs stored wwith corresponding buildID and envID --- .../internal/server/create_template.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 7a860cb6c..887c9dbb3 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -41,13 +41,11 @@ func (s *serverStore) TemplateCreate(templateRequest *template_manager.TemplateC attribute.Bool("env.huge_pages", config.HugePages), ) - logsWriter := writer.New( - stream, - s.buildLogger. - With(zap.Field{Type: zapcore.StringType, Key: "envID", String: config.TemplateID}). - With(zap.Field{Type: zapcore.StringType, Key: "buildID", String: config.BuildID}), - ) + loggerWithMetadata := s.logger. + With(zap.Field{Type: zapcore.StringType, Key: "envID", String: config.TemplateID}). + With(zap.Field{Type: zapcore.StringType, Key: "buildID", String: config.BuildID}) + logsWriter := writer.New(stream, loggerWithMetadata) template := &build.Env{ TemplateFiles: storage.NewTemplateFiles( config.TemplateID, From 259972441d92a9aad83e03d102e3824e02037197 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 5 Mar 2025 11:43:07 +0100 Subject: [PATCH 02/78] chore(template-manager): separate system and build stauts loggers --- .../internal/server/create_template.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 887c9dbb3..7a860cb6c 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -41,11 +41,13 @@ func (s *serverStore) TemplateCreate(templateRequest *template_manager.TemplateC attribute.Bool("env.huge_pages", config.HugePages), ) - loggerWithMetadata := s.logger. - With(zap.Field{Type: zapcore.StringType, Key: "envID", String: config.TemplateID}). - With(zap.Field{Type: zapcore.StringType, Key: "buildID", String: config.BuildID}) + logsWriter := writer.New( + stream, + s.buildLogger. + With(zap.Field{Type: zapcore.StringType, Key: "envID", String: config.TemplateID}). + With(zap.Field{Type: zapcore.StringType, Key: "buildID", String: config.BuildID}), + ) - logsWriter := writer.New(stream, loggerWithMetadata) template := &build.Env{ TemplateFiles: storage.NewTemplateFiles( config.TemplateID, From b3869c4825264e40ca6040559c66a60e7b0fed78 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 5 Mar 2025 18:00:06 +0100 Subject: [PATCH 03/78] feat(template-manager): separate build trigger and status scheck apis --- .../template-manager/template-manager.pb.go | 291 ++++++++++++++---- .../template-manager_grpc.pb.go | 84 +++-- .../template-manager/template-manager.proto | 23 +- 3 files changed, 315 insertions(+), 83 deletions(-) diff --git a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go index 8c9e7f8ed..8f8b0dcea 100644 --- a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go +++ b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go @@ -179,6 +179,61 @@ func (x *TemplateCreateRequest) GetTemplate() *TemplateConfig { return nil } +type TemplateStatusRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TemplateID string `protobuf:"bytes,1,opt,name=templateID,proto3" json:"templateID,omitempty"` + BuildID string `protobuf:"bytes,2,opt,name=buildID,proto3" json:"buildID,omitempty"` +} + +func (x *TemplateStatusRequest) Reset() { + *x = TemplateStatusRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_template_manager_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TemplateStatusRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TemplateStatusRequest) ProtoMessage() {} + +func (x *TemplateStatusRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_manager_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TemplateStatusRequest.ProtoReflect.Descriptor instead. +func (*TemplateStatusRequest) Descriptor() ([]byte, []int) { + return file_template_manager_proto_rawDescGZIP(), []int{2} +} + +func (x *TemplateStatusRequest) GetTemplateID() string { + if x != nil { + return x.TemplateID + } + return "" +} + +func (x *TemplateStatusRequest) GetBuildID() string { + if x != nil { + return x.BuildID + } + return "" +} + // Data required for deleting a template. type TemplateBuildDeleteRequest struct { state protoimpl.MessageState @@ -191,7 +246,7 @@ type TemplateBuildDeleteRequest struct { func (x *TemplateBuildDeleteRequest) Reset() { *x = TemplateBuildDeleteRequest{} if protoimpl.UnsafeEnabled { - mi := &file_template_manager_proto_msgTypes[2] + mi := &file_template_manager_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -204,7 +259,7 @@ func (x *TemplateBuildDeleteRequest) String() string { func (*TemplateBuildDeleteRequest) ProtoMessage() {} func (x *TemplateBuildDeleteRequest) ProtoReflect() protoreflect.Message { - mi := &file_template_manager_proto_msgTypes[2] + mi := &file_template_manager_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -217,7 +272,7 @@ func (x *TemplateBuildDeleteRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TemplateBuildDeleteRequest.ProtoReflect.Descriptor instead. func (*TemplateBuildDeleteRequest) Descriptor() ([]byte, []int) { - return file_template_manager_proto_rawDescGZIP(), []int{2} + return file_template_manager_proto_rawDescGZIP(), []int{3} } func (x *TemplateBuildDeleteRequest) GetBuildID() string { @@ -227,32 +282,32 @@ func (x *TemplateBuildDeleteRequest) GetBuildID() string { return "" } -// Logs from template build -type TemplateBuildLog struct { +type TemplateBuildMetadata struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Log string `protobuf:"bytes,1,opt,name=log,proto3" json:"log,omitempty"` + RootfsSizeKey int32 `protobuf:"varint,3,opt,name=rootfsSizeKey,proto3" json:"rootfsSizeKey,omitempty"` + EnvdVersionKey string `protobuf:"bytes,4,opt,name=envdVersionKey,proto3" json:"envdVersionKey,omitempty"` } -func (x *TemplateBuildLog) Reset() { - *x = TemplateBuildLog{} +func (x *TemplateBuildMetadata) Reset() { + *x = TemplateBuildMetadata{} if protoimpl.UnsafeEnabled { - mi := &file_template_manager_proto_msgTypes[3] + mi := &file_template_manager_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *TemplateBuildLog) String() string { +func (x *TemplateBuildMetadata) String() string { return protoimpl.X.MessageStringOf(x) } -func (*TemplateBuildLog) ProtoMessage() {} +func (*TemplateBuildMetadata) ProtoMessage() {} -func (x *TemplateBuildLog) ProtoReflect() protoreflect.Message { - mi := &file_template_manager_proto_msgTypes[3] +func (x *TemplateBuildMetadata) ProtoReflect() protoreflect.Message { + mi := &file_template_manager_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -263,18 +318,89 @@ func (x *TemplateBuildLog) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use TemplateBuildLog.ProtoReflect.Descriptor instead. -func (*TemplateBuildLog) Descriptor() ([]byte, []int) { - return file_template_manager_proto_rawDescGZIP(), []int{3} +// Deprecated: Use TemplateBuildMetadata.ProtoReflect.Descriptor instead. +func (*TemplateBuildMetadata) Descriptor() ([]byte, []int) { + return file_template_manager_proto_rawDescGZIP(), []int{4} } -func (x *TemplateBuildLog) GetLog() string { +func (x *TemplateBuildMetadata) GetRootfsSizeKey() int32 { if x != nil { - return x.Log + return x.RootfsSizeKey + } + return 0 +} + +func (x *TemplateBuildMetadata) GetEnvdVersionKey() string { + if x != nil { + return x.EnvdVersionKey } return "" } +// Logs from template build +type TemplateBuildStatusResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Done bool `protobuf:"varint,1,opt,name=done,proto3" json:"done,omitempty"` + Failed bool `protobuf:"varint,2,opt,name=failed,proto3" json:"failed,omitempty"` + Metadata *TemplateBuildMetadata `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` +} + +func (x *TemplateBuildStatusResponse) Reset() { + *x = TemplateBuildStatusResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_template_manager_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TemplateBuildStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TemplateBuildStatusResponse) ProtoMessage() {} + +func (x *TemplateBuildStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_template_manager_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TemplateBuildStatusResponse.ProtoReflect.Descriptor instead. +func (*TemplateBuildStatusResponse) Descriptor() ([]byte, []int) { + return file_template_manager_proto_rawDescGZIP(), []int{5} +} + +func (x *TemplateBuildStatusResponse) GetDone() bool { + if x != nil { + return x.Done + } + return false +} + +func (x *TemplateBuildStatusResponse) GetFailed() bool { + if x != nil { + return x.Failed + } + return false +} + +func (x *TemplateBuildStatusResponse) GetMetadata() *TemplateBuildMetadata { + if x != nil { + return x.Metadata + } + return nil +} + var File_template_manager_proto protoreflect.FileDescriptor var file_template_manager_proto_rawDesc = []byte{ @@ -305,27 +431,49 @@ var file_template_manager_proto_rawDesc = []byte{ 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x36, 0x0a, 0x1a, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x49, 0x44, 0x22, 0x24, 0x0a, 0x10, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, - 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x32, 0x9c, 0x01, 0x0a, 0x0f, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x0e, - 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x16, - 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x30, 0x01, 0x12, 0x4a, 0x0a, 0x13, 0x54, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x33, 0x5a, 0x31, 0x68, 0x74, 0x74, 0x70, 0x73, - 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x32, - 0x62, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x51, 0x0a, 0x15, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, + 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x22, 0x36, 0x0a, + 0x1a, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, + 0x69, 0x6c, 0x64, 0x49, 0x44, 0x22, 0x65, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x24, + 0x0a, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, + 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6e, + 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x22, 0x7d, 0x0a, 0x1b, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, + 0x6f, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x32, 0xee, 0x01, 0x0a, 0x0f, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x40, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x12, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x12, 0x4d, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1c, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, + 0x12, 0x4a, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, + 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x33, 0x5a, 0x31, + 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x65, 0x32, 0x62, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, + 0x2f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -340,25 +488,30 @@ func file_template_manager_proto_rawDescGZIP() []byte { return file_template_manager_proto_rawDescData } -var file_template_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_template_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_template_manager_proto_goTypes = []interface{}{ - (*TemplateConfig)(nil), // 0: TemplateConfig - (*TemplateCreateRequest)(nil), // 1: TemplateCreateRequest - (*TemplateBuildDeleteRequest)(nil), // 2: TemplateBuildDeleteRequest - (*TemplateBuildLog)(nil), // 3: TemplateBuildLog - (*emptypb.Empty)(nil), // 4: google.protobuf.Empty + (*TemplateConfig)(nil), // 0: TemplateConfig + (*TemplateCreateRequest)(nil), // 1: TemplateCreateRequest + (*TemplateStatusRequest)(nil), // 2: TemplateStatusRequest + (*TemplateBuildDeleteRequest)(nil), // 3: TemplateBuildDeleteRequest + (*TemplateBuildMetadata)(nil), // 4: TemplateBuildMetadata + (*TemplateBuildStatusResponse)(nil), // 5: TemplateBuildStatusResponse + (*emptypb.Empty)(nil), // 6: google.protobuf.Empty } var file_template_manager_proto_depIdxs = []int32{ 0, // 0: TemplateCreateRequest.template:type_name -> TemplateConfig - 1, // 1: TemplateService.TemplateCreate:input_type -> TemplateCreateRequest - 2, // 2: TemplateService.TemplateBuildDelete:input_type -> TemplateBuildDeleteRequest - 3, // 3: TemplateService.TemplateCreate:output_type -> TemplateBuildLog - 4, // 4: TemplateService.TemplateBuildDelete:output_type -> google.protobuf.Empty - 3, // [3:5] is the sub-list for method output_type - 1, // [1:3] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 4, // 1: TemplateBuildStatusResponse.metadata:type_name -> TemplateBuildMetadata + 1, // 2: TemplateService.TemplateCreate:input_type -> TemplateCreateRequest + 2, // 3: TemplateService.TemplateBuildStatus:input_type -> TemplateStatusRequest + 3, // 4: TemplateService.TemplateBuildDelete:input_type -> TemplateBuildDeleteRequest + 6, // 5: TemplateService.TemplateCreate:output_type -> google.protobuf.Empty + 5, // 6: TemplateService.TemplateBuildStatus:output_type -> TemplateBuildStatusResponse + 6, // 7: TemplateService.TemplateBuildDelete:output_type -> google.protobuf.Empty + 5, // [5:8] is the sub-list for method output_type + 2, // [2:5] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_template_manager_proto_init() } @@ -392,7 +545,7 @@ func file_template_manager_proto_init() { } } file_template_manager_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TemplateBuildDeleteRequest); i { + switch v := v.(*TemplateStatusRequest); i { case 0: return &v.state case 1: @@ -404,7 +557,31 @@ func file_template_manager_proto_init() { } } file_template_manager_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TemplateBuildLog); i { + switch v := v.(*TemplateBuildDeleteRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_template_manager_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TemplateBuildMetadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_template_manager_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TemplateBuildStatusResponse); i { case 0: return &v.state case 1: @@ -422,7 +599,7 @@ func file_template_manager_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_template_manager_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 6, NumExtensions: 0, NumServices: 1, }, diff --git a/packages/shared/pkg/grpc/template-manager/template-manager_grpc.pb.go b/packages/shared/pkg/grpc/template-manager/template-manager_grpc.pb.go index 930e41eb7..6def6bd34 100644 --- a/packages/shared/pkg/grpc/template-manager/template-manager_grpc.pb.go +++ b/packages/shared/pkg/grpc/template-manager/template-manager_grpc.pb.go @@ -24,8 +24,10 @@ const _ = grpc.SupportPackageIsVersion7 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type TemplateServiceClient interface { // TemplateCreate is a gRPC service that creates a new template - TemplateCreate(ctx context.Context, in *TemplateCreateRequest, opts ...grpc.CallOption) (TemplateService_TemplateCreateClient, error) - // EnvBuildDelete is a gRPC service that deletes files associated with a template build + TemplateCreate(ctx context.Context, in *TemplateCreateRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + // TemplateStatus is a gRPC service that streams the status of a template build + TemplateBuildStatus(ctx context.Context, in *TemplateStatusRequest, opts ...grpc.CallOption) (TemplateService_TemplateBuildStatusClient, error) + // TemplateBuildDelete is a gRPC service that deletes files associated with a template build TemplateBuildDelete(ctx context.Context, in *TemplateBuildDeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) } @@ -37,12 +39,21 @@ func NewTemplateServiceClient(cc grpc.ClientConnInterface) TemplateServiceClient return &templateServiceClient{cc} } -func (c *templateServiceClient) TemplateCreate(ctx context.Context, in *TemplateCreateRequest, opts ...grpc.CallOption) (TemplateService_TemplateCreateClient, error) { - stream, err := c.cc.NewStream(ctx, &TemplateService_ServiceDesc.Streams[0], "/TemplateService/TemplateCreate", opts...) +func (c *templateServiceClient) TemplateCreate(ctx context.Context, in *TemplateCreateRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, "/TemplateService/TemplateCreate", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *templateServiceClient) TemplateBuildStatus(ctx context.Context, in *TemplateStatusRequest, opts ...grpc.CallOption) (TemplateService_TemplateBuildStatusClient, error) { + stream, err := c.cc.NewStream(ctx, &TemplateService_ServiceDesc.Streams[0], "/TemplateService/TemplateBuildStatus", opts...) if err != nil { return nil, err } - x := &templateServiceTemplateCreateClient{stream} + x := &templateServiceTemplateBuildStatusClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -52,17 +63,17 @@ func (c *templateServiceClient) TemplateCreate(ctx context.Context, in *Template return x, nil } -type TemplateService_TemplateCreateClient interface { - Recv() (*TemplateBuildLog, error) +type TemplateService_TemplateBuildStatusClient interface { + Recv() (*TemplateBuildStatusResponse, error) grpc.ClientStream } -type templateServiceTemplateCreateClient struct { +type templateServiceTemplateBuildStatusClient struct { grpc.ClientStream } -func (x *templateServiceTemplateCreateClient) Recv() (*TemplateBuildLog, error) { - m := new(TemplateBuildLog) +func (x *templateServiceTemplateBuildStatusClient) Recv() (*TemplateBuildStatusResponse, error) { + m := new(TemplateBuildStatusResponse) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } @@ -83,8 +94,10 @@ func (c *templateServiceClient) TemplateBuildDelete(ctx context.Context, in *Tem // for forward compatibility type TemplateServiceServer interface { // TemplateCreate is a gRPC service that creates a new template - TemplateCreate(*TemplateCreateRequest, TemplateService_TemplateCreateServer) error - // EnvBuildDelete is a gRPC service that deletes files associated with a template build + TemplateCreate(context.Context, *TemplateCreateRequest) (*emptypb.Empty, error) + // TemplateStatus is a gRPC service that streams the status of a template build + TemplateBuildStatus(*TemplateStatusRequest, TemplateService_TemplateBuildStatusServer) error + // TemplateBuildDelete is a gRPC service that deletes files associated with a template build TemplateBuildDelete(context.Context, *TemplateBuildDeleteRequest) (*emptypb.Empty, error) mustEmbedUnimplementedTemplateServiceServer() } @@ -93,8 +106,11 @@ type TemplateServiceServer interface { type UnimplementedTemplateServiceServer struct { } -func (UnimplementedTemplateServiceServer) TemplateCreate(*TemplateCreateRequest, TemplateService_TemplateCreateServer) error { - return status.Errorf(codes.Unimplemented, "method TemplateCreate not implemented") +func (UnimplementedTemplateServiceServer) TemplateCreate(context.Context, *TemplateCreateRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method TemplateCreate not implemented") +} +func (UnimplementedTemplateServiceServer) TemplateBuildStatus(*TemplateStatusRequest, TemplateService_TemplateBuildStatusServer) error { + return status.Errorf(codes.Unimplemented, "method TemplateBuildStatus not implemented") } func (UnimplementedTemplateServiceServer) TemplateBuildDelete(context.Context, *TemplateBuildDeleteRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method TemplateBuildDelete not implemented") @@ -112,24 +128,42 @@ func RegisterTemplateServiceServer(s grpc.ServiceRegistrar, srv TemplateServiceS s.RegisterService(&TemplateService_ServiceDesc, srv) } -func _TemplateService_TemplateCreate_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(TemplateCreateRequest) +func _TemplateService_TemplateCreate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TemplateCreateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TemplateServiceServer).TemplateCreate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/TemplateService/TemplateCreate", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TemplateServiceServer).TemplateCreate(ctx, req.(*TemplateCreateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _TemplateService_TemplateBuildStatus_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(TemplateStatusRequest) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TemplateServiceServer).TemplateCreate(m, &templateServiceTemplateCreateServer{stream}) + return srv.(TemplateServiceServer).TemplateBuildStatus(m, &templateServiceTemplateBuildStatusServer{stream}) } -type TemplateService_TemplateCreateServer interface { - Send(*TemplateBuildLog) error +type TemplateService_TemplateBuildStatusServer interface { + Send(*TemplateBuildStatusResponse) error grpc.ServerStream } -type templateServiceTemplateCreateServer struct { +type templateServiceTemplateBuildStatusServer struct { grpc.ServerStream } -func (x *templateServiceTemplateCreateServer) Send(m *TemplateBuildLog) error { +func (x *templateServiceTemplateBuildStatusServer) Send(m *TemplateBuildStatusResponse) error { return x.ServerStream.SendMsg(m) } @@ -158,6 +192,10 @@ var TemplateService_ServiceDesc = grpc.ServiceDesc{ ServiceName: "TemplateService", HandlerType: (*TemplateServiceServer)(nil), Methods: []grpc.MethodDesc{ + { + MethodName: "TemplateCreate", + Handler: _TemplateService_TemplateCreate_Handler, + }, { MethodName: "TemplateBuildDelete", Handler: _TemplateService_TemplateBuildDelete_Handler, @@ -165,8 +203,8 @@ var TemplateService_ServiceDesc = grpc.ServiceDesc{ }, Streams: []grpc.StreamDesc{ { - StreamName: "TemplateCreate", - Handler: _TemplateService_TemplateCreate_Handler, + StreamName: "TemplateBuildStatus", + Handler: _TemplateService_TemplateBuildStatus_Handler, ServerStreams: true, }, }, diff --git a/packages/template-manager/template-manager.proto b/packages/template-manager/template-manager.proto index 84476c7ab..fd984fc58 100644 --- a/packages/template-manager/template-manager.proto +++ b/packages/template-manager/template-manager.proto @@ -23,20 +23,37 @@ message TemplateCreateRequest { TemplateConfig template = 1; } +message TemplateStatusRequest { + string templateID = 1; + string buildID = 2; +} + // Data required for deleting a template. message TemplateBuildDeleteRequest { string buildID = 1; } +message TemplateBuildMetadata { + int32 rootfsSizeKey = 3; + string envdVersionKey = 4; +} + // Logs from template build -message TemplateBuildLog { - string log = 1; +message TemplateBuildStatusResponse { + bool done = 1; + bool failed = 2; + + TemplateBuildMetadata metadata = 3; } // Interface exported by the server. service TemplateService { // TemplateCreate is a gRPC service that creates a new template - rpc TemplateCreate (TemplateCreateRequest) returns (stream TemplateBuildLog); + rpc TemplateCreate (TemplateCreateRequest) returns (google.protobuf.Empty); + + // TemplateStatus is a gRPC service that streams the status of a template build + rpc TemplateBuildStatus (TemplateStatusRequest) returns (stream TemplateBuildStatusResponse); + // TemplateBuildDelete is a gRPC service that deletes files associated with a template build rpc TemplateBuildDelete (TemplateBuildDeleteRequest) returns (google.protobuf.Empty); } From 0e51d09108b6f6d0a24feab0a6002ed966b6c5fb Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 5 Mar 2025 18:03:16 +0100 Subject: [PATCH 04/78] chore(template-manager): builds logs wirter uses only logger, streaming no longer used --- .../internal/build/writer/writer.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/packages/template-manager/internal/build/writer/writer.go b/packages/template-manager/internal/build/writer/writer.go index 91228bfb2..7760188a8 100644 --- a/packages/template-manager/internal/build/writer/writer.go +++ b/packages/template-manager/internal/build/writer/writer.go @@ -1,31 +1,19 @@ package writer import ( - "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" "go.uber.org/zap" ) type BuildLogsWriter struct { - stream template_manager.TemplateService_TemplateCreateServer logger *zap.Logger } func (w BuildLogsWriter) Write(p []byte) (n int, err error) { - log := string(p) - w.logger.Info(log) - err = w.stream.Send(&template_manager.TemplateBuildLog{Log: log}) - if err != nil { - return 0, err - } - + w.logger.Info(string(p)) return len(p), nil } -func New(stream template_manager.TemplateService_TemplateCreateServer, logger *zap.Logger) BuildLogsWriter { - writer := BuildLogsWriter{ - stream: stream, - logger: logger, - } - +func New(logger *zap.Logger) BuildLogsWriter { + writer := BuildLogsWriter{logger: logger} return writer } From 1eba6ac079aac0f36c0c3924049acefae1dd7731 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 5 Mar 2025 18:09:00 +0100 Subject: [PATCH 05/78] feat(template-manager): template creation runs background - build cache holding latest builds logs before ttl expires - template creation grcp endpoint no longer streams logs - build logic and async execution separated to own service --- .../internal/build/builder.go | 145 ++++++++++++++ .../internal/cache/build_cache.go | 184 ++++++++++++++++++ .../internal/server/create_template.go | 90 +-------- .../template-manager/internal/server/main.go | 34 ++-- 4 files changed, 357 insertions(+), 96 deletions(-) create mode 100644 packages/template-manager/internal/build/builder.go create mode 100644 packages/template-manager/internal/cache/build_cache.go diff --git a/packages/template-manager/internal/build/builder.go b/packages/template-manager/internal/build/builder.go new file mode 100644 index 000000000..51c3467e6 --- /dev/null +++ b/packages/template-manager/internal/build/builder.go @@ -0,0 +1,145 @@ +package build + +import ( + "context" + "fmt" + "github.com/docker/docker/client" + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "github.com/e2b-dev/infra/packages/template-manager/internal/cache" + "github.com/e2b-dev/infra/packages/template-manager/internal/template" + docker "github.com/fsouza/go-dockerclient" + "go.opentelemetry.io/otel/trace" + + "go.uber.org/zap" + "os/exec" + "strings" + "time" +) + +type TemplateBuilder struct { + tracer trace.Tracer + + logger *zap.Logger + buildCache *cache.BuildCache + buildLogger *zap.Logger + dockerClient *client.Client + legacyDockerClient *docker.Client + templateStorage *template.Storage +} + +const cleanupTimeout = time.Second * 10 + +func NewBuilder(logger *zap.Logger, buildLogger *zap.Logger, tracer trace.Tracer, dockerClient *client.Client, legacyDockerClient *docker.Client, templateStorage *template.Storage, buildCache *cache.BuildCache) *TemplateBuilder { + return &TemplateBuilder{ + logger: logger, + tracer: tracer, + buildCache: buildCache, + buildLogger: buildLogger, + dockerClient: dockerClient, + legacyDockerClient: legacyDockerClient, + templateStorage: templateStorage, + } +} + +func (b *TemplateBuilder) Builder(ctx context.Context, template *Env, envID string, buildID string) error { + buildStorage := b.templateStorage.NewBuild(template.TemplateFiles) + buildEntry, err := b.buildCache.Get(buildID) + if err != nil { + return err + } + + logsWriter := template.BuildLogsWriter + + // Remove local template files if build fails + defer func() { + removeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) + defer cancel() + + removeErr := template.Remove(removeCtx, b.tracer) + if removeErr != nil { + b.logger.Error("Error while removing template files", zap.Error(removeErr)) + telemetry.ReportError(ctx, removeErr) + } + }() + + err = template.Build(ctx, b.tracer, b.dockerClient, b.legacyDockerClient) + if err != nil { + _, _ = logsWriter.Write([]byte(fmt.Sprintf("Error building environment: %v", err))) + telemetry.ReportCriticalError(ctx, err) + + buildStateErr := b.buildCache.SetSucceeded(envID, buildID) + if buildStateErr != nil { + b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) + telemetry.ReportError(ctx, buildStateErr) + } + } + + // Remove build files if build fails or times out + defer func() { + if err != nil { + removeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) + defer cancel() + + removeErr := buildStorage.Remove(removeCtx) + if removeErr != nil { + b.logger.Error("Error while removing build files", zap.Error(removeErr)) + telemetry.ReportError(ctx, removeErr) + } + + buildStateErr := b.buildCache.SetSucceeded(envID, buildID) + if buildStateErr != nil { + b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) + telemetry.ReportError(ctx, buildStateErr) + } + } + }() + + memfilePath := template.BuildMemfilePath() + rootfsPath := template.BuildRootfsPath() + + upload := buildStorage.Upload( + ctx, + template.BuildSnapfilePath(), + &memfilePath, + &rootfsPath, + ) + + cmd := exec.Command(storage.HostEnvdPath, "-version") + out, err := cmd.Output() + if err != nil { + _, _ = logsWriter.Write([]byte(fmt.Sprintf("Error while getting envd version: %v", err))) + telemetry.ReportError(ctx, err) + + buildStateErr := b.buildCache.SetSucceeded(envID, buildID) + if buildStateErr != nil { + b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) + telemetry.ReportError(ctx, buildStateErr) + } + } + + uploadErr := <-upload + if uploadErr != nil { + errMsg := fmt.Sprintf("Error while uploading build files: %v", uploadErr) + _, _ = logsWriter.Write([]byte(errMsg)) + telemetry.ReportError(ctx, uploadErr) + + buildStateErr := b.buildCache.SetSucceeded(envID, buildID) + if buildStateErr != nil { + b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) + telemetry.ReportError(ctx, buildStateErr) + } + } + + buildEntry.SetEnvdVersionKey(strings.TrimSpace(string(out))) + buildEntry.SetRootFsSizeKey(int32(template.RootfsSizeMB())) + + err = b.buildCache.SetSucceeded(envID, buildID) + if err != nil { + b.logger.Error("Error while setting build state to succeeded", zap.Error(err)) + telemetry.ReportError(ctx, err) + return err + } + + return nil +} diff --git a/packages/template-manager/internal/cache/build_cache.go b/packages/template-manager/internal/cache/build_cache.go new file mode 100644 index 000000000..14ed68289 --- /dev/null +++ b/packages/template-manager/internal/cache/build_cache.go @@ -0,0 +1,184 @@ +package cache + +import ( + "context" + "fmt" + "github.com/e2b-dev/infra/packages/shared/pkg/meters" + "github.com/jellydator/ttlcache/v3" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.uber.org/zap" + "sync" + "time" +) + +const ( + buildInfoExpiration = time.Minute * 5 // 5 minutes +) + +type BuildInfo struct { + envID string + ended bool + failed bool + + rootfsSizeKey int32 + envdVersionKey string + + mu sync.RWMutex +} + +func (b *BuildInfo) IsRunning() bool { + b.mu.RLock() + defer b.mu.RUnlock() + + return !b.ended +} + +func (b *BuildInfo) IsFailed() bool { + b.mu.RLock() + defer b.mu.RUnlock() + + return b.failed +} + +func (b *BuildInfo) SetRootFsSizeKey(size int32) { + b.mu.RLock() + defer b.mu.RUnlock() + + b.rootfsSizeKey = size +} + +func (b *BuildInfo) GetRootFsSizeKey() int32 { + b.mu.RLock() + defer b.mu.RUnlock() + + return b.rootfsSizeKey +} + +func (b *BuildInfo) SetEnvdVersionKey(version string) { + b.mu.RLock() + defer b.mu.RUnlock() + + b.envdVersionKey = version +} + +func (b *BuildInfo) GetEnvdVersionKey() string { + b.mu.RLock() + defer b.mu.RUnlock() + + return b.envdVersionKey +} + +func (b *BuildInfo) GetBuildEnvID() string { + b.mu.RLock() + defer b.mu.RUnlock() + + return b.envID +} + +type BuildCache struct { + cache *ttlcache.Cache[string, *BuildInfo] + counter metric.Int64UpDownCounter + + mu sync.Mutex +} + +func NewBuildCache() *BuildCache { + cache := ttlcache.New(ttlcache.WithTTL[string, *BuildInfo](buildInfoExpiration)) + counter, err := meters.GetUpDownCounter(meters.BuildCounterMeterName) + if err != nil { + zap.L().Error("error creating counter", zap.Error(err)) + } + + go cache.Start() + + return &BuildCache{ + cache: cache, + counter: counter, + } +} + +// Get returns the build info. +func (c *BuildCache) Get(buildID string) (*BuildInfo, error) { + item := c.cache.Get(buildID) + if item == nil { + return nil, fmt.Errorf("build %s not found in cache", buildID) + } + + value := item.Value() + if value == nil { + return nil, fmt.Errorf("build %s not found in cache", buildID) + } + + return value, nil +} + +// Create creates a new build if it doesn't exist in the cache or the build was already finished. +func (c *BuildCache) Create(buildID string, envID string) error { + c.mu.Lock() + defer c.mu.Unlock() + + item := c.cache.Get(buildID, ttlcache.WithDisableTouchOnHit[string, *BuildInfo]()) + if item != nil { + return fmt.Errorf("build %s for env %s already exists in cache", buildID, envID) + } + + info := BuildInfo{ + envID: envID, + ended: false, + } + + c.cache.Set(buildID, &info, buildInfoExpiration) + c.updateCounter(envID, buildID, 1) + + return nil +} + +// SetDone marks the build as ended. +func (c *BuildCache) SetDone(envID string, buildID string) error { + item, err := c.Get(buildID) + if err != nil { + return fmt.Errorf("build %s not found in cache: %w", buildID, err) + } + + if !item.IsRunning() { + return fmt.Errorf("build %s is already marked as done", buildID) + } + + item.ended = true + item.failed = false + + c.updateCounter(envID, buildID, -1) + + return nil +} + +func (c *BuildCache) SetSucceeded(envID string, buildID string) error { + item, err := c.Get(buildID) + if err != nil { + return fmt.Errorf("build %s not found in cache: %w", buildID, err) + } + + if !item.IsRunning() { + return fmt.Errorf("build %s is already marked as done", buildID) + } + + item.ended = true + item.failed = true + + c.updateCounter(envID, buildID, -1) + + return nil +} + +func (c *BuildCache) updateCounter(envID string, buildID string, value int64) { + c.counter.Add(context.Background(), value, + metric.WithAttributes(attribute.String("env_id", envID)), + metric.WithAttributes(attribute.String("build_id", buildID)), + ) +} + +func (c *BuildCache) Delete(envID string, buildID string) { + c.cache.Delete(envID) + c.updateCounter(envID, buildID, -1) +} diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 7a860cb6c..b6941ab12 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -3,15 +3,10 @@ package server import ( "context" "fmt" + "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "os/exec" - "strconv" - "strings" - "time" - - "go.opentelemetry.io/otel/attribute" - "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/types/known/emptypb" template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" "github.com/e2b-dev/infra/packages/shared/pkg/storage" @@ -20,16 +15,11 @@ import ( "github.com/e2b-dev/infra/packages/template-manager/internal/build/writer" ) -const cleanupTimeout = time.Second * 10 - -func (s *serverStore) TemplateCreate(templateRequest *template_manager.TemplateCreateRequest, stream template_manager.TemplateService_TemplateCreateServer) error { - ctx := stream.Context() - +func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *template_manager.TemplateCreateRequest) (*emptypb.Empty, error) { childCtx, childSpan := s.tracer.Start(ctx, "template-create") defer childSpan.End() config := templateRequest.Template - childSpan.SetAttributes( attribute.String("env.id", config.TemplateID), attribute.String("env.build.id", config.BuildID), @@ -42,7 +32,6 @@ func (s *serverStore) TemplateCreate(templateRequest *template_manager.TemplateC ) logsWriter := writer.New( - stream, s.buildLogger. With(zap.Field{Type: zapcore.StringType, Key: "envID", String: config.TemplateID}). With(zap.Field{Type: zapcore.StringType, Key: "buildID", String: config.BuildID}), @@ -63,79 +52,18 @@ func (s *serverStore) TemplateCreate(templateRequest *template_manager.TemplateC BuildLogsWriter: logsWriter, } - buildStorage := s.templateStorage.NewBuild(template.TemplateFiles) - - var err error - - // Remove local template files if build fails - defer func() { - removeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) - defer cancel() - - removeErr := template.Remove(removeCtx, s.tracer) - if removeErr != nil { - telemetry.ReportError(childCtx, removeErr) - } - }() - - err = template.Build(childCtx, s.tracer, s.dockerClient, s.legacyDockerClient) + err := s.buildCache.Create(config.BuildID, config.TemplateID) if err != nil { - _, _ = logsWriter.Write([]byte(fmt.Sprintf("Error building environment: %v", err))) - - telemetry.ReportCriticalError(childCtx, err) - - return err + return nil, fmt.Errorf("error while creating build cache: %w", err) } - // Remove build files if build fails or times out - defer func() { - if err != nil { - removeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) - defer cancel() - - removeErr := buildStorage.Remove(removeCtx) - if removeErr != nil { - telemetry.ReportError(childCtx, removeErr) - } - } - }() - - memfilePath := template.BuildMemfilePath() - rootfsPath := template.BuildRootfsPath() - - upload := buildStorage.Upload( - childCtx, - template.BuildSnapfilePath(), - &memfilePath, - &rootfsPath, - ) - - cmd := exec.Command(storage.HostEnvdPath, "-version") - - out, err := cmd.Output() + err = s.builder.Builder(childCtx, template, config.TemplateID, config.BuildID) if err != nil { - _, _ = logsWriter.Write([]byte(fmt.Sprintf("Error while getting envd version: %v", err))) - - return err + s.logger.Error("Error while building template", zap.Error(err)) + return nil, fmt.Errorf("error while building template: %w", err) } - uploadErr := <-upload - if uploadErr != nil { - errMsg := fmt.Sprintf("Error while uploading build files: %v", uploadErr) - _, _ = logsWriter.Write([]byte(errMsg)) - - return uploadErr - } - - version := strings.TrimSpace(string(out)) - trailerMetadata := metadata.Pairs( - storage.RootfsSizeKey, strconv.FormatInt(template.RootfsSizeMB(), 10), - storage.EnvdVersionKey, version, - ) - - stream.SetTrailer(trailerMetadata) - telemetry.ReportEvent(childCtx, "Environment built") - return nil + return nil, nil } diff --git a/packages/template-manager/internal/server/main.go b/packages/template-manager/internal/server/main.go index e2bfa1cff..c24e33e49 100644 --- a/packages/template-manager/internal/server/main.go +++ b/packages/template-manager/internal/server/main.go @@ -2,6 +2,8 @@ package server import ( "context" + "github.com/e2b-dev/infra/packages/template-manager/internal/build" + "github.com/e2b-dev/infra/packages/template-manager/internal/cache" "time" artifactregistry "cloud.google.com/go/artifactregistry/apiv1" @@ -28,14 +30,14 @@ import ( type serverStore struct { templatemanager.UnimplementedTemplateServiceServer - server *grpc.Server - tracer trace.Tracer - logger *zap.Logger - buildLogger *zap.Logger - dockerClient *client.Client - legacyDockerClient *docker.Client - artifactRegistry *artifactregistry.Client - templateStorage *template.Storage + server *grpc.Server + tracer trace.Tracer + logger *zap.Logger + builder *build.TemplateBuilder + buildCache *cache.BuildCache + buildLogger *zap.Logger + templateStorage *template.Storage + artifactRegistry *artifactregistry.Client } func New(logger *zap.Logger, buildLogger *zap.Logger) *grpc.Server { @@ -82,15 +84,17 @@ func New(logger *zap.Logger, buildLogger *zap.Logger) *grpc.Server { } templateStorage := template.NewStorage(ctx) + buildCache := cache.NewBuildCache() + builder := build.NewBuilder(logger, buildLogger, otel.Tracer(constants.ServiceName), dockerClient, legacyClient, templateStorage, buildCache) templatemanager.RegisterTemplateServiceServer(s, &serverStore{ - tracer: otel.Tracer(constants.ServiceName), - logger: logger, - buildLogger: buildLogger, - dockerClient: dockerClient, - legacyDockerClient: legacyClient, - artifactRegistry: artifactRegistry, - templateStorage: templateStorage, + tracer: otel.Tracer(constants.ServiceName), + logger: logger, + builder: builder, + buildCache: buildCache, + buildLogger: buildLogger, + artifactRegistry: artifactRegistry, + templateStorage: templateStorage, }) grpc_health_v1.RegisterHealthServer(s, health.NewServer()) From 99e612b7361e810bf24b3408c5e81b97bd548593 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 5 Mar 2025 18:09:33 +0100 Subject: [PATCH 06/78] feat(template-manager): template status endpoint will pulling of build changes --- .../internal/server/template_status.go | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 packages/template-manager/internal/server/template_status.go diff --git a/packages/template-manager/internal/server/template_status.go b/packages/template-manager/internal/server/template_status.go new file mode 100644 index 000000000..7665d6bf4 --- /dev/null +++ b/packages/template-manager/internal/server/template_status.go @@ -0,0 +1,67 @@ +package server + +import ( + "context" + "fmt" + "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "go.uber.org/zap" + "time" +) + +const statusTimeout = time.Minute * 5 + +func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusRequest, stream template_manager.TemplateService_TemplateBuildStatusServer) error { + ctx, cancel := context.WithTimeout(stream.Context(), statusTimeout) + defer cancel() + + childCtx, childSpan := s.tracer.Start(ctx, "template-build-status-request") + defer childSpan.End() + + logger := s.logger.With(zap.String("buildID", in.BuildID), zap.String("envID", in.TemplateID)) + + // stream updated status until build is done or timeout will kick in + for { + buildInfo, err := s.buildCache.Get(in.BuildID) + if err != nil { + logger.Error("error while getting build info from cache", zap.Error(err)) + return fmt.Errorf("error while getting build info, maybe already expired") + } + + var metadata *template_manager.TemplateBuildMetadata = nil + if !buildInfo.IsRunning() { + metadata = &template_manager.TemplateBuildMetadata{ + RootfsSizeKey: buildInfo.GetRootFsSizeKey(), + EnvdVersionKey: buildInfo.GetEnvdVersionKey(), + } + } + + err = stream.Send( + &template_manager.TemplateBuildStatusResponse{ + Done: !buildInfo.IsRunning(), + Failed: buildInfo.IsFailed(), + Metadata: metadata, + }, + ) + + if err != nil { + logger.Error("Error while sending status stream back to API", zap.Error(err)) + telemetry.ReportError(childCtx, err) + return err + } + + if buildInfo.IsFailed() { + logger.Error("Template build failed") + return fmt.Errorf("template build failed") + } + + if !buildInfo.IsRunning() { + logger.Info("Template build finished") + break + } + + time.Sleep(time.Second * 5) + } + + return nil +} From ceedeeccd532f604f2ae31b2b9eb7dd71050e5b9 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Thu, 6 Mar 2025 10:24:01 +0100 Subject: [PATCH 07/78] chore(api): build cache removed from api store --- .../api/internal/cache/builds/build_cache.go | 180 ------------------ packages/api/internal/handlers/store.go | 4 - 2 files changed, 184 deletions(-) delete mode 100644 packages/api/internal/cache/builds/build_cache.go diff --git a/packages/api/internal/cache/builds/build_cache.go b/packages/api/internal/cache/builds/build_cache.go deleted file mode 100644 index 623be8d28..000000000 --- a/packages/api/internal/cache/builds/build_cache.go +++ /dev/null @@ -1,180 +0,0 @@ -package builds - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/google/uuid" - "github.com/jellydator/ttlcache/v3" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" - "go.uber.org/zap" - - "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/shared/pkg/meters" -) - -const ( - buildInfoExpiration = time.Minute * 5 // 5 minutes -) - -type BuildInfo struct { - buildID uuid.UUID - teamID uuid.UUID - status api.TemplateBuildStatus - logs []string - - mu sync.RWMutex -} - -func (b *BuildInfo) GetLogs() []string { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.logs -} - -func (b *BuildInfo) GetStatus() api.TemplateBuildStatus { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.status -} - -func (b *BuildInfo) GetBuildID() uuid.UUID { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.buildID -} - -func (b *BuildInfo) GetTeamID() uuid.UUID { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.teamID -} - -func (b *BuildInfo) addLog(log string) { - b.mu.Lock() - defer b.mu.Unlock() - - b.logs = append(b.logs, log) -} - -func (b *BuildInfo) setStatus(status api.TemplateBuildStatus) { - b.mu.Lock() - defer b.mu.Unlock() - - b.status = status -} - -type BuildCache struct { - cache *ttlcache.Cache[string, *BuildInfo] - counter metric.Int64UpDownCounter - - mu sync.Mutex -} - -func NewBuildCache() *BuildCache { - cache := ttlcache.New(ttlcache.WithTTL[string, *BuildInfo](buildInfoExpiration)) - counter, err := meters.GetUpDownCounter(meters.BuildCounterMeterName) - if err != nil { - zap.L().Error("error creating counter", zap.Error(err)) - } - - go cache.Start() - - return &BuildCache{ - cache: cache, - counter: counter, - } -} - -// Get returns the build info. -func (c *BuildCache) Get(envID string, buildID uuid.UUID) (*BuildInfo, error) { - item := c.cache.Get(envID) - - if item == nil { - return nil, fmt.Errorf("build for %s not found in cache", envID) - } - - value := item.Value() - - if value == nil { - return nil, fmt.Errorf("build for %s not found in cache", envID) - } - - if value.GetBuildID() != buildID { - return nil, fmt.Errorf("received logs for another build %s env %s", buildID, envID) - } - - return value, nil -} - -// Append appends logs to the build. -func (c *BuildCache) Append(envID string, buildID uuid.UUID, log string) error { - item, err := c.Get(envID, buildID) - if err != nil { - errMsg := fmt.Errorf("build for %s not found in cache: %w", envID, err) - - return errMsg - } - - item.addLog(log) - - return nil -} - -// Create creates a new build if it doesn't exist in the cache or the build was already finished. -func (c *BuildCache) Create(envID string, buildID uuid.UUID, teamID uuid.UUID) error { - c.mu.Lock() - defer c.mu.Unlock() - - item := c.cache.Get(envID, ttlcache.WithDisableTouchOnHit[string, *BuildInfo]()) - if item != nil && item.Value().GetStatus() == api.TemplateBuildStatusBuilding { - return fmt.Errorf("build for %s already exists in cache", envID) - } - - info := BuildInfo{ - buildID: buildID, - teamID: teamID, - status: api.TemplateBuildStatusBuilding, - // We need to explicitly set the logs to an empty slice for the json serialization to work in CLI - logs: []string{}, - } - - c.cache.Set(envID, &info, buildInfoExpiration) - - c.updateCounter(envID, buildID, teamID, 1) - - return nil -} - -// SetDone marks the build as finished. -func (c *BuildCache) SetDone(envID string, buildID uuid.UUID, status api.TemplateBuildStatus) error { - item, err := c.Get(envID, buildID) - if err != nil { - return fmt.Errorf("build %s not found in cache: %w", buildID, err) - } - - item.setStatus(status) - c.updateCounter(envID, buildID, item.teamID, -1) - - return nil -} - -func (c *BuildCache) updateCounter(envID string, buildID, teamID uuid.UUID, value int64) { - c.counter.Add(context.Background(), value, - metric.WithAttributes(attribute.String("env_id", envID)), - metric.WithAttributes(attribute.String("build_id", buildID.String())), - metric.WithAttributes(attribute.String("team_id", teamID.String())), - ) -} - -func (c *BuildCache) Delete(envID string, buildID, teamID uuid.UUID) { - c.cache.Delete(envID) - c.updateCounter(envID, buildID, teamID, -1) -} diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 811abb922..b73dd98fa 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -23,7 +23,6 @@ import ( analyticscollector "github.com/e2b-dev/infra/packages/api/internal/analytics_collector" "github.com/e2b-dev/infra/packages/api/internal/api" authcache "github.com/e2b-dev/infra/packages/api/internal/cache/auth" - "github.com/e2b-dev/infra/packages/api/internal/cache/builds" templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" "github.com/e2b-dev/infra/packages/api/internal/orchestrator" template_manager "github.com/e2b-dev/infra/packages/api/internal/template-manager" @@ -137,8 +136,6 @@ func NewAPIStore(ctx context.Context) *APIStore { zap.L().Warn("LOKI_ADDRESS not set, disabling Loki client") } - buildCache := builds.NewBuildCache() - templateCache := templatecache.NewTemplateCache(dbClient) authCache := authcache.NewTeamAuthCache() templateSpawnCounter := utils.NewTemplateSpawnCounter(time.Minute, dbClient) @@ -150,7 +147,6 @@ func NewAPIStore(ctx context.Context) *APIStore { db: dbClient, Tracer: tracer, posthog: posthogClient, - buildCache: buildCache, lokiClient: lokiClient, templateCache: templateCache, authCache: authCache, From 0db8efad428de4360d767cd586331423c424f878 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Thu, 6 Mar 2025 10:25:54 +0100 Subject: [PATCH 08/78] chore(api): template builds logs are using db instead of locally cached data --- .../internal/handlers/template_build_logs.go | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index e81f22b6a..3daa87061 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -46,27 +46,21 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem if err != nil { errMsg := fmt.Errorf("error when parsing build id: %w", err) telemetry.ReportError(ctx, errMsg) - a.sendAPIStoreError(c, http.StatusBadRequest, "Invalid build id") - return } - dockerBuild, err := a.buildCache.Get(templateID, buildUUID) + templateDB, _, err := a.db.GetEnv(ctx, templateID) if err != nil { - msg := fmt.Errorf("error finding cache for env %s and build %s", templateID, buildID) - telemetry.ReportError(ctx, msg) - - a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Build (%s) not found", buildID)) - + errMsg := fmt.Errorf("error when getting env: %w", err) + telemetry.ReportError(ctx, errMsg) + a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Template '%s' not found", templateID)) return } - templateTeamID := dockerBuild.GetTeamID() - var team *models.Team for _, t := range teams { - if t.ID == templateTeamID { + if t.ID == templateDB.TeamID { team = t break } @@ -81,7 +75,13 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem return } - status := dockerBuild.GetStatus() + buildDB, err := a.db.GetEnvBuild(ctx, buildUUID) + if err != nil { + errMsg := fmt.Errorf("error when getting build: %w", err) + telemetry.ReportError(ctx, errMsg) + a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Build '%s' not found", buildID)) + return + } // Sanitize env ID // https://grafana.com/blog/2021/01/05/how-to-escape-special-characters-with-lokis-logql/ @@ -138,7 +138,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem Logs: logs, TemplateID: templateID, BuildID: buildID, - Status: status, + Status: api.TemplateBuildStatus(buildDB.Status), } c.JSON(http.StatusOK, result) From 3b81051e33d85253842489bb20de31f9b6e56901 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:09:54 +0100 Subject: [PATCH 09/78] reguster template manager for api store, run background process for build cleanups --- packages/api/internal/handlers/store.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index b73dd98fa..38c36d1a3 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -25,7 +25,7 @@ import ( authcache "github.com/e2b-dev/infra/packages/api/internal/cache/auth" templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" "github.com/e2b-dev/infra/packages/api/internal/orchestrator" - template_manager "github.com/e2b-dev/infra/packages/api/internal/template-manager" + "github.com/e2b-dev/infra/packages/api/internal/template-manager" "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/shared/pkg/chdb" "github.com/e2b-dev/infra/packages/shared/pkg/db" @@ -122,11 +122,14 @@ func NewAPIStore(ctx context.Context) *APIStore { zap.L().Fatal("initializing Orchestrator client", zap.Error(err)) } - templateManager, err := template_manager.New() + templateManager, err := template_manager.New(dbClient) if err != nil { zap.L().Fatal("initializing Template manager client", zap.Error(err)) } + // Start the periodic sync of template builds statuses + go templateManager.BuildsStatusPeriodicalSync() + var lokiClient *loki.DefaultClient if laddr := os.Getenv("LOKI_ADDRESS"); laddr != "" { lokiClient = &loki.DefaultClient{ From 1b3721c51f1ab8ca68ba7bb15d866db4c6aac54d Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:10:08 +0100 Subject: [PATCH 10/78] fixed build status logs mapping --- .../internal/handlers/template_build_logs.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 3daa87061..86b8a5d3c 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -11,13 +11,13 @@ import ( "github.com/grafana/loki/pkg/logproto" "go.uber.org/zap" - "github.com/e2b-dev/infra/packages/shared/pkg/models" - "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/auth" + "github.com/e2b-dev/infra/packages/shared/pkg/models" + "github.com/e2b-dev/infra/packages/shared/pkg/models/envbuild" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -138,8 +138,21 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem Logs: logs, TemplateID: templateID, BuildID: buildID, - Status: api.TemplateBuildStatus(buildDB.Status), + Status: getCorrespondingTemplateBuildStatus(buildDB.Status), } c.JSON(http.StatusOK, result) } + +func getCorrespondingTemplateBuildStatus(s envbuild.Status) api.TemplateBuildStatus { + switch s { + case envbuild.StatusWaiting: + return api.TemplateBuildStatusBuilding + case envbuild.StatusFailed: + return api.TemplateBuildStatusError + case envbuild.StatusUploaded: + return api.TemplateBuildStatusReady + default: + return api.TemplateBuildStatusBuilding + } +} From f88ef58cf9f00c5ad56057425290784f7233599b Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:11:49 +0100 Subject: [PATCH 11/78] fixed issues in template manager cache --- .../internal/build/builder.go | 12 ++------ .../internal/cache/build_cache.go | 26 ++++++++--------- .../internal/server/create_template.go | 28 +++++++++++++++---- .../internal/server/template_status.go | 13 ++++----- 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/packages/template-manager/internal/build/builder.go b/packages/template-manager/internal/build/builder.go index 51c3467e6..8480a7a7d 100644 --- a/packages/template-manager/internal/build/builder.go +++ b/packages/template-manager/internal/build/builder.go @@ -68,7 +68,7 @@ func (b *TemplateBuilder) Builder(ctx context.Context, template *Env, envID stri _, _ = logsWriter.Write([]byte(fmt.Sprintf("Error building environment: %v", err))) telemetry.ReportCriticalError(ctx, err) - buildStateErr := b.buildCache.SetSucceeded(envID, buildID) + buildStateErr := b.buildCache.SetFailed(envID, buildID) if buildStateErr != nil { b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) telemetry.ReportError(ctx, buildStateErr) @@ -86,12 +86,6 @@ func (b *TemplateBuilder) Builder(ctx context.Context, template *Env, envID stri b.logger.Error("Error while removing build files", zap.Error(removeErr)) telemetry.ReportError(ctx, removeErr) } - - buildStateErr := b.buildCache.SetSucceeded(envID, buildID) - if buildStateErr != nil { - b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) - telemetry.ReportError(ctx, buildStateErr) - } } }() @@ -111,7 +105,7 @@ func (b *TemplateBuilder) Builder(ctx context.Context, template *Env, envID stri _, _ = logsWriter.Write([]byte(fmt.Sprintf("Error while getting envd version: %v", err))) telemetry.ReportError(ctx, err) - buildStateErr := b.buildCache.SetSucceeded(envID, buildID) + buildStateErr := b.buildCache.SetFailed(envID, buildID) if buildStateErr != nil { b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) telemetry.ReportError(ctx, buildStateErr) @@ -124,7 +118,7 @@ func (b *TemplateBuilder) Builder(ctx context.Context, template *Env, envID stri _, _ = logsWriter.Write([]byte(errMsg)) telemetry.ReportError(ctx, uploadErr) - buildStateErr := b.buildCache.SetSucceeded(envID, buildID) + buildStateErr := b.buildCache.SetFailed(envID, buildID) if buildStateErr != nil { b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) telemetry.ReportError(ctx, buildStateErr) diff --git a/packages/template-manager/internal/cache/build_cache.go b/packages/template-manager/internal/cache/build_cache.go index 14ed68289..3c6f93eab 100644 --- a/packages/template-manager/internal/cache/build_cache.go +++ b/packages/template-manager/internal/cache/build_cache.go @@ -124,8 +124,9 @@ func (c *BuildCache) Create(buildID string, envID string) error { } info := BuildInfo{ - envID: envID, - ended: false, + envID: envID, + ended: false, + failed: false, } c.cache.Set(buildID, &info, buildInfoExpiration) @@ -134,40 +135,35 @@ func (c *BuildCache) Create(buildID string, envID string) error { return nil } -// SetDone marks the build as ended. -func (c *BuildCache) SetDone(envID string, buildID string) error { +func (c *BuildCache) SetSucceeded(envID string, buildID string) error { + c.mu.Lock() + defer c.mu.Unlock() + item, err := c.Get(buildID) if err != nil { return fmt.Errorf("build %s not found in cache: %w", buildID, err) } - if !item.IsRunning() { - return fmt.Errorf("build %s is already marked as done", buildID) - } - item.ended = true item.failed = false c.updateCounter(envID, buildID, -1) - return nil } -func (c *BuildCache) SetSucceeded(envID string, buildID string) error { +func (c *BuildCache) SetFailed(envID string, buildID string) error { + c.mu.Lock() + defer c.mu.Unlock() + item, err := c.Get(buildID) if err != nil { return fmt.Errorf("build %s not found in cache: %w", buildID, err) } - if !item.IsRunning() { - return fmt.Errorf("build %s is already marked as done", buildID) - } - item.ended = true item.failed = true c.updateCounter(envID, buildID, -1) - return nil } diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index b6941ab12..ec61a99f6 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -16,7 +16,7 @@ import ( ) func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *template_manager.TemplateCreateRequest) (*emptypb.Empty, error) { - childCtx, childSpan := s.tracer.Start(ctx, "template-create") + childCtx, childSpan := s.tracer.Start(context.Background(), "template-create") defer childSpan.End() config := templateRequest.Template @@ -57,13 +57,29 @@ func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *templ return nil, fmt.Errorf("error while creating build cache: %w", err) } - err = s.builder.Builder(childCtx, template, config.TemplateID, config.BuildID) - if err != nil { - s.logger.Error("Error while building template", zap.Error(err)) - return nil, fmt.Errorf("error while building template: %w", err) + + go func() { + + err = s.builder.Builder(childCtx, template, config.TemplateID, config.BuildID) + if err != nil { + s.logger.Error("Error while building template", zap.Error(err)) + telemetry.ReportEvent(childCtx, "Environment built failed") + } + + cacheIn, cacheErr := s.buildCache.Get(config.BuildID) + if cacheErr != nil { + zap.L().Error("template creation cache fetch failed", zap.Error(cacheErr)) + } + + telemetry.ReportEvent(childCtx, "Environment built") + }() + + cacheIn, cacheErr := s.buildCache.Get(config.BuildID) + if cacheErr != nil { + zap.L().Error("template creation cache fetch failed", zap.Error(cacheErr)) + return nil, fmt.Errorf("error while getting build info, maybe already expired") } - telemetry.ReportEvent(childCtx, "Environment built") return nil, nil } diff --git a/packages/template-manager/internal/server/template_status.go b/packages/template-manager/internal/server/template_status.go index 7665d6bf4..4cb434196 100644 --- a/packages/template-manager/internal/server/template_status.go +++ b/packages/template-manager/internal/server/template_status.go @@ -24,7 +24,6 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq for { buildInfo, err := s.buildCache.Get(in.BuildID) if err != nil { - logger.Error("error while getting build info from cache", zap.Error(err)) return fmt.Errorf("error while getting build info, maybe already expired") } @@ -36,6 +35,11 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq } } + if buildInfo.IsFailed() { + logger.Error("Template build failed") + return fmt.Errorf("template build failed") + } + err = stream.Send( &template_manager.TemplateBuildStatusResponse{ Done: !buildInfo.IsRunning(), @@ -50,14 +54,9 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq return err } - if buildInfo.IsFailed() { - logger.Error("Template build failed") - return fmt.Errorf("template build failed") - } - if !buildInfo.IsRunning() { logger.Info("Template build finished") - break + return nil } time.Sleep(time.Second * 5) From 5b073ca5801c8962abb6d5f821bd940f237d2001 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:13:55 +0100 Subject: [PATCH 12/78] api template build refactor --- .../internal/handlers/template_start_build.go | 139 ++++++++--------- .../template-manager/create_template.go | 67 +------- .../template-manager/template_manager.go | 143 +++++++++++++++++- packages/shared/pkg/db/envs.go | 57 +++++++ 4 files changed, 273 insertions(+), 133 deletions(-) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index 451ecedd9..7246a8162 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -88,9 +88,9 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template attribute.String("template.id", templateID), ) - // Create a new build cache for storing logs - err = a.buildCache.Create(templateID, buildUUID, team.ID) - if err != nil { + // Check if some other build from same template is in process, excludes currently created build + envBuildDB, err := a.db.GetRunningEnvBuild(ctx, envDB.ID, []uuid.UUID{buildUUID}) + if err == nil && envBuildDB != nil { a.sendAPIStoreError(c, http.StatusConflict, fmt.Sprintf("there's already running build for %s", templateID)) err = fmt.Errorf("build is already running build for %s", templateID) @@ -99,82 +99,83 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template return } - // Set the build status to building - err = a.db.EnvBuildSetStatus(ctx, envDB.ID, buildUUID, envbuild.StatusBuilding) - if err != nil { - err = fmt.Errorf("error when setting build status: %w", err) - telemetry.ReportCriticalError(ctx, err) + // Trigger the build in the background + buildContext, childSpan := a.Tracer.Start( + trace.ContextWithSpanContext(context.Background(), span.SpanContext()), + "background-build-env", + ) + defer childSpan.End() - a.buildCache.Delete(templateID, buildUUID, team.ID) + startTime := time.Now() + build := envDB.Edges.Builds[0] + startCmd := "" + if build.StartCmd != nil { + startCmd = *build.StartCmd + } + + // Call the Template Manager to build the environment + buildErr := a.templateManager.CreateTemplate( + a.Tracer, + buildContext, + templateID, + buildUUID, + build.KernelVersion, + build.FirecrackerVersion, + startCmd, + build.Vcpu, + build.FreeDiskSizeMB, + build.RAMMB, + ) + + if buildErr != nil { + buildErr = fmt.Errorf("error when building env: %w", buildErr) + zap.L().Error("build failed", zap.Error(buildErr)) + telemetry.ReportCriticalError(buildContext, buildErr) + + dbErr := a.db.EnvBuildSetStatus(buildContext, templateID, buildUUID, envbuild.StatusFailed) + if dbErr != nil { + telemetry.ReportCriticalError(buildContext, fmt.Errorf("error when setting build status: %w", dbErr)) + } return } - // Trigger the build in the background - go func() { - buildContext, childSpan := a.Tracer.Start( - trace.ContextWithSpanContext(context.Background(), span.SpanContext()), - "background-build-env", - ) - defer childSpan.End() - - startTime := time.Now() - build := envDB.Edges.Builds[0] - startCmd := "" - if build.StartCmd != nil { - startCmd = *build.StartCmd - } + // status building must be set after build is triggered because then + // it's possible build status job will be triggered before build cache on template manager is created and build will fail + err = a.db.EnvBuildSetStatus(ctx, templateID, buildUUID, envbuild.StatusBuilding) + if err != nil { + err = fmt.Errorf("error when setting build status: %w", err) + telemetry.ReportCriticalError(ctx, err) + } - // Call the Template Manager to build the environment - buildErr := a.templateManager.CreateTemplate( - a.Tracer, - buildContext, - a.db, - a.buildCache, - templateID, - buildUUID, - build.KernelVersion, - build.FirecrackerVersion, - startCmd, - build.Vcpu, - build.FreeDiskSizeMB, - build.RAMMB, - ) - if buildErr != nil { - buildErr = fmt.Errorf("error when building env: %w", buildErr) - zap.L().Error("build failed", zap.Error(buildErr)) - telemetry.ReportCriticalError(buildContext, buildErr) - - dbErr := a.db.EnvBuildSetStatus(buildContext, templateID, buildUUID, envbuild.StatusFailed) - if dbErr != nil { - telemetry.ReportCriticalError(buildContext, fmt.Errorf("error when setting build status: %w", dbErr)) - } - - // Save the error in the logs - buildCacheErr := a.buildCache.Append(templateID, buildUUID, fmt.Sprintf("Build failed: %s\n", buildErr)) - if buildCacheErr != nil { - telemetry.ReportCriticalError(buildContext, fmt.Errorf("error when appending build logs: %w", buildCacheErr)) - } - - cacheErr := a.buildCache.SetDone(templateID, buildUUID, api.TemplateBuildStatusError) - if cacheErr != nil { - telemetry.ReportCriticalError(buildContext, fmt.Errorf("error when setting build done in logs: %w", cacheErr)) - } - - return - } + // Do not wait for global build sync trigger it immediately + go func() { + a.templateManager.BuildStatusSync(context.Background(), buildUUID, templateID) // Invalidate the cache a.templateCache.Invalidate(templateID) - - a.posthog.CreateAnalyticsUserEvent(userID.String(), team.ID.String(), "built environment", posthog.NewProperties(). - Set("user_id", userID). - Set("environment", templateID). - Set("build_id", buildID). - Set("duration", time.Since(startTime).String()). - Set("success", err != nil), - ) }() + // todo + //telemetry.ReportEvent(childCtx, "created new environment", attribute.String("env.id", templateID)) + + // todo + //// status building must be set after build is triggered because then + //// it's possible build status job will be triggered before build cache on template manager is created and build will fail + //err = a.db.EnvBuildSetStatus(ctx, envDB.ID, buildUUID, envbuild.StatusBuilding) + //if err != nil { + // err = fmt.Errorf("error when setting build status: %w", err) + // telemetry.ReportCriticalError(ctx, err) + // return + //} + + a.posthog.CreateAnalyticsUserEvent(userID.String(), team.ID.String(), "built environment", posthog.NewProperties(). + Set("user_id", userID). + Set("environment", templateID). + Set("build_id", buildID). + Set("duration", time.Since(startTime).String()). + Set("success", err != nil), + ) + c.Status(http.StatusAccepted) } diff --git a/packages/api/internal/template-manager/create_template.go b/packages/api/internal/template-manager/create_template.go index c22158e8c..c3e1d2e9e 100644 --- a/packages/api/internal/template-manager/create_template.go +++ b/packages/api/internal/template-manager/create_template.go @@ -4,28 +4,17 @@ import ( "context" _ "embed" "fmt" - "io" - "strconv" - - "github.com/google/uuid" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/e2b-dev/infra/packages/api/internal/api" - "github.com/e2b-dev/infra/packages/api/internal/cache/builds" "github.com/e2b-dev/infra/packages/api/internal/sandbox" "github.com/e2b-dev/infra/packages/api/internal/utils" - "github.com/e2b-dev/infra/packages/shared/pkg/db" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" - "github.com/e2b-dev/infra/packages/shared/pkg/storage" - "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" + "github.com/google/uuid" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) func (tm *TemplateManager) CreateTemplate( t trace.Tracer, ctx context.Context, - db *db.DB, - buildCache *builds.BuildCache, templateID string, buildID uuid.UUID, kernelVersion, @@ -49,9 +38,7 @@ func (tm *TemplateManager) CreateTemplate( return errMsg } - telemetry.ReportEvent(childCtx, "Got FC version info") - - logs, err := tm.grpc.Client.TemplateCreate(ctx, &template_manager.TemplateCreateRequest{ + _, err = tm.grpc.Client.TemplateCreate(childCtx, &template_manager.TemplateCreateRequest{ Template: &template_manager.TemplateConfig{ TemplateID: templateID, BuildID: buildID.String(), @@ -64,52 +51,10 @@ func (tm *TemplateManager) CreateTemplate( StartCommand: startCommand, }, }) - err = utils.UnwrapGRPCError(err) - if err != nil { - return fmt.Errorf("failed to create template '%s': %w", templateID, err) - } - - // Wait for the build to finish and save logs - for { - _, receiveErr := logs.Recv() - if receiveErr == io.EOF { - break - } else if receiveErr != nil { - // There was an error during the build - return fmt.Errorf("error when building env: %w", receiveErr) - } - } - - trailer := logs.Trailer() - rootfsSizeStr, ok := trailer[storage.RootfsSizeKey] - if !ok { - return fmt.Errorf("rootfs size not found in trailer") - } - - diskSize, parseErr := strconv.ParseInt(rootfsSizeStr[0], 10, 64) - if parseErr != nil { - return fmt.Errorf("error when parsing rootfs size: %w", parseErr) - } - - envdVersion, ok := trailer[storage.EnvdVersionKey] - if !ok { - return fmt.Errorf("envd version not found in trailer") - } - err = db.FinishEnvBuild(childCtx, templateID, buildID, diskSize, envdVersion[0]) - if err != nil { - return fmt.Errorf("error when finishing build: %w", err) - } - - telemetry.ReportEvent(childCtx, "created new environment", attribute.String("env.id", templateID)) - - cacheErr := buildCache.SetDone(templateID, buildID, api.TemplateBuildStatusReady) - if cacheErr != nil { - err = fmt.Errorf("error when setting build done in logs: %w", cacheErr) - telemetry.ReportCriticalError(childCtx, cacheErr) + if utils.UnwrapGRPCError(err) != nil { + return fmt.Errorf("failed to create template '%s': %w", templateID, err) } - telemetry.ReportEvent(childCtx, "Template build started") - return nil } diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index ed84e1d3b..08df6fa3b 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -1,20 +1,157 @@ package template_manager +import ( + "context" + "github.com/e2b-dev/infra/packages/api/internal/utils" + "github.com/e2b-dev/infra/packages/shared/pkg/db" + "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + "github.com/e2b-dev/infra/packages/shared/pkg/models/envbuild" + "github.com/google/uuid" + "go.uber.org/zap" + "io" + "sync" + "time" +) + type TemplateManager struct { - grpc *GRPCClient + grpc *GRPCClient + db *db.DB + lock sync.Mutex + processing map[uuid.UUID]struct{ templateID string } } -func New() (*TemplateManager, error) { +var ( + //syncInterval = time.Minute * 1 + // todo: just for testing + syncInterval = time.Second * 10 + syncTimeout = time.Minute * 5 +) + +func New(db *db.DB) (*TemplateManager, error) { client, err := NewClient() if err != nil { return nil, err } return &TemplateManager{ - grpc: client, + grpc: client, + db: db, + lock: sync.Mutex{}, + processing: make(map[uuid.UUID]struct{ templateID string }), }, nil } func (tm *TemplateManager) Close() error { return tm.grpc.Close() } + +func (tm *TemplateManager) BuildsStatusPeriodicalSync() { + // todo: disabled debugging + + // todo: implement different timeout for different states + // todo: how to do not fast-cancel new build? + return + + for { + buildsRunning, err := tm.db.GetRunningEnvBuilds(context.Background()) + if err != nil { + zap.L().Error("Error getting running builds for periodical sync", zap.Error(err)) + continue + } + + zap.L().Info("Running periodical sync of builds statuses", zap.Int("count", len(buildsRunning))) + for _, buildDB := range buildsRunning { + go tm.BuildStatusSync(context.Background(), buildDB.ID, *buildDB.EnvID) + } + + time.Sleep(syncInterval) + } +} + +func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUID, templateID string) { + childCtx, childCtxCancel := context.WithTimeout(ctx, syncTimeout) + defer childCtxCancel() + + if tm.createInProcessingQueue(buildID, templateID) { + // already processing, skip + return + } + + // remove from processing queue when done + defer tm.removeFromProcessingQueue(buildID) + + logger := zap.L().With(zap.String("buildID", buildID.String()), zap.String("envID", templateID)) + + // stream build status + stream, err := tm.grpc.Client.TemplateBuildStatus(childCtx, &template_manager.TemplateStatusRequest{BuildID: buildID.String(), TemplateID: templateID}) + if utils.UnwrapGRPCError(err) != nil { + logger.Error("Error when fetching template build status", zap.Error(err)) + + dbErr := tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) + if dbErr != nil { + logger.Error("Error when setting build status", zap.Error(dbErr)) + } + + return + } + + for { + status, receiveErr := stream.Recv() + if receiveErr == io.EOF { + break + } else if receiveErr != nil { + logger.Error("Error when receiving template build status", zap.Error(receiveErr)) + + err = tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) + if err != nil { + logger.Error("Error when setting build status", zap.Error(err)) + } + + return + } + + if status.Done { + // marked as finished but failed + if status.Failed { + err = tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) + if err != nil { + logger.Error("Error when setting build status", zap.Error(err)) + } + + logger.Error("Template build failed according to status") + return + } + + if status.Metadata == nil { + logger.Error("Metadata not found in template build status") + return + } + + err = tm.db.FinishEnvBuild(childCtx, templateID, buildID, int64(status.Metadata.RootfsSizeKey), status.Metadata.EnvdVersionKey) + if err != nil { + logger.Error("Error when finishing build", zap.Error(err)) + return + } + } + } +} + +func (tm *TemplateManager) removeFromProcessingQueue(buildID uuid.UUID) { + tm.lock.Lock() + delete(tm.processing, buildID) + tm.lock.Unlock() +} + +func (tm *TemplateManager) createInProcessingQueue(buildID uuid.UUID, templateID string) bool { + tm.lock.Lock() + defer tm.lock.Unlock() + + _, exists := tm.processing[buildID] + if exists { + // already in processing queue, skip + return true + } + + tm.processing[buildID] = struct{ templateID string }{templateID: templateID} + return false +} diff --git a/packages/shared/pkg/db/envs.go b/packages/shared/pkg/db/envs.go index 443dcdc28..521448cd6 100644 --- a/packages/shared/pkg/db/envs.go +++ b/packages/shared/pkg/db/envs.go @@ -164,6 +164,63 @@ func (db *DB) GetEnv(ctx context.Context, aliasOrEnvID string) (result *Template }, build, nil } +func (db *DB) GetRunningEnvBuilds(ctx context.Context) ([]*models.EnvBuild, error) { + envBuilds, err := db. + Client. + EnvBuild. + Query(). + Where(envbuild.StatusIn(envbuild.StatusWaiting, envbuild.StatusBuilding)). + Order(models.Desc(envbuild.FieldCreatedAt)). + All(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get running env builds: %w", err) + } + + return envBuilds, nil +} + +func (db *DB) GetRunningEnvBuild(ctx context.Context, envID string, excludedBuildUUIDs []uuid.UUID) (build *models.EnvBuild, err error) { + dbBuild, err := db. + Client. + EnvBuild. + Query(). + Where( + envbuild.And( + envbuild.EnvID(envID), + envbuild.StatusIn(envbuild.StatusWaiting, envbuild.StatusBuilding), + envbuild.IDNotIn(excludedBuildUUIDs...), + ), + ). + Order(models.Desc(envbuild.FieldCreatedAt)). + Limit(10). + First(ctx) + + if err != nil { + return nil, fmt.Errorf("failed to get latest env builds '%s': %w", envID, err) + } + + return dbBuild, nil +} + +func (db *DB) GetEnvBuild(ctx context.Context, buildID uuid.UUID) (build *models.EnvBuild, err error) { + dbBuild, err := db. + Client. + EnvBuild. + Query(). + Where(envbuild.ID(buildID)). + Limit(1). + First(ctx) + + notFound := models.IsNotFound(err) + if notFound { + return nil, fmt.Errorf("template build '%s' not found", buildID) + } else if err != nil { + return nil, fmt.Errorf("failed to get env build '%s': %w", buildID, err) + } + + return dbBuild, nil +} + func (db *DB) CheckBaseEnvHasSnapshots(ctx context.Context, envID string) (result bool, err error) { result, err = db.Client.Snapshot.Query().Where(snapshot.BaseEnvID(envID)).Exist(ctx) if err != nil { From 56588664674e8352cf1479d7e229711dbe104267 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:33:03 +0100 Subject: [PATCH 13/78] adjusted build cache ttl --- packages/template-manager/internal/cache/build_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/template-manager/internal/cache/build_cache.go b/packages/template-manager/internal/cache/build_cache.go index 3c6f93eab..208b8003e 100644 --- a/packages/template-manager/internal/cache/build_cache.go +++ b/packages/template-manager/internal/cache/build_cache.go @@ -13,7 +13,7 @@ import ( ) const ( - buildInfoExpiration = time.Minute * 5 // 5 minutes + buildInfoExpiration = time.Minute * 30 // 30 minutes ) type BuildInfo struct { From 7124b93072c49e067bb39ed68f02ac0feddb7e01 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:33:16 +0100 Subject: [PATCH 14/78] sorted imports --- packages/template-manager/internal/server/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/template-manager/internal/server/main.go b/packages/template-manager/internal/server/main.go index c24e33e49..fe7306e95 100644 --- a/packages/template-manager/internal/server/main.go +++ b/packages/template-manager/internal/server/main.go @@ -2,13 +2,12 @@ package server import ( "context" - "github.com/e2b-dev/infra/packages/template-manager/internal/build" - "github.com/e2b-dev/infra/packages/template-manager/internal/cache" "time" artifactregistry "cloud.google.com/go/artifactregistry/apiv1" "github.com/docker/docker/client" docker "github.com/fsouza/go-dockerclient" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/selector" @@ -24,6 +23,9 @@ import ( e2bgrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc" templatemanager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" l "github.com/e2b-dev/infra/packages/shared/pkg/logger" + + "github.com/e2b-dev/infra/packages/template-manager/internal/build" + "github.com/e2b-dev/infra/packages/template-manager/internal/cache" "github.com/e2b-dev/infra/packages/template-manager/internal/constants" "github.com/e2b-dev/infra/packages/template-manager/internal/template" ) From 2b3dfd6d924645f8c62a32a7123f2485f6b52ef0 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:35:56 +0100 Subject: [PATCH 15/78] added telemetry record for template creation --- packages/api/internal/template-manager/create_template.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/api/internal/template-manager/create_template.go b/packages/api/internal/template-manager/create_template.go index c3e1d2e9e..2e74404da 100644 --- a/packages/api/internal/template-manager/create_template.go +++ b/packages/api/internal/template-manager/create_template.go @@ -7,6 +7,7 @@ import ( "github.com/e2b-dev/infra/packages/api/internal/sandbox" "github.com/e2b-dev/infra/packages/api/internal/utils" "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" "github.com/google/uuid" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -56,5 +57,7 @@ func (tm *TemplateManager) CreateTemplate( return fmt.Errorf("failed to create template '%s': %w", templateID, err) } + telemetry.ReportEvent(childCtx, "Template build started") + return nil } From 690798d386bc40b00e1c722b769baab97176f5dd Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:56:39 +0100 Subject: [PATCH 16/78] cleanup template start build endpoint --- .../api/internal/handlers/template_start_build.go | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index 7246a8162..5c3e22242 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -148,6 +148,8 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template telemetry.ReportCriticalError(ctx, err) } + telemetry.ReportEvent(ctx, "created new environment", attribute.String("env.id", templateID)) + // Do not wait for global build sync trigger it immediately go func() { a.templateManager.BuildStatusSync(context.Background(), buildUUID, templateID) @@ -156,19 +158,6 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template a.templateCache.Invalidate(templateID) }() - // todo - //telemetry.ReportEvent(childCtx, "created new environment", attribute.String("env.id", templateID)) - - // todo - //// status building must be set after build is triggered because then - //// it's possible build status job will be triggered before build cache on template manager is created and build will fail - //err = a.db.EnvBuildSetStatus(ctx, envDB.ID, buildUUID, envbuild.StatusBuilding) - //if err != nil { - // err = fmt.Errorf("error when setting build status: %w", err) - // telemetry.ReportCriticalError(ctx, err) - // return - //} - a.posthog.CreateAnalyticsUserEvent(userID.String(), team.ID.String(), "built environment", posthog.NewProperties(). Set("user_id", userID). Set("environment", templateID). From 4c70a2b5832af55d9c5d7a3dc3c9144042b078db Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:56:47 +0100 Subject: [PATCH 17/78] removed unused cache calls --- .../internal/server/create_template.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index ec61a99f6..7fedb0ad2 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -57,29 +57,15 @@ func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *templ return nil, fmt.Errorf("error while creating build cache: %w", err) } - go func() { - err = s.builder.Builder(childCtx, template, config.TemplateID, config.BuildID) if err != nil { s.logger.Error("Error while building template", zap.Error(err)) telemetry.ReportEvent(childCtx, "Environment built failed") } - cacheIn, cacheErr := s.buildCache.Get(config.BuildID) - if cacheErr != nil { - zap.L().Error("template creation cache fetch failed", zap.Error(cacheErr)) - } - telemetry.ReportEvent(childCtx, "Environment built") }() - cacheIn, cacheErr := s.buildCache.Get(config.BuildID) - if cacheErr != nil { - zap.L().Error("template creation cache fetch failed", zap.Error(cacheErr)) - return nil, fmt.Errorf("error while getting build info, maybe already expired") - } - - return nil, nil } From c5a6f9c7b449a48e9bab95186fd82e07d3197538 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 11:57:02 +0100 Subject: [PATCH 18/78] enabled background sync for tempalte builds --- packages/api/internal/template-manager/template_manager.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 08df6fa3b..ee039e9af 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -46,12 +46,6 @@ func (tm *TemplateManager) Close() error { } func (tm *TemplateManager) BuildsStatusPeriodicalSync() { - // todo: disabled debugging - - // todo: implement different timeout for different states - // todo: how to do not fast-cancel new build? - return - for { buildsRunning, err := tm.db.GetRunningEnvBuilds(context.Background()) if err != nil { @@ -68,6 +62,7 @@ func (tm *TemplateManager) BuildsStatusPeriodicalSync() { } } +// BuildStatusSync todo: implement failing build when there is build in some state for too long func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUID, templateID string) { childCtx, childCtxCancel := context.WithTimeout(ctx, syncTimeout) defer childCtxCancel() From ee8527e3bed5279a407caa38eb8d0983677fa84c Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 12:21:45 +0100 Subject: [PATCH 19/78] lowered limit for template manager cache --- packages/template-manager/internal/cache/build_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/template-manager/internal/cache/build_cache.go b/packages/template-manager/internal/cache/build_cache.go index 208b8003e..2fd1adb8d 100644 --- a/packages/template-manager/internal/cache/build_cache.go +++ b/packages/template-manager/internal/cache/build_cache.go @@ -13,7 +13,7 @@ import ( ) const ( - buildInfoExpiration = time.Minute * 30 // 30 minutes + buildInfoExpiration = time.Minute * 10 // 10 minutes ) type BuildInfo struct { From 5a8c14132451eb17fc2f7446ecdc75f974368d33 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 12:22:34 +0100 Subject: [PATCH 20/78] mark build failed when error during build in template-manager --- .../template-manager/internal/server/create_template.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 7fedb0ad2..449d34682 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -60,8 +60,14 @@ func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *templ go func() { err = s.builder.Builder(childCtx, template, config.TemplateID, config.BuildID) if err != nil { + cacheErr := s.buildCache.SetFailed(config.TemplateID, config.BuildID) + if cacheErr != nil { + s.logger.Error("Error during failing template build in cache", zap.Error(err)) + } + s.logger.Error("Error while building template", zap.Error(err)) telemetry.ReportEvent(childCtx, "Environment built failed") + return } telemetry.ReportEvent(childCtx, "Environment built") From 5270e417f56fab79f09f018e22cd39c2e93cfa1d Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 12:47:34 +0100 Subject: [PATCH 21/78] fail builds in waiting phase after it takes too long --- .../template-manager/template_manager.go | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index ee039e9af..8fe144eea 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -25,6 +25,7 @@ var ( // todo: just for testing syncInterval = time.Second * 10 syncTimeout = time.Minute * 5 + syncWaitingStateDeadline = time.Minute * 10 ) func New(db *db.DB) (*TemplateManager, error) { @@ -77,6 +78,27 @@ func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUI logger := zap.L().With(zap.String("buildID", buildID.String()), zap.String("envID", templateID)) + envBuildDb, err := tm.db.GetEnvBuild(childCtx, buildID) + if err != nil { + logger.Error("Error when fetching env build for background sync", zap.Error(err)) + return + } + + // waiting for build to start, local docker build and push can take some time + // so just check if it's not too long + if envBuildDb.Status == envbuild.StatusWaiting { + // if waiting for too long, fail the build + if time.Since(envBuildDb.CreatedAt) > syncWaitingStateDeadline { + logger.Error("Build is in waiting state for too long, failing it") + + dbErr := tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) + if dbErr != nil { + logger.Error("Error when setting build status", zap.Error(dbErr)) + } + return + } + } + // stream build status stream, err := tm.grpc.Client.TemplateBuildStatus(childCtx, &template_manager.TemplateStatusRequest{BuildID: buildID.String(), TemplateID: templateID}) if utils.UnwrapGRPCError(err) != nil { From 492fb9dd6c278884fdd15a11c9bef14f025e5a7f Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 12:48:11 +0100 Subject: [PATCH 22/78] background sync of template builds adjusted to one per minute --- packages/api/internal/template-manager/template_manager.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 8fe144eea..cffafdcad 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -21,10 +21,8 @@ type TemplateManager struct { } var ( - //syncInterval = time.Minute * 1 - // todo: just for testing - syncInterval = time.Second * 10 - syncTimeout = time.Minute * 5 + syncInterval = time.Minute * 1 + syncTimeout = time.Minute * 5 syncWaitingStateDeadline = time.Minute * 10 ) From 27e8f50748706d0f4fca41d870fee3588725c6b3 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Mon, 10 Mar 2025 16:22:53 +0100 Subject: [PATCH 23/78] build status sync that support ctx --- packages/api/internal/handlers/store.go | 2 +- .../template-manager/template_manager.go | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 38c36d1a3..7404ffba3 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -128,7 +128,7 @@ func NewAPIStore(ctx context.Context) *APIStore { } // Start the periodic sync of template builds statuses - go templateManager.BuildsStatusPeriodicalSync() + go templateManager.BuildsStatusPeriodicalSync(ctx) var lokiClient *loki.DefaultClient if laddr := os.Getenv("LOKI_ADDRESS"); laddr != "" { diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index cffafdcad..77ed382cc 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -44,20 +44,24 @@ func (tm *TemplateManager) Close() error { return tm.grpc.Close() } -func (tm *TemplateManager) BuildsStatusPeriodicalSync() { +func (tm *TemplateManager) BuildsStatusPeriodicalSync(ctx context.Context) { for { - buildsRunning, err := tm.db.GetRunningEnvBuilds(context.Background()) - if err != nil { - zap.L().Error("Error getting running builds for periodical sync", zap.Error(err)) - continue - } + select { + case <-ctx.Done(): + return + case <-time.After(syncInterval): + buildsRunning, err := tm.db.GetRunningEnvBuilds(ctx) + if err != nil { + zap.L().Error("Error getting running builds for periodical sync", zap.Error(err)) + time.Sleep(time.Second) // just not hammer database if there is an error + continue + } - zap.L().Info("Running periodical sync of builds statuses", zap.Int("count", len(buildsRunning))) - for _, buildDB := range buildsRunning { - go tm.BuildStatusSync(context.Background(), buildDB.ID, *buildDB.EnvID) + zap.L().Info("Running periodical sync of builds statuses", zap.Int("count", len(buildsRunning))) + for _, buildDB := range buildsRunning { + go tm.BuildStatusSync(ctx, buildDB.ID, *buildDB.EnvID) + } } - - time.Sleep(syncInterval) } } From b3e6c6e3c66dc66c34c93e3275046c8efd07edce Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 10:15:29 +0100 Subject: [PATCH 24/78] changed order of params in protobuf spec --- .../pkg/grpc/template-manager/template-manager.pb.go | 8 ++++---- packages/template-manager/template-manager.proto | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go index 8f8b0dcea..242692424 100644 --- a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go +++ b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go @@ -287,8 +287,8 @@ type TemplateBuildMetadata struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - RootfsSizeKey int32 `protobuf:"varint,3,opt,name=rootfsSizeKey,proto3" json:"rootfsSizeKey,omitempty"` - EnvdVersionKey string `protobuf:"bytes,4,opt,name=envdVersionKey,proto3" json:"envdVersionKey,omitempty"` + RootfsSizeKey int32 `protobuf:"varint,1,opt,name=rootfsSizeKey,proto3" json:"rootfsSizeKey,omitempty"` + EnvdVersionKey string `protobuf:"bytes,2,opt,name=envdVersionKey,proto3" json:"envdVersionKey,omitempty"` } func (x *TemplateBuildMetadata) Reset() { @@ -443,9 +443,9 @@ var file_template_manager_proto_rawDesc = []byte{ 0x69, 0x6c, 0x64, 0x49, 0x44, 0x22, 0x65, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x24, 0x0a, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6e, + 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x22, 0x7d, 0x0a, 0x1b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, diff --git a/packages/template-manager/template-manager.proto b/packages/template-manager/template-manager.proto index fd984fc58..05b484046 100644 --- a/packages/template-manager/template-manager.proto +++ b/packages/template-manager/template-manager.proto @@ -34,8 +34,8 @@ message TemplateBuildDeleteRequest { } message TemplateBuildMetadata { - int32 rootfsSizeKey = 3; - string envdVersionKey = 4; + int32 rootfsSizeKey = 1; + string envdVersionKey = 2; } // Logs from template build From 48ab346952312a5bab93cd7d63cf12fcea4b07e9 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 10:17:40 +0100 Subject: [PATCH 25/78] fixed missed limits --- packages/shared/pkg/db/envs.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/shared/pkg/db/envs.go b/packages/shared/pkg/db/envs.go index 521448cd6..74da725a4 100644 --- a/packages/shared/pkg/db/envs.go +++ b/packages/shared/pkg/db/envs.go @@ -192,7 +192,6 @@ func (db *DB) GetRunningEnvBuild(ctx context.Context, envID string, excludedBuil ), ). Order(models.Desc(envbuild.FieldCreatedAt)). - Limit(10). First(ctx) if err != nil { @@ -208,7 +207,6 @@ func (db *DB) GetEnvBuild(ctx context.Context, buildID uuid.UUID) (build *models EnvBuild. Query(). Where(envbuild.ID(buildID)). - Limit(1). First(ctx) notFound := models.IsNotFound(err) From d28585d91b13fb47cf59a77c99d280b6d8283505 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 10:17:53 +0100 Subject: [PATCH 26/78] do not create separate ctx with telemetry context --- .../template-manager/internal/server/template_status.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/template-manager/internal/server/template_status.go b/packages/template-manager/internal/server/template_status.go index 4cb434196..25b35872c 100644 --- a/packages/template-manager/internal/server/template_status.go +++ b/packages/template-manager/internal/server/template_status.go @@ -15,8 +15,8 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq ctx, cancel := context.WithTimeout(stream.Context(), statusTimeout) defer cancel() - childCtx, childSpan := s.tracer.Start(ctx, "template-build-status-request") - defer childSpan.End() + ctx, ctxSpan := s.tracer.Start(ctx, "template-build-status-request") + defer ctxSpan.End() logger := s.logger.With(zap.String("buildID", in.BuildID), zap.String("envID", in.TemplateID)) @@ -50,7 +50,7 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq if err != nil { logger.Error("Error while sending status stream back to API", zap.Error(err)) - telemetry.ReportError(childCtx, err) + telemetry.ReportError(ctx, err) return err } From 38e6437e1d0b6feb0653f73e06c4c90a834f200c Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 10:37:07 +0100 Subject: [PATCH 27/78] use state-machine enumerations instead of bools --- .../template-manager/template_manager.go | 27 ++-- .../template-manager/template-manager.pb.go | 133 ++++++++++++------ .../internal/build/{builder.go => build.go} | 11 +- .../internal/cache/build_cache.go | 60 +++----- .../internal/server/create_template.go | 2 +- .../internal/server/template_status.go | 13 +- .../template-manager/template-manager.proto | 12 +- 7 files changed, 139 insertions(+), 119 deletions(-) rename packages/template-manager/internal/build/{builder.go => build.go} (89%) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 77ed382cc..a5f6d16e2 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -129,24 +129,21 @@ func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUI return } - if status.Done { - // marked as finished but failed - if status.Failed { - err = tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) - if err != nil { - logger.Error("Error when setting build status", zap.Error(err)) - } - - logger.Error("Template build failed according to status") - return + // build failed + if status.GetStatus() == template_manager.TemplateBuildState_Failed { + err = tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) + if err != nil { + logger.Error("Error when setting build status", zap.Error(err)) } - if status.Metadata == nil { - logger.Error("Metadata not found in template build status") - return - } + logger.Error("Template build failed according to status") + return + } - err = tm.db.FinishEnvBuild(childCtx, templateID, buildID, int64(status.Metadata.RootfsSizeKey), status.Metadata.EnvdVersionKey) + // build completed + if status.GetStatus() == template_manager.TemplateBuildState_Completed { + meta := status.GetMetadata() + err = tm.db.FinishEnvBuild(childCtx, templateID, buildID, int64(meta.RootfsSizeKey), meta.EnvdVersionKey) if err != nil { logger.Error("Error when finishing build", zap.Error(err)) return diff --git a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go index 242692424..00c19c294 100644 --- a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go +++ b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go @@ -21,6 +21,55 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type TemplateBuildState int32 + +const ( + TemplateBuildState_Building TemplateBuildState = 0 + TemplateBuildState_Failed TemplateBuildState = 1 + TemplateBuildState_Completed TemplateBuildState = 2 +) + +// Enum value maps for TemplateBuildState. +var ( + TemplateBuildState_name = map[int32]string{ + 0: "Building", + 1: "Failed", + 2: "Completed", + } + TemplateBuildState_value = map[string]int32{ + "Building": 0, + "Failed": 1, + "Completed": 2, + } +) + +func (x TemplateBuildState) Enum() *TemplateBuildState { + p := new(TemplateBuildState) + *p = x + return p +} + +func (x TemplateBuildState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TemplateBuildState) Descriptor() protoreflect.EnumDescriptor { + return file_template_manager_proto_enumTypes[0].Descriptor() +} + +func (TemplateBuildState) Type() protoreflect.EnumType { + return &file_template_manager_proto_enumTypes[0] +} + +func (x TemplateBuildState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TemplateBuildState.Descriptor instead. +func (TemplateBuildState) EnumDescriptor() ([]byte, []int) { + return file_template_manager_proto_rawDescGZIP(), []int{0} +} + type TemplateConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -343,9 +392,8 @@ type TemplateBuildStatusResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Done bool `protobuf:"varint,1,opt,name=done,proto3" json:"done,omitempty"` - Failed bool `protobuf:"varint,2,opt,name=failed,proto3" json:"failed,omitempty"` - Metadata *TemplateBuildMetadata `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` + Status TemplateBuildState `protobuf:"varint,1,opt,name=status,proto3,enum=TemplateBuildState" json:"status,omitempty"` + Metadata *TemplateBuildMetadata `protobuf:"bytes,2,opt,name=metadata,proto3" json:"metadata,omitempty"` } func (x *TemplateBuildStatusResponse) Reset() { @@ -380,18 +428,11 @@ func (*TemplateBuildStatusResponse) Descriptor() ([]byte, []int) { return file_template_manager_proto_rawDescGZIP(), []int{5} } -func (x *TemplateBuildStatusResponse) GetDone() bool { +func (x *TemplateBuildStatusResponse) GetStatus() TemplateBuildState { if x != nil { - return x.Done + return x.Status } - return false -} - -func (x *TemplateBuildStatusResponse) GetFailed() bool { - if x != nil { - return x.Failed - } - return false + return TemplateBuildState_Building } func (x *TemplateBuildStatusResponse) GetMetadata() *TemplateBuildMetadata { @@ -446,15 +487,19 @@ var file_template_manager_proto_rawDesc = []byte{ 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6e, - 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x22, 0x7d, 0x0a, 0x1b, + 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x22, 0x7e, 0x0a, 0x1b, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x3d, 0x0a, 0x12, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, - 0x6f, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x12, - 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x32, 0xee, 0x01, 0x0a, 0x0f, + 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x00, + 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x10, 0x02, 0x32, 0xee, 0x01, 0x0a, 0x0f, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, @@ -488,30 +533,33 @@ func file_template_manager_proto_rawDescGZIP() []byte { return file_template_manager_proto_rawDescData } +var file_template_manager_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_template_manager_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_template_manager_proto_goTypes = []interface{}{ - (*TemplateConfig)(nil), // 0: TemplateConfig - (*TemplateCreateRequest)(nil), // 1: TemplateCreateRequest - (*TemplateStatusRequest)(nil), // 2: TemplateStatusRequest - (*TemplateBuildDeleteRequest)(nil), // 3: TemplateBuildDeleteRequest - (*TemplateBuildMetadata)(nil), // 4: TemplateBuildMetadata - (*TemplateBuildStatusResponse)(nil), // 5: TemplateBuildStatusResponse - (*emptypb.Empty)(nil), // 6: google.protobuf.Empty + (TemplateBuildState)(0), // 0: TemplateBuildState + (*TemplateConfig)(nil), // 1: TemplateConfig + (*TemplateCreateRequest)(nil), // 2: TemplateCreateRequest + (*TemplateStatusRequest)(nil), // 3: TemplateStatusRequest + (*TemplateBuildDeleteRequest)(nil), // 4: TemplateBuildDeleteRequest + (*TemplateBuildMetadata)(nil), // 5: TemplateBuildMetadata + (*TemplateBuildStatusResponse)(nil), // 6: TemplateBuildStatusResponse + (*emptypb.Empty)(nil), // 7: google.protobuf.Empty } var file_template_manager_proto_depIdxs = []int32{ - 0, // 0: TemplateCreateRequest.template:type_name -> TemplateConfig - 4, // 1: TemplateBuildStatusResponse.metadata:type_name -> TemplateBuildMetadata - 1, // 2: TemplateService.TemplateCreate:input_type -> TemplateCreateRequest - 2, // 3: TemplateService.TemplateBuildStatus:input_type -> TemplateStatusRequest - 3, // 4: TemplateService.TemplateBuildDelete:input_type -> TemplateBuildDeleteRequest - 6, // 5: TemplateService.TemplateCreate:output_type -> google.protobuf.Empty - 5, // 6: TemplateService.TemplateBuildStatus:output_type -> TemplateBuildStatusResponse - 6, // 7: TemplateService.TemplateBuildDelete:output_type -> google.protobuf.Empty - 5, // [5:8] is the sub-list for method output_type - 2, // [2:5] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 1, // 0: TemplateCreateRequest.template:type_name -> TemplateConfig + 0, // 1: TemplateBuildStatusResponse.status:type_name -> TemplateBuildState + 5, // 2: TemplateBuildStatusResponse.metadata:type_name -> TemplateBuildMetadata + 2, // 3: TemplateService.TemplateCreate:input_type -> TemplateCreateRequest + 3, // 4: TemplateService.TemplateBuildStatus:input_type -> TemplateStatusRequest + 4, // 5: TemplateService.TemplateBuildDelete:input_type -> TemplateBuildDeleteRequest + 7, // 6: TemplateService.TemplateCreate:output_type -> google.protobuf.Empty + 6, // 7: TemplateService.TemplateBuildStatus:output_type -> TemplateBuildStatusResponse + 7, // 8: TemplateService.TemplateBuildDelete:output_type -> google.protobuf.Empty + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_template_manager_proto_init() } @@ -598,13 +646,14 @@ func file_template_manager_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_template_manager_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 6, NumExtensions: 0, NumServices: 1, }, GoTypes: file_template_manager_proto_goTypes, DependencyIndexes: file_template_manager_proto_depIdxs, + EnumInfos: file_template_manager_proto_enumTypes, MessageInfos: file_template_manager_proto_msgTypes, }.Build() File_template_manager_proto = out.File diff --git a/packages/template-manager/internal/build/builder.go b/packages/template-manager/internal/build/build.go similarity index 89% rename from packages/template-manager/internal/build/builder.go rename to packages/template-manager/internal/build/build.go index 8480a7a7d..06cf4c4eb 100644 --- a/packages/template-manager/internal/build/builder.go +++ b/packages/template-manager/internal/build/build.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/docker/docker/client" + template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" "github.com/e2b-dev/infra/packages/shared/pkg/storage" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" "github.com/e2b-dev/infra/packages/template-manager/internal/cache" @@ -42,9 +43,9 @@ func NewBuilder(logger *zap.Logger, buildLogger *zap.Logger, tracer trace.Tracer } } -func (b *TemplateBuilder) Builder(ctx context.Context, template *Env, envID string, buildID string) error { +func (b *TemplateBuilder) Build(ctx context.Context, template *Env, envID string, buildID string) error { buildStorage := b.templateStorage.NewBuild(template.TemplateFiles) - buildEntry, err := b.buildCache.Get(buildID) + _, err := b.buildCache.Get(buildID) if err != nil { return err } @@ -125,10 +126,8 @@ func (b *TemplateBuilder) Builder(ctx context.Context, template *Env, envID stri } } - buildEntry.SetEnvdVersionKey(strings.TrimSpace(string(out))) - buildEntry.SetRootFsSizeKey(int32(template.RootfsSizeMB())) - - err = b.buildCache.SetSucceeded(envID, buildID) + buildMetadata := &template_manager.TemplateBuildMetadata{RootfsSizeKey: int32(template.RootfsSizeMB()), EnvdVersionKey: strings.TrimSpace(string(out))} + err = b.buildCache.SetSucceeded(envID, buildID, buildMetadata) if err != nil { b.logger.Error("Error while setting build state to succeeded", zap.Error(err)) telemetry.ReportError(ctx, err) diff --git a/packages/template-manager/internal/cache/build_cache.go b/packages/template-manager/internal/cache/build_cache.go index 2fd1adb8d..773add992 100644 --- a/packages/template-manager/internal/cache/build_cache.go +++ b/packages/template-manager/internal/cache/build_cache.go @@ -3,6 +3,7 @@ package cache import ( "context" "fmt" + template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" "github.com/e2b-dev/infra/packages/shared/pkg/meters" "github.com/jellydator/ttlcache/v3" "go.opentelemetry.io/otel/attribute" @@ -17,63 +18,45 @@ const ( ) type BuildInfo struct { - envID string - ended bool - failed bool - - rootfsSizeKey int32 - envdVersionKey string - - mu sync.RWMutex + envID string + status template_manager.TemplateBuildState + metadata *template_manager.TemplateBuildMetadata + mu sync.RWMutex } func (b *BuildInfo) IsRunning() bool { b.mu.RLock() defer b.mu.RUnlock() - return !b.ended + return b.status == template_manager.TemplateBuildState_Building } func (b *BuildInfo) IsFailed() bool { b.mu.RLock() defer b.mu.RUnlock() - return b.failed -} - -func (b *BuildInfo) SetRootFsSizeKey(size int32) { - b.mu.RLock() - defer b.mu.RUnlock() - - b.rootfsSizeKey = size -} - -func (b *BuildInfo) GetRootFsSizeKey() int32 { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.rootfsSizeKey + return b.status == template_manager.TemplateBuildState_Failed } -func (b *BuildInfo) SetEnvdVersionKey(version string) { +func (b *BuildInfo) GetBuildEnvID() string { b.mu.RLock() defer b.mu.RUnlock() - b.envdVersionKey = version + return b.envID } -func (b *BuildInfo) GetEnvdVersionKey() string { +func (b *BuildInfo) GetMetadata() *template_manager.TemplateBuildMetadata { b.mu.RLock() defer b.mu.RUnlock() - return b.envdVersionKey + return b.metadata } -func (b *BuildInfo) GetBuildEnvID() string { +func (b *BuildInfo) GetStatus() template_manager.TemplateBuildState { b.mu.RLock() defer b.mu.RUnlock() - return b.envID + return b.status } type BuildCache struct { @@ -124,9 +107,9 @@ func (c *BuildCache) Create(buildID string, envID string) error { } info := BuildInfo{ - envID: envID, - ended: false, - failed: false, + envID: envID, + status: template_manager.TemplateBuildState_Building, + metadata: nil, } c.cache.Set(buildID, &info, buildInfoExpiration) @@ -135,7 +118,7 @@ func (c *BuildCache) Create(buildID string, envID string) error { return nil } -func (c *BuildCache) SetSucceeded(envID string, buildID string) error { +func (c *BuildCache) SetSucceeded(envID string, buildID string, metadata *template_manager.TemplateBuildMetadata) error { c.mu.Lock() defer c.mu.Unlock() @@ -144,9 +127,8 @@ func (c *BuildCache) SetSucceeded(envID string, buildID string) error { return fmt.Errorf("build %s not found in cache: %w", buildID, err) } - item.ended = true - item.failed = false - + item.status = template_manager.TemplateBuildState_Completed + item.metadata = metadata c.updateCounter(envID, buildID, -1) return nil } @@ -160,9 +142,7 @@ func (c *BuildCache) SetFailed(envID string, buildID string) error { return fmt.Errorf("build %s not found in cache: %w", buildID, err) } - item.ended = true - item.failed = true - + item.status = template_manager.TemplateBuildState_Failed c.updateCounter(envID, buildID, -1) return nil } diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 449d34682..7b2dd020a 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -58,7 +58,7 @@ func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *templ } go func() { - err = s.builder.Builder(childCtx, template, config.TemplateID, config.BuildID) + err = s.builder.Build(childCtx, template, config.TemplateID, config.BuildID) if err != nil { cacheErr := s.buildCache.SetFailed(config.TemplateID, config.BuildID) if cacheErr != nil { diff --git a/packages/template-manager/internal/server/template_status.go b/packages/template-manager/internal/server/template_status.go index 25b35872c..a301acb63 100644 --- a/packages/template-manager/internal/server/template_status.go +++ b/packages/template-manager/internal/server/template_status.go @@ -27,14 +27,6 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq return fmt.Errorf("error while getting build info, maybe already expired") } - var metadata *template_manager.TemplateBuildMetadata = nil - if !buildInfo.IsRunning() { - metadata = &template_manager.TemplateBuildMetadata{ - RootfsSizeKey: buildInfo.GetRootFsSizeKey(), - EnvdVersionKey: buildInfo.GetEnvdVersionKey(), - } - } - if buildInfo.IsFailed() { logger.Error("Template build failed") return fmt.Errorf("template build failed") @@ -42,9 +34,8 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq err = stream.Send( &template_manager.TemplateBuildStatusResponse{ - Done: !buildInfo.IsRunning(), - Failed: buildInfo.IsFailed(), - Metadata: metadata, + Status: buildInfo.GetStatus(), + Metadata: buildInfo.GetMetadata(), }, ) diff --git a/packages/template-manager/template-manager.proto b/packages/template-manager/template-manager.proto index 05b484046..85be5ead2 100644 --- a/packages/template-manager/template-manager.proto +++ b/packages/template-manager/template-manager.proto @@ -38,12 +38,16 @@ message TemplateBuildMetadata { string envdVersionKey = 2; } +enum TemplateBuildState { + Building = 0; + Failed = 1; + Completed = 2; +} + // Logs from template build message TemplateBuildStatusResponse { - bool done = 1; - bool failed = 2; - - TemplateBuildMetadata metadata = 3; + TemplateBuildState status = 1; + TemplateBuildMetadata metadata = 2; } // Interface exported by the server. From 95155b8b80a4124d435abac72442fc51c9712063 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 10:38:41 +0100 Subject: [PATCH 28/78] dockerbuild and docker push can take up to 20 minutes now --- packages/api/internal/template-manager/template_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index a5f6d16e2..5a731dd94 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -23,7 +23,7 @@ type TemplateManager struct { var ( syncInterval = time.Minute * 1 syncTimeout = time.Minute * 5 - syncWaitingStateDeadline = time.Minute * 10 + syncWaitingStateDeadline = time.Minute * 20 ) func New(db *db.DB) (*TemplateManager, error) { From d2016e7272b950df270fefa60a18965fcd5d7f75 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 11:00:39 +0100 Subject: [PATCH 29/78] custom error type for not found template build --- packages/api/internal/handlers/template_build_logs.go | 10 +++++++++- packages/shared/pkg/db/envs.go | 2 +- packages/shared/pkg/db/errors.go | 7 +++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 86b8a5d3c..da1808788 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -2,7 +2,9 @@ package handlers import ( "encoding/json" + "errors" "fmt" + "github.com/e2b-dev/infra/packages/shared/pkg/db" "net/http" "strings" "time" @@ -79,7 +81,13 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem if err != nil { errMsg := fmt.Errorf("error when getting build: %w", err) telemetry.ReportError(ctx, errMsg) - a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Build '%s' not found", buildID)) + + if errors.Is(err, db.TemplateBuildNotFound{}) { + a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Build '%s' not found", buildID)) + return + } + + a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error during build '%s'", buildID)) return } diff --git a/packages/shared/pkg/db/envs.go b/packages/shared/pkg/db/envs.go index 74da725a4..c8af9f861 100644 --- a/packages/shared/pkg/db/envs.go +++ b/packages/shared/pkg/db/envs.go @@ -211,7 +211,7 @@ func (db *DB) GetEnvBuild(ctx context.Context, buildID uuid.UUID) (build *models notFound := models.IsNotFound(err) if notFound { - return nil, fmt.Errorf("template build '%s' not found", buildID) + return nil, TemplateBuildNotFound{} } else if err != nil { return nil, fmt.Errorf("failed to get env build '%s': %w", buildID, err) } diff --git a/packages/shared/pkg/db/errors.go b/packages/shared/pkg/db/errors.go index f69df2566..3745ca610 100644 --- a/packages/shared/pkg/db/errors.go +++ b/packages/shared/pkg/db/errors.go @@ -1,12 +1,19 @@ package db type ErrNotFound error + type TemplateNotFound struct{ ErrNotFound } func (TemplateNotFound) Error() string { return "Template not found" } +type TemplateBuildNotFound struct{ ErrNotFound } + +func (TemplateBuildNotFound) Error() string { + return "Template build not found" +} + type SnapshotNotFound struct{ ErrNotFound } func (SnapshotNotFound) Error() string { From 582a5b86f2e554eb3758b098364cf352a24744e0 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 11:07:31 +0100 Subject: [PATCH 30/78] use struct def instad of inlined one --- .../api/internal/template-manager/template_manager.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 5a731dd94..193b832d4 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -13,11 +13,15 @@ import ( "time" ) +type processingBuilds struct { + templateID string +} + type TemplateManager struct { grpc *GRPCClient db *db.DB lock sync.Mutex - processing map[uuid.UUID]struct{ templateID string } + processing map[uuid.UUID]processingBuilds } var ( @@ -36,7 +40,7 @@ func New(db *db.DB) (*TemplateManager, error) { grpc: client, db: db, lock: sync.Mutex{}, - processing: make(map[uuid.UUID]struct{ templateID string }), + processing: make(map[uuid.UUID]processingBuilds), }, nil } @@ -168,6 +172,6 @@ func (tm *TemplateManager) createInProcessingQueue(buildID uuid.UUID, templateID return true } - tm.processing[buildID] = struct{ templateID string }{templateID: templateID} + tm.processing[buildID] = processingBuilds{templateID: templateID} return false } From 9dc8e44fc6fee9dac9408418819481f11e6d7a02 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 11:57:39 +0100 Subject: [PATCH 31/78] proper catch error during template build fetch --- .../internal/handlers/template_start_build.go | 22 +++++++++++++++---- packages/shared/pkg/db/envs.go | 7 ++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index 5c3e22242..10546e9e8 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -2,7 +2,9 @@ package handlers import ( "context" + "errors" "fmt" + "github.com/e2b-dev/infra/packages/shared/pkg/db" "net/http" "time" @@ -90,15 +92,27 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template // Check if some other build from same template is in process, excludes currently created build envBuildDB, err := a.db.GetRunningEnvBuild(ctx, envDB.ID, []uuid.UUID{buildUUID}) - if err == nil && envBuildDB != nil { - a.sendAPIStoreError(c, http.StatusConflict, fmt.Sprintf("there's already running build for %s", templateID)) - err = fmt.Errorf("build is already running build for %s", templateID) - telemetry.ReportCriticalError(ctx, err) + // error is present but different from not found + if err != nil && !errors.Is(err, db.TemplateBuildNotFound{}) { + zap.L().Error("Error when getting running build", zap.Error(err)) + + dbErr := a.db.EnvBuildSetStatus(ctx, templateID, buildUUID, envbuild.StatusFailed) + if dbErr != nil { + zap.L().Error("Error when setting build status", zap.Error(dbErr)) + } + a.sendAPIStoreError(c, http.StatusInternalServerError, "error during template build request") return } + // we found conflicting build for same env + if envBuildDB != nil { + a.sendAPIStoreError(c, http.StatusConflict, fmt.Sprintf("there's already running build for %s", templateID)) + err = fmt.Errorf("build is already running build for %s", templateID) + telemetry.ReportCriticalError(ctx, err) + } + // Trigger the build in the background buildContext, childSpan := a.Tracer.Start( trace.ContextWithSpanContext(context.Background(), span.SpanContext()), diff --git a/packages/shared/pkg/db/envs.go b/packages/shared/pkg/db/envs.go index c8af9f861..3c2855728 100644 --- a/packages/shared/pkg/db/envs.go +++ b/packages/shared/pkg/db/envs.go @@ -194,8 +194,11 @@ func (db *DB) GetRunningEnvBuild(ctx context.Context, envID string, excludedBuil Order(models.Desc(envbuild.FieldCreatedAt)). First(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get latest env builds '%s': %w", envID, err) + notFound := models.IsNotFound(err) + if notFound { + return nil, TemplateBuildNotFound{} + } else if err != nil { + return nil, fmt.Errorf("error during env build fetch '%s': %w", envID, err) } return dbBuild, nil From 0b1b66a9168b2a6b1c7c7cfc5b7d2496738d57ed Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 11:57:54 +0100 Subject: [PATCH 32/78] typo in defer cleanup function in template build --- packages/template-manager/internal/build/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/template-manager/internal/build/build.go b/packages/template-manager/internal/build/build.go index 06cf4c4eb..7b2707946 100644 --- a/packages/template-manager/internal/build/build.go +++ b/packages/template-manager/internal/build/build.go @@ -52,7 +52,7 @@ func (b *TemplateBuilder) Build(ctx context.Context, template *Env, envID string logsWriter := template.BuildLogsWriter - // Remove local template files if build fails + // Remove local template files when exiting defer func() { removeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() From d7b0e15db1f72ffa9aab0ef5b00371e127f8ae1e Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 12:08:42 +0100 Subject: [PATCH 33/78] tracking span created in background sync routine --- .../api/internal/handlers/template_start_build.go | 15 ++++----------- .../internal/template-manager/create_template.go | 12 +++++++----- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index 10546e9e8..134b3e8c6 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -113,13 +113,6 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template telemetry.ReportCriticalError(ctx, err) } - // Trigger the build in the background - buildContext, childSpan := a.Tracer.Start( - trace.ContextWithSpanContext(context.Background(), span.SpanContext()), - "background-build-env", - ) - defer childSpan.End() - startTime := time.Now() build := envDB.Edges.Builds[0] startCmd := "" @@ -130,7 +123,7 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template // Call the Template Manager to build the environment buildErr := a.templateManager.CreateTemplate( a.Tracer, - buildContext, + span.SpanContext(), templateID, buildUUID, build.KernelVersion, @@ -144,11 +137,11 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template if buildErr != nil { buildErr = fmt.Errorf("error when building env: %w", buildErr) zap.L().Error("build failed", zap.Error(buildErr)) - telemetry.ReportCriticalError(buildContext, buildErr) + telemetry.ReportCriticalError(ctx, buildErr) - dbErr := a.db.EnvBuildSetStatus(buildContext, templateID, buildUUID, envbuild.StatusFailed) + dbErr := a.db.EnvBuildSetStatus(ctx, templateID, buildUUID, envbuild.StatusFailed) if dbErr != nil { - telemetry.ReportCriticalError(buildContext, fmt.Errorf("error when setting build status: %w", dbErr)) + telemetry.ReportCriticalError(ctx, fmt.Errorf("error when setting build status: %w", dbErr)) } return diff --git a/packages/api/internal/template-manager/create_template.go b/packages/api/internal/template-manager/create_template.go index 2e74404da..1dc82894c 100644 --- a/packages/api/internal/template-manager/create_template.go +++ b/packages/api/internal/template-manager/create_template.go @@ -15,7 +15,7 @@ import ( func (tm *TemplateManager) CreateTemplate( t trace.Tracer, - ctx context.Context, + span trace.SpanContext, templateID string, buildID uuid.UUID, kernelVersion, @@ -25,12 +25,14 @@ func (tm *TemplateManager) CreateTemplate( diskSizeMB, memoryMB int64, ) error { - childCtx, childSpan := t.Start(ctx, "create-template", + ctx, ctxSpan := t.Start( + trace.ContextWithSpanContext(context.Background(), span), + "background-build-env", trace.WithAttributes( attribute.String("env.id", templateID), ), ) - defer childSpan.End() + defer ctxSpan.End() features, err := sandbox.NewVersionInfo(firecrackerVersion) if err != nil { @@ -39,7 +41,7 @@ func (tm *TemplateManager) CreateTemplate( return errMsg } - _, err = tm.grpc.Client.TemplateCreate(childCtx, &template_manager.TemplateCreateRequest{ + _, err = tm.grpc.Client.TemplateCreate(ctx, &template_manager.TemplateCreateRequest{ Template: &template_manager.TemplateConfig{ TemplateID: templateID, BuildID: buildID.String(), @@ -57,7 +59,7 @@ func (tm *TemplateManager) CreateTemplate( return fmt.Errorf("failed to create template '%s': %w", templateID, err) } - telemetry.ReportEvent(childCtx, "Template build started") + telemetry.ReportEvent(ctx, "Template build started") return nil } From 57b9d73f75e425b5e2210f319a8599a5c167b82a Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 11 Mar 2025 12:24:36 +0100 Subject: [PATCH 34/78] after-rebase fix, removed build cache from apistore struct --- packages/api/internal/handlers/store.go | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 7404ffba3..ee494e979 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -50,7 +50,6 @@ type APIStore struct { Tracer trace.Tracer orchestrator *orchestrator.Orchestrator templateManager *template_manager.TemplateManager - buildCache *builds.BuildCache db *db.DB lokiClient *loki.DefaultClient templateCache *templatecache.TemplateCache From bbddca31d281257c076e00462d470aa9ba0d32fe Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 11:44:21 +0100 Subject: [PATCH 35/78] check if build is waiting state when build trigger --- packages/api/internal/handlers/template_start_build.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index 134b3e8c6..feab7a97d 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -120,6 +120,14 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template startCmd = *build.StartCmd } + // only waiting builds can be triggered + if build.Status != envbuild.StatusWaiting { + a.sendAPIStoreError(c, http.StatusBadRequest, "build is not in waiting state") + err = fmt.Errorf("build is not in waiting state: %s", build.Status) + telemetry.ReportCriticalError(ctx, err) + return + } + // Call the Template Manager to build the environment buildErr := a.templateManager.CreateTemplate( a.Tracer, From e5aca3f1a9328dcb1ab7902cf2530f4ec958aeba Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 12:02:04 +0100 Subject: [PATCH 36/78] make concurrent builds check more readable --- .../internal/handlers/template_start_build.go | 39 ++++++++++--------- packages/shared/pkg/db/envs.go | 25 ------------ 2 files changed, 20 insertions(+), 44 deletions(-) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index feab7a97d..4c6d04a32 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -2,9 +2,7 @@ package handlers import ( "context" - "errors" "fmt" - "github.com/e2b-dev/infra/packages/shared/pkg/db" "net/http" "time" @@ -90,27 +88,30 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template attribute.String("template.id", templateID), ) - // Check if some other build from same template is in process, excludes currently created build - envBuildDB, err := a.db.GetRunningEnvBuild(ctx, envDB.ID, []uuid.UUID{buildUUID}) - - // error is present but different from not found - if err != nil && !errors.Is(err, db.TemplateBuildNotFound{}) { - zap.L().Error("Error when getting running build", zap.Error(err)) - - dbErr := a.db.EnvBuildSetStatus(ctx, templateID, buildUUID, envbuild.StatusFailed) - if dbErr != nil { - zap.L().Error("Error when setting build status", zap.Error(dbErr)) - } - - a.sendAPIStoreError(c, http.StatusInternalServerError, "error during template build request") + concurrentlyRunningBuilds, err := a.db. + Client. + EnvBuild. + Query(). + Where( + envbuild.And( + envbuild.EnvID(envDB.ID), + envbuild.StatusIn(envbuild.StatusWaiting, envbuild.StatusBuilding), + envbuild.IDNotIn(buildUUID), + ), + ). + Count(ctx) + if err != nil { + zap.L().Error("Error when getting count of running builds", zap.Error(err)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "Error during template build request") return } - // we found conflicting build for same env - if envBuildDB != nil { - a.sendAPIStoreError(c, http.StatusConflict, fmt.Sprintf("there's already running build for %s", templateID)) - err = fmt.Errorf("build is already running build for %s", templateID) + // make sure there is no other build in progress for the same template + if concurrentlyRunningBuilds > 0 { + a.sendAPIStoreError(c, http.StatusConflict, fmt.Sprintf("There is already a build in progress for the template")) + err = fmt.Errorf("there is already a build in progress for the template '%s'", templateID) telemetry.ReportCriticalError(ctx, err) + return } startTime := time.Now() diff --git a/packages/shared/pkg/db/envs.go b/packages/shared/pkg/db/envs.go index 3c2855728..d19861264 100644 --- a/packages/shared/pkg/db/envs.go +++ b/packages/shared/pkg/db/envs.go @@ -179,31 +179,6 @@ func (db *DB) GetRunningEnvBuilds(ctx context.Context) ([]*models.EnvBuild, erro return envBuilds, nil } -func (db *DB) GetRunningEnvBuild(ctx context.Context, envID string, excludedBuildUUIDs []uuid.UUID) (build *models.EnvBuild, err error) { - dbBuild, err := db. - Client. - EnvBuild. - Query(). - Where( - envbuild.And( - envbuild.EnvID(envID), - envbuild.StatusIn(envbuild.StatusWaiting, envbuild.StatusBuilding), - envbuild.IDNotIn(excludedBuildUUIDs...), - ), - ). - Order(models.Desc(envbuild.FieldCreatedAt)). - First(ctx) - - notFound := models.IsNotFound(err) - if notFound { - return nil, TemplateBuildNotFound{} - } else if err != nil { - return nil, fmt.Errorf("error during env build fetch '%s': %w", envID, err) - } - - return dbBuild, nil -} - func (db *DB) GetEnvBuild(ctx context.Context, buildID uuid.UUID) (build *models.EnvBuild, err error) { dbBuild, err := db. Client. From 7a728f0f46df7fe48773f486ee8c2513ecf4754e Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 12:02:18 +0100 Subject: [PATCH 37/78] increased timeout for template build status --- packages/template-manager/internal/server/template_status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/template-manager/internal/server/template_status.go b/packages/template-manager/internal/server/template_status.go index a301acb63..fc2a92cc5 100644 --- a/packages/template-manager/internal/server/template_status.go +++ b/packages/template-manager/internal/server/template_status.go @@ -9,7 +9,7 @@ import ( "time" ) -const statusTimeout = time.Minute * 5 +const statusTimeout = time.Minute * 10 func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusRequest, stream template_manager.TemplateService_TemplateBuildStatusServer) error { ctx, cancel := context.WithTimeout(stream.Context(), statusTimeout) From 9d6acf1e981f36a08f595a64f4d0934e95a97bf1 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 12:04:18 +0100 Subject: [PATCH 38/78] removed not needed and for db query --- packages/api/internal/handlers/template_start_build.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index 4c6d04a32..abd1daad3 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -93,11 +93,9 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template EnvBuild. Query(). Where( - envbuild.And( - envbuild.EnvID(envDB.ID), - envbuild.StatusIn(envbuild.StatusWaiting, envbuild.StatusBuilding), - envbuild.IDNotIn(buildUUID), - ), + envbuild.EnvID(envDB.ID), + envbuild.StatusIn(envbuild.StatusWaiting, envbuild.StatusBuilding), + envbuild.IDNotIn(buildUUID), ). Count(ctx) if err != nil { From 82446c8b7013045aca7c76fd1e23841cdb323219 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 12:04:32 +0100 Subject: [PATCH 39/78] lock mutex during buidl cache removal --- packages/template-manager/internal/cache/build_cache.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/template-manager/internal/cache/build_cache.go b/packages/template-manager/internal/cache/build_cache.go index 773add992..3e6bec779 100644 --- a/packages/template-manager/internal/cache/build_cache.go +++ b/packages/template-manager/internal/cache/build_cache.go @@ -155,6 +155,9 @@ func (c *BuildCache) updateCounter(envID string, buildID string, value int64) { } func (c *BuildCache) Delete(envID string, buildID string) { + c.mu.Lock() + defer c.mu.Unlock() + c.cache.Delete(envID) c.updateCounter(envID, buildID, -1) } From 037ff30cd5011762d28b8c4daeb2caa85c6473f7 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 12:06:38 +0100 Subject: [PATCH 40/78] fixed printing different error than received via grpc --- packages/api/internal/template-manager/create_template.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/api/internal/template-manager/create_template.go b/packages/api/internal/template-manager/create_template.go index 1dc82894c..a704986d0 100644 --- a/packages/api/internal/template-manager/create_template.go +++ b/packages/api/internal/template-manager/create_template.go @@ -55,7 +55,8 @@ func (tm *TemplateManager) CreateTemplate( }, }) - if utils.UnwrapGRPCError(err) != nil { + err = utils.UnwrapGRPCError(err) + if err != nil { return fmt.Errorf("failed to create template '%s': %w", templateID, err) } From 28b8719c5337f4a857e093560641c4e00f8b28b2 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 12:14:12 +0100 Subject: [PATCH 41/78] ctx fix, removed todo --- packages/api/internal/template-manager/template_manager.go | 1 - packages/template-manager/internal/server/create_template.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 193b832d4..71712cf71 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -69,7 +69,6 @@ func (tm *TemplateManager) BuildsStatusPeriodicalSync(ctx context.Context) { } } -// BuildStatusSync todo: implement failing build when there is build in some state for too long func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUID, templateID string) { childCtx, childCtxCancel := context.WithTimeout(ctx, syncTimeout) defer childCtxCancel() diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 7b2dd020a..2bc485c5b 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -16,7 +16,7 @@ import ( ) func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *template_manager.TemplateCreateRequest) (*emptypb.Empty, error) { - childCtx, childSpan := s.tracer.Start(context.Background(), "template-create") + childCtx, childSpan := s.tracer.Start(ctx, "template-create") defer childSpan.End() config := templateRequest.Template From 05b623f1a4c108f4f1a0c4e307694bcbe8e9c8a4 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 13:19:22 +0100 Subject: [PATCH 42/78] refactor background job for builds sync to use ticker and timeout context for db query --- .../api/internal/template-manager/template_manager.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 71712cf71..427bf2be0 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -49,15 +49,17 @@ func (tm *TemplateManager) Close() error { } func (tm *TemplateManager) BuildsStatusPeriodicalSync(ctx context.Context) { + ticker := time.NewTicker(syncInterval) for { select { case <-ctx.Done(): return - case <-time.After(syncInterval): - buildsRunning, err := tm.db.GetRunningEnvBuilds(ctx) + case <-ticker.C: + dbCtx, dbxCtxCancel := context.WithTimeout(ctx, 5*time.Second) + buildsRunning, err := tm.db.GetRunningEnvBuilds(dbCtx) if err != nil { zap.L().Error("Error getting running builds for periodical sync", zap.Error(err)) - time.Sleep(time.Second) // just not hammer database if there is an error + dbxCtxCancel() continue } @@ -65,6 +67,8 @@ func (tm *TemplateManager) BuildsStatusPeriodicalSync(ctx context.Context) { for _, buildDB := range buildsRunning { go tm.BuildStatusSync(ctx, buildDB.ID, *buildDB.EnvID) } + + dbxCtxCancel() } } } From d37eb35b60ce11b9a7a23219dd18b990465dff0a Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 14:28:05 +0100 Subject: [PATCH 43/78] create background build span for tracing --- packages/api/internal/handlers/template_start_build.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index abd1daad3..befb68afb 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -130,7 +130,7 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template // Call the Template Manager to build the environment buildErr := a.templateManager.CreateTemplate( a.Tracer, - span.SpanContext(), + ctx, templateID, buildUUID, build.KernelVersion, @@ -166,7 +166,13 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template // Do not wait for global build sync trigger it immediately go func() { - a.templateManager.BuildStatusSync(context.Background(), buildUUID, templateID) + buildContext, buildSpan := a.Tracer.Start( + trace.ContextWithSpanContext(context.Background(), span.SpanContext()), + "template-background-build-env", + ) + defer buildSpan.End() + + a.templateManager.BuildStatusSync(buildContext, buildUUID, templateID) // Invalidate the cache a.templateCache.Invalidate(templateID) From 4e9eb1300ae8059687d5fb75bab9683600cf3632 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 14:29:28 +0100 Subject: [PATCH 44/78] take ctx from parent and extend it for tracing --- packages/api/internal/template-manager/create_template.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/api/internal/template-manager/create_template.go b/packages/api/internal/template-manager/create_template.go index a704986d0..dcf2e0d87 100644 --- a/packages/api/internal/template-manager/create_template.go +++ b/packages/api/internal/template-manager/create_template.go @@ -15,7 +15,7 @@ import ( func (tm *TemplateManager) CreateTemplate( t trace.Tracer, - span trace.SpanContext, + ctx context.Context, templateID string, buildID uuid.UUID, kernelVersion, @@ -25,14 +25,12 @@ func (tm *TemplateManager) CreateTemplate( diskSizeMB, memoryMB int64, ) error { - ctx, ctxSpan := t.Start( - trace.ContextWithSpanContext(context.Background(), span), - "background-build-env", + ctx, span := t.Start(ctx, "create-template", trace.WithAttributes( attribute.String("env.id", templateID), ), ) - defer ctxSpan.End() + defer span.End() features, err := sandbox.NewVersionInfo(firecrackerVersion) if err != nil { From 531e9709b5a867ed4a73f5c5c6fef42857391065 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 14:30:09 +0100 Subject: [PATCH 45/78] return typed error for `GetEnv` method in db catalog --- packages/shared/pkg/db/envs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared/pkg/db/envs.go b/packages/shared/pkg/db/envs.go index d19861264..7023b0e2a 100644 --- a/packages/shared/pkg/db/envs.go +++ b/packages/shared/pkg/db/envs.go @@ -129,7 +129,7 @@ func (db *DB) GetEnv(ctx context.Context, aliasOrEnvID string) (result *Template notFound := models.IsNotFound(err) if notFound { - return nil, nil, fmt.Errorf("template '%s' not found: %w", aliasOrEnvID, err) + return nil, nil, TemplateNotFound{} } else if err != nil { return nil, nil, fmt.Errorf("failed to get template '%s': %w", aliasOrEnvID, err) } From e0083d3188a0946d747f8b318be860795c042160 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 15:12:07 +0100 Subject: [PATCH 46/78] template-manager: exist instantly when error during build --- packages/template-manager/internal/build/build.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/template-manager/internal/build/build.go b/packages/template-manager/internal/build/build.go index 7b2707946..78739a5a7 100644 --- a/packages/template-manager/internal/build/build.go +++ b/packages/template-manager/internal/build/build.go @@ -74,6 +74,8 @@ func (b *TemplateBuilder) Build(ctx context.Context, template *Env, envID string b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) telemetry.ReportError(ctx, buildStateErr) } + + return err } // Remove build files if build fails or times out @@ -111,6 +113,8 @@ func (b *TemplateBuilder) Build(ctx context.Context, template *Env, envID string b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) telemetry.ReportError(ctx, buildStateErr) } + + return err } uploadErr := <-upload @@ -124,6 +128,8 @@ func (b *TemplateBuilder) Build(ctx context.Context, template *Env, envID string b.logger.Error("Error while setting build state to failed", zap.Error(buildStateErr)) telemetry.ReportError(ctx, buildStateErr) } + + return uploadErr } buildMetadata := &template_manager.TemplateBuildMetadata{RootfsSizeKey: int32(template.RootfsSizeMB()), EnvdVersionKey: strings.TrimSpace(string(out))} From 8269fe0322ae6d484fdf18b2c34fd91733080b83 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 15:12:34 +0100 Subject: [PATCH 47/78] use cache for template builds to make less queries to db --- .../api/internal/cache/templates/cache.go | 68 +++++++++++++++++++ packages/api/internal/handlers/store.go | 5 +- .../internal/handlers/template_build_logs.go | 20 ++++-- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/packages/api/internal/cache/templates/cache.go b/packages/api/internal/cache/templates/cache.go index 0d435bed3..2c97a69a1 100644 --- a/packages/api/internal/cache/templates/cache.go +++ b/packages/api/internal/cache/templates/cache.go @@ -2,6 +2,7 @@ package templatecache import ( "context" + "errors" "fmt" "net/http" "time" @@ -119,3 +120,70 @@ func (c *TemplateCache) Get(ctx context.Context, aliasOrEnvID string, teamID uui func (c *TemplateCache) Invalidate(templateID string) { c.cache.Delete(templateID) } + +type TemplateBuildInfo struct { + TeamID uuid.UUID + TemplateID string +} + +type TemplateBuildInfoNotFound struct{ error } + +func (TemplateBuildInfoNotFound) Error() string { + return "Template build info not found" +} + +type TemplatesBuildCache struct { + cache *ttlcache.Cache[uuid.UUID, *TemplateBuildInfo] + db *db.DB +} + +func NewTemplateBuildCache(db *db.DB) *TemplatesBuildCache { + cache := ttlcache.New(ttlcache.WithTTL[uuid.UUID, *TemplateBuildInfo](templateInfoExpiration)) + go cache.Start() + + return &TemplatesBuildCache{ + cache: cache, + db: db, + } +} + +func (c *TemplatesBuildCache) Get(ctx context.Context, buildID uuid.UUID, templateID string) (*TemplateBuildInfo, error) { + item := c.cache.Get(buildID) + if item == nil { + print("build cache: not present in cache") + + envDB, _, envDBErr := c.db.GetEnv(ctx, templateID) + if envDBErr != nil { + if errors.Is(envDBErr, db.TemplateNotFound{}) { + return nil, TemplateBuildInfoNotFound{} + } + + return nil, fmt.Errorf("failed to get template '%s': %w", buildID, envDBErr) + } + + // making sure associated template build really exists + _, envBuildDBErr := c.db.GetEnvBuild(ctx, buildID) + if envBuildDBErr != nil { + if errors.Is(envBuildDBErr, db.TemplateBuildNotFound{}) { + return nil, TemplateBuildInfoNotFound{} + } + + return nil, fmt.Errorf("failed to get template build '%s': %w", buildID, envBuildDBErr) + } + + item = c.cache.Set( + buildID, + &TemplateBuildInfo{ + TeamID: envDB.TeamID, + TemplateID: envDB.TemplateID, + }, + templateInfoExpiration, + ) + + return item.Value(), nil + } + + print("build cache: hitting") + + return item.Value(), nil +} diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index ee494e979..623842da4 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -53,6 +53,7 @@ type APIStore struct { db *db.DB lokiClient *loki.DefaultClient templateCache *templatecache.TemplateCache + templateBuildsCache *templatecache.TemplatesBuildCache authCache *authcache.TeamAuthCache templateSpawnCounter *utils.TemplateSpawnCounter clickhouseStore chdb.Store @@ -138,8 +139,9 @@ func NewAPIStore(ctx context.Context) *APIStore { zap.L().Warn("LOKI_ADDRESS not set, disabling Loki client") } + authCache := authcache.NewTeamAuthCache(dbClient) templateCache := templatecache.NewTemplateCache(dbClient) - authCache := authcache.NewTeamAuthCache() + templateBuildsCache := templatecache.NewTemplateBuildCache(dbClient) templateSpawnCounter := utils.NewTemplateSpawnCounter(time.Minute, dbClient) a := &APIStore{ @@ -151,6 +153,7 @@ func NewAPIStore(ctx context.Context) *APIStore { posthog: posthogClient, lokiClient: lokiClient, templateCache: templateCache, + templateBuildsCache: templateBuildsCache, authCache: authCache, templateSpawnCounter: templateSpawnCounter, clickhouseStore: clickhouseStore, diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index da1808788..04672fda8 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -52,17 +52,27 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem return } - templateDB, _, err := a.db.GetEnv(ctx, templateID) + buildInfo, err := a.templateBuildsCache.Get(ctx, buildUUID, templateID) if err != nil { - errMsg := fmt.Errorf("error when getting env: %w", err) + if errors.Is(err, db.TemplateBuildNotFound{}) { + a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Build '%s' not found", buildUUID)) + return + } + + if errors.Is(err, db.TemplateNotFound{}) { + a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Template '%s' not found", templateID)) + return + } + + errMsg := fmt.Errorf("error when getting template: %w", err) telemetry.ReportError(ctx, errMsg) - a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Template '%s' not found", templateID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "Error when getting template") return } var team *models.Team for _, t := range teams { - if t.ID == templateDB.TeamID { + if t.ID == buildInfo.TeamID { team = t break } @@ -71,9 +81,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem if team == nil { msg := fmt.Errorf("user doesn't have access to env '%s'", templateID) telemetry.ReportError(ctx, msg) - a.sendAPIStoreError(c, http.StatusForbidden, fmt.Sprintf("You don't have access to this sandbox template (%s)", templateID)) - return } From 8856848ac737596499ed44ff71f5bf8f004c3f94 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Wed, 12 Mar 2025 15:33:45 +0100 Subject: [PATCH 48/78] fixed using grpc request ctx for background job, using just trace span --- .../internal/server/create_template.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 2bc485c5b..623285cfb 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/emptypb" @@ -16,7 +17,7 @@ import ( ) func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *template_manager.TemplateCreateRequest) (*emptypb.Empty, error) { - childCtx, childSpan := s.tracer.Start(ctx, "template-create") + _, childSpan := s.tracer.Start(ctx, "template-create") defer childSpan.End() config := templateRequest.Template @@ -58,7 +59,13 @@ func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *templ } go func() { - err = s.builder.Build(childCtx, template, config.TemplateID, config.BuildID) + buildContext, buildSpan := s.tracer.Start( + trace.ContextWithSpanContext(context.Background(), childSpan.SpanContext()), + "template-background-build", + ) + defer buildSpan.End() + + err = s.builder.Build(buildContext, template, config.TemplateID, config.BuildID) if err != nil { cacheErr := s.buildCache.SetFailed(config.TemplateID, config.BuildID) if cacheErr != nil { @@ -66,11 +73,11 @@ func (s *serverStore) TemplateCreate(ctx context.Context, templateRequest *templ } s.logger.Error("Error while building template", zap.Error(err)) - telemetry.ReportEvent(childCtx, "Environment built failed") + telemetry.ReportEvent(buildContext, "Environment built failed") return } - telemetry.ReportEvent(childCtx, "Environment built") + telemetry.ReportEvent(buildContext, "Environment built") }() return nil, nil From 7c90349ea339c694b14969cec170f05129c9d8ab Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 14:00:24 +0100 Subject: [PATCH 49/78] merge issue --- packages/api/internal/handlers/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 623842da4..c51fefa6f 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -139,7 +139,7 @@ func NewAPIStore(ctx context.Context) *APIStore { zap.L().Warn("LOKI_ADDRESS not set, disabling Loki client") } - authCache := authcache.NewTeamAuthCache(dbClient) + authCache := authcache.NewTeamAuthCache() templateCache := templatecache.NewTemplateCache(dbClient) templateBuildsCache := templatecache.NewTemplateBuildCache(dbClient) templateSpawnCounter := utils.NewTemplateSpawnCounter(time.Minute, dbClient) From 7da3d4489c9c4f9ee5622c4d381b2559e4453c02 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 14:03:33 +0100 Subject: [PATCH 50/78] removed temp logging --- packages/api/internal/cache/templates/cache.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/api/internal/cache/templates/cache.go b/packages/api/internal/cache/templates/cache.go index 2c97a69a1..c656052e4 100644 --- a/packages/api/internal/cache/templates/cache.go +++ b/packages/api/internal/cache/templates/cache.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "go.uber.org/zap" "net/http" "time" @@ -150,7 +151,7 @@ func NewTemplateBuildCache(db *db.DB) *TemplatesBuildCache { func (c *TemplatesBuildCache) Get(ctx context.Context, buildID uuid.UUID, templateID string) (*TemplateBuildInfo, error) { item := c.cache.Get(buildID) if item == nil { - print("build cache: not present in cache") + zap.L().Debug("Template build info not found in cache, fetching from DB", zap.String("buildID", buildID.String())) envDB, _, envDBErr := c.db.GetEnv(ctx, templateID) if envDBErr != nil { @@ -183,7 +184,5 @@ func (c *TemplatesBuildCache) Get(ctx context.Context, buildID uuid.UUID, templa return item.Value(), nil } - print("build cache: hitting") - return item.Value(), nil } From d0244393455a506b33b2ebfc732da66f6c14429f Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 14:03:47 +0100 Subject: [PATCH 51/78] speed up interval for checking template status --- packages/template-manager/internal/server/template_status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/template-manager/internal/server/template_status.go b/packages/template-manager/internal/server/template_status.go index fc2a92cc5..3308a5db5 100644 --- a/packages/template-manager/internal/server/template_status.go +++ b/packages/template-manager/internal/server/template_status.go @@ -50,7 +50,7 @@ func (s *serverStore) TemplateBuildStatus(in *template_manager.TemplateStatusReq return nil } - time.Sleep(time.Second * 5) + time.Sleep(time.Second) } return nil From 29f4c8a4fade00b3346d76088d3e9f5d32bd04e0 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 14:04:28 +0100 Subject: [PATCH 52/78] log template cache hit --- packages/api/internal/cache/templates/cache.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/api/internal/cache/templates/cache.go b/packages/api/internal/cache/templates/cache.go index c656052e4..b2b52b809 100644 --- a/packages/api/internal/cache/templates/cache.go +++ b/packages/api/internal/cache/templates/cache.go @@ -184,5 +184,7 @@ func (c *TemplatesBuildCache) Get(ctx context.Context, buildID uuid.UUID, templa return item.Value(), nil } + zap.L().Debug("Template build info found in cache", zap.String("buildID", buildID.String())) + return item.Value(), nil } From 747fcf85edce5841851ca05fb79df5b79428a385 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 14:26:36 +0100 Subject: [PATCH 53/78] take build status from cache instead of db query --- .../api/internal/cache/templates/cache.go | 33 +++++++++++++++---- packages/api/internal/handlers/store.go | 4 +-- .../internal/handlers/template_build_logs.go | 16 +-------- .../internal/handlers/template_start_build.go | 2 ++ .../template-manager/template_manager.go | 17 +++++++--- packages/shared/pkg/db/envs.go | 27 ++++++++++++++- 6 files changed, 69 insertions(+), 30 deletions(-) diff --git a/packages/api/internal/cache/templates/cache.go b/packages/api/internal/cache/templates/cache.go index b2b52b809..2a3f35cd5 100644 --- a/packages/api/internal/cache/templates/cache.go +++ b/packages/api/internal/cache/templates/cache.go @@ -6,11 +6,13 @@ import ( "fmt" "go.uber.org/zap" "net/http" + "sync" "time" "github.com/google/uuid" "github.com/jellydator/ttlcache/v3" + "github.com/e2b-dev/infra/packages/shared/pkg/models/envbuild" "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/shared/pkg/db" "github.com/e2b-dev/infra/packages/shared/pkg/models" @@ -78,7 +80,7 @@ func (c *TemplateCache) Get(ctx context.Context, aliasOrEnvID string, teamID uui } if item == nil { - envDB, build, err = c.db.GetEnv(ctx, aliasOrEnvID) + envDB, build, err = c.db.GetEnvWithBuild(ctx, aliasOrEnvID) if err != nil { if models.IsNotFound(err) == true { return nil, nil, &api.APIError{Code: http.StatusNotFound, ClientMsg: fmt.Sprintf("template '%s' not found", aliasOrEnvID), Err: err} @@ -123,8 +125,9 @@ func (c *TemplateCache) Invalidate(templateID string) { } type TemplateBuildInfo struct { - TeamID uuid.UUID - TemplateID string + TeamID uuid.UUID + TemplateID string + BuildStatus envbuild.Status } type TemplateBuildInfoNotFound struct{ error } @@ -136,6 +139,7 @@ func (TemplateBuildInfoNotFound) Error() string { type TemplatesBuildCache struct { cache *ttlcache.Cache[uuid.UUID, *TemplateBuildInfo] db *db.DB + mx sync.Mutex } func NewTemplateBuildCache(db *db.DB) *TemplatesBuildCache { @@ -148,12 +152,26 @@ func NewTemplateBuildCache(db *db.DB) *TemplatesBuildCache { } } +func (c *TemplatesBuildCache) SetStatus(buildID uuid.UUID, status envbuild.Status) { + c.mx.Lock() + defer c.mx.Unlock() + + item := c.cache.Get(buildID) + if item == nil { + return + } + + zap.L().Debug("Setting template build status", zap.String("buildID", buildID.String()), zap.String("status", status.String())) + + item.Value().BuildStatus = status +} + func (c *TemplatesBuildCache) Get(ctx context.Context, buildID uuid.UUID, templateID string) (*TemplateBuildInfo, error) { item := c.cache.Get(buildID) if item == nil { zap.L().Debug("Template build info not found in cache, fetching from DB", zap.String("buildID", buildID.String())) - envDB, _, envDBErr := c.db.GetEnv(ctx, templateID) + envDB, envDBErr := c.db.GetEnv(ctx, templateID) if envDBErr != nil { if errors.Is(envDBErr, db.TemplateNotFound{}) { return nil, TemplateBuildInfoNotFound{} @@ -163,7 +181,7 @@ func (c *TemplatesBuildCache) Get(ctx context.Context, buildID uuid.UUID, templa } // making sure associated template build really exists - _, envBuildDBErr := c.db.GetEnvBuild(ctx, buildID) + envBuildDB, envBuildDBErr := c.db.GetEnvBuild(ctx, buildID) if envBuildDBErr != nil { if errors.Is(envBuildDBErr, db.TemplateBuildNotFound{}) { return nil, TemplateBuildInfoNotFound{} @@ -175,8 +193,9 @@ func (c *TemplatesBuildCache) Get(ctx context.Context, buildID uuid.UUID, templa item = c.cache.Set( buildID, &TemplateBuildInfo{ - TeamID: envDB.TeamID, - TemplateID: envDB.TemplateID, + TeamID: envDB.TeamID, + TemplateID: envDB.ID, + BuildStatus: envBuildDB.Status, }, templateInfoExpiration, ) diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index c51fefa6f..64446c90b 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -122,7 +122,8 @@ func NewAPIStore(ctx context.Context) *APIStore { zap.L().Fatal("initializing Orchestrator client", zap.Error(err)) } - templateManager, err := template_manager.New(dbClient) + templateBuildsCache := templatecache.NewTemplateBuildCache(dbClient) + templateManager, err := template_manager.New(dbClient, templateBuildsCache) if err != nil { zap.L().Fatal("initializing Template manager client", zap.Error(err)) } @@ -141,7 +142,6 @@ func NewAPIStore(ctx context.Context) *APIStore { authCache := authcache.NewTeamAuthCache() templateCache := templatecache.NewTemplateCache(dbClient) - templateBuildsCache := templatecache.NewTemplateBuildCache(dbClient) templateSpawnCounter := utils.NewTemplateSpawnCounter(time.Minute, dbClient) a := &APIStore{ diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 04672fda8..45140be47 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -85,20 +85,6 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem return } - buildDB, err := a.db.GetEnvBuild(ctx, buildUUID) - if err != nil { - errMsg := fmt.Errorf("error when getting build: %w", err) - telemetry.ReportError(ctx, errMsg) - - if errors.Is(err, db.TemplateBuildNotFound{}) { - a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Build '%s' not found", buildID)) - return - } - - a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error during build '%s'", buildID)) - return - } - // Sanitize env ID // https://grafana.com/blog/2021/01/05/how-to-escape-special-characters-with-lokis-logql/ templateIdSanitized := strings.ReplaceAll(templateID, "`", "") @@ -154,7 +140,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem Logs: logs, TemplateID: templateID, BuildID: buildID, - Status: getCorrespondingTemplateBuildStatus(buildDB.Status), + Status: getCorrespondingTemplateBuildStatus(buildInfo.BuildStatus), } c.JSON(http.StatusOK, result) diff --git a/packages/api/internal/handlers/template_start_build.go b/packages/api/internal/handlers/template_start_build.go index befb68afb..08d5db9e5 100644 --- a/packages/api/internal/handlers/template_start_build.go +++ b/packages/api/internal/handlers/template_start_build.go @@ -146,6 +146,7 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template zap.L().Error("build failed", zap.Error(buildErr)) telemetry.ReportCriticalError(ctx, buildErr) + a.templateBuildsCache.SetStatus(buildUUID, envbuild.StatusFailed) dbErr := a.db.EnvBuildSetStatus(ctx, templateID, buildUUID, envbuild.StatusFailed) if dbErr != nil { telemetry.ReportCriticalError(ctx, fmt.Errorf("error when setting build status: %w", dbErr)) @@ -156,6 +157,7 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template // status building must be set after build is triggered because then // it's possible build status job will be triggered before build cache on template manager is created and build will fail + a.templateBuildsCache.SetStatus(buildUUID, envbuild.StatusBuilding) err = a.db.EnvBuildSetStatus(ctx, templateID, buildUUID, envbuild.StatusBuilding) if err != nil { err = fmt.Errorf("error when setting build status: %w", err) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 427bf2be0..74f04746c 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -2,15 +2,17 @@ package template_manager import ( "context" - "github.com/e2b-dev/infra/packages/api/internal/utils" - "github.com/e2b-dev/infra/packages/shared/pkg/db" - "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" - "github.com/e2b-dev/infra/packages/shared/pkg/models/envbuild" "github.com/google/uuid" "go.uber.org/zap" "io" "sync" "time" + + templatecache "github.com/e2b-dev/infra/packages/api/internal/cache/templates" + "github.com/e2b-dev/infra/packages/api/internal/utils" + "github.com/e2b-dev/infra/packages/shared/pkg/db" + "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" + "github.com/e2b-dev/infra/packages/shared/pkg/models/envbuild" ) type processingBuilds struct { @@ -22,6 +24,7 @@ type TemplateManager struct { db *db.DB lock sync.Mutex processing map[uuid.UUID]processingBuilds + buildCache *templatecache.TemplatesBuildCache } var ( @@ -30,7 +33,7 @@ var ( syncWaitingStateDeadline = time.Minute * 20 ) -func New(db *db.DB) (*TemplateManager, error) { +func New(db *db.DB, buildCache *templatecache.TemplatesBuildCache) (*TemplateManager, error) { client, err := NewClient() if err != nil { return nil, err @@ -41,6 +44,7 @@ func New(db *db.DB) (*TemplateManager, error) { db: db, lock: sync.Mutex{}, processing: make(map[uuid.UUID]processingBuilds), + buildCache: buildCache, }, nil } @@ -113,6 +117,7 @@ func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUI if utils.UnwrapGRPCError(err) != nil { logger.Error("Error when fetching template build status", zap.Error(err)) + tm.buildCache.SetStatus(buildID, envbuild.StatusFailed) dbErr := tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) if dbErr != nil { logger.Error("Error when setting build status", zap.Error(dbErr)) @@ -128,6 +133,7 @@ func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUI } else if receiveErr != nil { logger.Error("Error when receiving template build status", zap.Error(receiveErr)) + tm.buildCache.SetStatus(buildID, envbuild.StatusFailed) err = tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) if err != nil { logger.Error("Error when setting build status", zap.Error(err)) @@ -138,6 +144,7 @@ func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUI // build failed if status.GetStatus() == template_manager.TemplateBuildState_Failed { + tm.buildCache.SetStatus(buildID, envbuild.StatusFailed) err = tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) if err != nil { logger.Error("Error when setting build status", zap.Error(err)) diff --git a/packages/shared/pkg/db/envs.go b/packages/shared/pkg/db/envs.go index 7023b0e2a..1827c3e63 100644 --- a/packages/shared/pkg/db/envs.go +++ b/packages/shared/pkg/db/envs.go @@ -112,7 +112,32 @@ func (db *DB) GetEnvs(ctx context.Context, teamID uuid.UUID) (result []*Template return result, nil } -func (db *DB) GetEnv(ctx context.Context, aliasOrEnvID string) (result *Template, build *models.EnvBuild, err error) { +func (db *DB) GetEnv(ctx context.Context, aliasOrEnvID string) (result *models.Env, err error) { + template, err := db. + Client. + Env. + Query(). + Where( + env.Or( + env.HasEnvAliasesWith(envalias.ID(aliasOrEnvID)), + env.ID(aliasOrEnvID), + ), + ). + WithEnvAliases(func(query *models.EnvAliasQuery) { + query.Order(models.Asc(envalias.FieldID)) // TODO: remove once we have only 1 alias per env + }).Only(ctx) + + notFound := models.IsNotFound(err) + if notFound { + return nil, TemplateNotFound{} + } else if err != nil { + return nil, fmt.Errorf("failed to get template '%s': %w", aliasOrEnvID, err) + } + + return template, nil +} + +func (db *DB) GetEnvWithBuild(ctx context.Context, aliasOrEnvID string) (result *Template, build *models.EnvBuild, err error) { template, err := db. Client. Env. From b73d62ceb02dd206ed6805d87d9f53253b6250b9 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 14:29:57 +0100 Subject: [PATCH 54/78] go deps sync --- packages/api/go.mod | 24 ------------- packages/api/go.sum | 58 -------------------------------- packages/template-manager/go.mod | 3 +- packages/template-manager/go.sum | 2 ++ 4 files changed, 4 insertions(+), 83 deletions(-) diff --git a/packages/api/go.mod b/packages/api/go.mod index 99662cb93..45c43a01e 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -40,14 +40,6 @@ require ( require ( ariga.io/atlas v0.15.0 // indirect - cel.dev/expr v0.19.1 // indirect - cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/auth v0.13.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect - cloud.google.com/go/iam v1.2.2 // indirect - cloud.google.com/go/monitoring v1.21.2 // indirect - cloud.google.com/go/storage v1.50.0 // indirect entgo.io/ent v0.12.5 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect @@ -55,9 +47,6 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/ClickHouse/ch-go v0.65.1 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.32.2 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect @@ -71,7 +60,6 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.49.6 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.17.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect @@ -81,7 +69,6 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -89,8 +76,6 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect - github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -111,12 +96,8 @@ require ( github.com/gogo/googleapis v1.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-migrate/migrate/v4 v4.18.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/s2a-go v0.1.8 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.14.0 // indirect github.com/grafana/dskit v0.0.0-20231120170505-765e343eda4f // indirect github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586 // indirect github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect @@ -161,7 +142,6 @@ require ( github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/alertmanager v0.26.0 // indirect github.com/prometheus/client_golang v1.20.2 // indirect @@ -194,12 +174,10 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect go.etcd.io/etcd/client/v3 v3.5.4 // indirect go.mongodb.org/mongo-driver v1.17.1 // indirect - go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/collector/pdata v1.0.0-rcv0015 // indirect go.opentelemetry.io/collector/semconv v0.81.0 // indirect go.opentelemetry.io/contrib/bridges/otelzap v0.9.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.10.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.34.0 // indirect @@ -220,8 +198,6 @@ require ( golang.org/x/oauth2 v0.25.0 // indirect golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.30.0 // indirect - google.golang.org/api v0.214.0 // indirect - google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index ceb2be569..5339e71e6 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -1,7 +1,5 @@ ariga.io/atlas v0.15.0 h1:9lwSVcO/D3WgaCzstSGqR1hEDtsGibu6JqUofEI/0sY= ariga.io/atlas v0.15.0/go.mod h1:isZrlzJ5cpoCoKFoY9knZug7Lq4pP1cm8g3XciLZ0Pw= -cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= -cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -17,30 +15,14 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= -cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= -cloud.google.com/go/auth v0.13.0 h1:8Fu8TZy167JkW8Tj3q7dIkr2v4cndv41ouecJx0PAHs= -cloud.google.com/go/auth v0.13.0/go.mod h1:COOjD9gwfKNKz+IIduatIhYJQIc0mG3H102r/EMxX6Q= -cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= -cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA= -cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY= -cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk= -cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM= -cloud.google.com/go/longrunning v0.6.3 h1:A2q2vuyXysRcwzqDpMMLSI6mb6o39miS52UEG/Rd2ng= -cloud.google.com/go/longrunning v0.6.3/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= -cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU= -cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -50,10 +32,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= -cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= -cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= -cloud.google.com/go/trace v1.11.2/go.mod h1:bn7OwXd4pd5rFuAnTrzBuoZ4ax2XQeG3qNgYmfCy0Io= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= entgo.io/ent v0.12.5 h1:KREM5E4CSoej4zeGa88Ou/gfturAnpUv0mzAjch1sj4= entgo.io/ent v0.12.5/go.mod h1:Y3JVAjtlIk8xVZYSn3t3mf8xlZIn5SAOXZQxD6kKI+Q= @@ -96,14 +74,6 @@ github.com/ClickHouse/clickhouse-go/v2 v2.32.2/go.mod h1:/vE8N/+9pozLkIiTMWbNUGv github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 h1:o90wcURuxekmXrtxmYWTyNla0+ZEHhud6DI1ZTxd1vI= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0/go.mod h1:6fTWu4m3jocfUZLYF5KsZC1TUfRvEjs7lM4crme/irw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0 h1:jJKWl98inONJAr/IZrdFQUWcwUO95DLY1XMD1ZIut+g= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0/go.mod h1:l2fIqmwB+FKSfvn3bAD/0i+AXAxhIZjTK2svT/mgUXs= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 h1:GYUJLfvd++4DMuMhCFLgLXvFwofIxh/qOwoGuS/LTew= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0/go.mod h1:wRbFgBQUVm1YXrvWKofAEmq9HNJTDphbAaJSSX01KUI= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -160,8 +130,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.17.0 h1:1X2TS7aHz1ELcC0yU1y2stUs/0ig5oMU6STFZGrhvHI= -github.com/bits-and-blooms/bitset v1.17.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -238,11 +206,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= -github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= @@ -369,8 +334,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -413,7 +376,6 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -425,11 +387,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= -github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -438,19 +397,13 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= -github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= -github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= github.com/gophercloud/gophercloud v1.5.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -885,8 +838,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/collector/pdata v1.0.0-rcv0015 h1:8PzrQFk3oKiT1Sd5EmNEcagdMyt1KcBy5/OyF5He5gY= @@ -895,8 +846,6 @@ go.opentelemetry.io/collector/semconv v0.81.0 h1:lCYNNo3powDvFIaTPP2jDKIrBiV1T92 go.opentelemetry.io/collector/semconv v0.81.0/go.mod h1:TlYPtzvsXyHOgr5eATi43qEMqwSmIziivJB2uctKswo= go.opentelemetry.io/contrib/bridges/otelzap v0.9.0 h1:f+xpAfhQTjR8beiSMe1bnT/25PkeyWmOcI+SjXWguNw= go.opentelemetry.io/contrib/bridges/otelzap v0.9.0/go.mod h1:T1Z1jyS5FttgQoF6UcGhnM+gF9wU32B4lHO69nXw4FE= -go.opentelemetry.io/contrib/detectors/gcp v1.34.0 h1:JRxssobiPg23otYU5SbWtQC//snGVIM3Tx6QRzlQBao= -go.opentelemetry.io/contrib/detectors/gcp v1.34.0/go.mod h1:cV4BMFcscUR/ckqLkbfQmF0PRsq8w/lMGzdbCSveBHo= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.57.0 h1:1wEousrQOXTAhk16quIMIo1gSaUp1J3PEVlsiEAtmeU= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.57.0/go.mod h1:rUWyQu4HfRAG0jkr1TixDHP9IERQ/iEq/YwFoU73ddo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 h1:qtFISDHKolvIxzSs0gIaiPUPR0Cucb0F2coHC7ZLdps= @@ -915,8 +864,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glB go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= go.opentelemetry.io/otel/log v0.10.0 h1:1CXmspaRITvFcjA4kyVszuG4HjA61fPDxMb7q3BuyF0= go.opentelemetry.io/otel/log v0.10.0/go.mod h1:PbVdm9bXKku/gL0oFfUF4wwsQsOPlpo4VEqjvxih+FM= go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= @@ -1234,8 +1181,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.214.0 h1:h2Gkq07OYi6kusGOaT/9rnNljuXmqPnaig7WGPmKbwA= -google.golang.org/api v0.214.0/go.mod h1:bYPpLG8AyeMWwDU6NXoB00xC0DFkikVvd5MfwoxjLqE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1275,8 +1220,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= -google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA= google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= @@ -1296,7 +1239,6 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= diff --git a/packages/template-manager/go.mod b/packages/template-manager/go.mod index 9f240cba6..4f684a7fa 100644 --- a/packages/template-manager/go.mod +++ b/packages/template-manager/go.mod @@ -14,12 +14,14 @@ require ( github.com/go-openapi/strfmt v0.23.0 github.com/gogo/status v1.1.1 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 + github.com/jellydator/ttlcache/v3 v3.3.1-0.20250207140243-aefc35918359 github.com/opencontainers/image-spec v1.1.0 github.com/rs/zerolog v1.33.0 github.com/vishvananda/netlink v1.3.0 github.com/vishvananda/netns v0.0.5 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.57.0 go.opentelemetry.io/otel v1.34.0 + go.opentelemetry.io/otel/metric v1.34.0 go.opentelemetry.io/otel/trace v1.34.0 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.71.0 @@ -113,7 +115,6 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect go.opentelemetry.io/otel/log v0.10.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect go.opentelemetry.io/otel/sdk v1.34.0 // indirect go.opentelemetry.io/otel/sdk/log v0.10.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect diff --git a/packages/template-manager/go.sum b/packages/template-manager/go.sum index 0ec109606..3d680d8ef 100644 --- a/packages/template-manager/go.sum +++ b/packages/template-manager/go.sum @@ -557,6 +557,8 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= +github.com/jellydator/ttlcache/v3 v3.3.1-0.20250207140243-aefc35918359 h1:uzTOUCYbGERlXB3wX2/u9AsMeXnZCd8yLl2DMAY1Wxs= +github.com/jellydator/ttlcache/v3 v3.3.1-0.20250207140243-aefc35918359/go.mod h1:aqa3CYl8S7MwpMXtFH3uNIEEfOjcn1MUNO+bQIGbFAQ= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= From 37c39be5db7af29ba3d3567fc176dbbef7d3110b Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 14:47:07 +0100 Subject: [PATCH 55/78] fixed missing cache status update --- packages/api/internal/template-manager/template_manager.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 74f04746c..7f424a1f6 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -156,6 +156,8 @@ func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUI // build completed if status.GetStatus() == template_manager.TemplateBuildState_Completed { + tm.buildCache.SetStatus(buildID, envbuild.StatusUploaded) + meta := status.GetMetadata() err = tm.db.FinishEnvBuild(childCtx, templateID, buildID, int64(meta.RootfsSizeKey), meta.EnvdVersionKey) if err != nil { From 5f8cc3291577e2de07e9996b88829ec1f4becea0 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 16:00:26 +0100 Subject: [PATCH 56/78] return proper state when build is still in waiting phase --- packages/api/internal/api/spec.gen.go | 59 ++++++++++--------- packages/api/internal/api/types.gen.go | 1 + .../internal/handlers/template_build_logs.go | 2 +- spec/openapi.yml | 1 + 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/packages/api/internal/api/spec.gen.go b/packages/api/internal/api/spec.gen.go index 051e8783b..aff5e686a 100644 --- a/packages/api/internal/api/spec.gen.go +++ b/packages/api/internal/api/spec.gen.go @@ -46,35 +46,36 @@ var swaggerSpec = []string{ "Vvnppsxqj92HzSm2J8WNtdeZyLCs2I4/X9kzgV8X0emHzUxWkF7PRhEtswzPMzAn2+tRpMR0WeAbujPr", "WsDK0x51/1SU8ywUOJseybJFBDLjEeOI0WyFsNY/mWeA5quAt/BclVBS2BfDbTlsCDV7JbMhcZZFsgfi", "jNrMp3uGLz8rrhtHwvsgqz/fPnzOfUS3wdhQScPH+J5OF4O77m4HT6GHhgRcZ6k2LH6YdVoxtFfRA3fx", - "l2JQkdpTvitUa15NTdrVrE3LwOxg+6Z99V8V7KsEu6Gid7Zh5fDb4D2cdcLij8AXJAskJy+qd17G1D/9", - "Pk5Nlw+e50kQAFyimOW5yv4lQ/AJ4lK5tpYp15X5XvgeOIPyZOYr97225V7t3pf/1scAAuKSE7m6VDI3", - "859pAlfsI9CzUqbaNQDmwH9yjs9M8V+phkS270aT1sPqqVIpCyXWsyQntEFQt5OlgBM93DaU/eeJHvjk", - "ytJ1LsDknYqO/msbjYtXT0ye2vn+sizwHAt4OoQXN7ifHTfiROULg6k1lOGIKVUQumDaFRKpjCx6eXKO", - "zi5eRaPo2hWboun46Xiq5mYFUFyQ6DT6ZjwdT1XYwDLV+puYgzf15xICke5f+jWKU4g/RpoS131Zr5Lo", - "NPoZpHkftdr0TkxLV5OUxbCpVlaJo9dhFzLviuxEDTIwnFCWmHmCLOsjWZxlyAwLMP3WvgjxPLgNrYpG", - "w9JE3XaynnWrJ91WtUo22QpxkCWnkHgL2klgVXvd5rFqkG/hejltS/wwUymuxCpqf4iwehvNaoVMbs3h", - "8rpXMz+D1GtAGr19innrjqj9Btse6dZDJvZkW7F4J71uU6LtmxisuOo4fEe92c7PbWOf3YeOR1HBRKh0", - "pc/jkajSKuxO/puqvWDicLrVXuScJauDqrXRYLDudh6fGHW09gFWt04CesupSSSei8tWj1n3yr4bDTCb", - "na47vvDbSjp27ndWtZDQylLRXyW4iqNkaEEyl5fVHTf/gPFyjP6ISgH8RzyP/yin05PvcFH8WHCW/BH9", - "c4z+ramonA9wnOpynfrPNc5KECgvhURzQO/fvUZAY5ZAMlb7DcWBnr8Ozu6//S3rs/uNK+2eobtFmK72", - "NBqnQ9A4vcfI5OV2H2br0W0wW1PbyUDa1YR5vdINbs40neHqgEsffbV2Ll2P56P8KG6r7lleN3cTtvjU", - "wuHhrjE0pu26RP9c2tYfAu7wCwFVw39OvKaDHf2oOb5y329yqm+qMV996wF9q9/Wc2g321TuYzCPQWi/", - "rQ711wbqGchASegXkmW1d+1g+4X+rIL3pdcosFsaWbcYBJDUk975vuwjybLHkdkdKz72buvq2DhfIX1o", - "1u+fjqTAw23z2jnVLls9UTdvfzG46DX6iSvx9+LGocaW+AeA5rUZuTdwRsGasPLGMtC3I5BMsUQiZWWW", - "qDhWKZtQlJMsI7avuiem6VJ0I6Z1zrA2X2XqtGCYe22IVkdmm7js4SojOWlyVTeWT6fTXTvEj2mLfp/T", - "PoZokPXVGpU1bks+fYMckmhWNtmbcd6fPz9EC+8+8GrkbF88wgp3MSa8kdb3Zlo9fRv2zRW+zH2b+073", - "9GKa6Z7e2cSYGh+rLwodU/P2Lv62sT88MpRwWHAQqT3BDCLlnRnSMDX4JIEmujdaCh2t3X2ngTB6V817", - "VyjtV8xpHt0mpWE4cERu3+gDctNh7cuhDvMfoZAI6xtf9Q0v//L7N9+paL7lsrR9xOZ/QiwH175brtFI", - "9p5y4AeAYGX7m+Cr3u/h6cyHnwmfG/dEzWuRD7fgaN3yve3S/6Y+2rt0Gob4JUj/5mr7yukYXYWvg6FP", - "zlF5ZXRSNyxb8I7Rc5xleluWEqHSrJQlKC8zSYoMbJ8guwZ+w4m0LYNXV69HpgipCZbCfA4oLjkHKv3m", - "f3tjxe39CkbUe4ZywKLk0Fia89TjgUZ8VV3m/fxRpnF5uN3DqBZXB45aH768bCtUbxjqXvLb5zc6LJez", - "g0QjYaHpOHXUv7QcXQLOBxw1mGGBjd6VfXGfNXfd8nTH8rpZ0P1Vxts9cJvU2DhiV8+cqsxB4iB1uaFB", - "ldUvW84nVAiqrm34laC9mhln9w0Te/B6Z6g4eT18uNS8DjiqpnCz+XTaR8oxEsZgd/KgtPHk4Dz05Y3m", - "0orKGnEcQyF3381/Zhg0XMfktm4d33jeZg7UEO4HiBlRQeTKb0nfLafxutmHV2EaNyrMKu6W0H9+a8Uy", - "TruLNd3dGwxVfXYUNRzP4Jsd64MsfjoABvZSy2NoJDmmg38HxmlhOtC9Pw7QfI0SnyVKTMwPuE1u7T2j", - "9Yb9vr4649+IGQQ682Ni59U1pv0RONo62l2WCgSak7CHMapNvd9P+dtqdlJfius9/KscrpFLX5v+NjVf", - "uqtq96LszhH5K5rAp+rHI1yFZ+6uEvae6Jvfh2jd0Q6dnrOl+HWxENBzhP6gzs+b9zh3OuKsxPAw6yYH", - "sR9NlV87hJY8s1fRxOlkggsyhpP5OIHryKNw2/5Rc6FB2PwJ9eZDvdNfz9b/CwAA//8Gg3j/RF4AAA==", + "l2JQkdpTvitUa15NTfoGE1uddtVr0zwwO9gOal8kVKX7KtVuKOudbV05/IZ4D7edsPgj8AXJAmnKi+qd", + "lzv1T7+Pe9OFhOd5EoQClyhmea72AZIh+ARxqZxcy6jrGn0vkA+cS3ky85X7Xlt1r3bvy5PrAwEBccmJ", + "XF0qmZv5zzSBK/YR6FkpU+0kAHPgPzkXaKb4r1RDItuBo0nrYfVUqZSFEutZkhPaIKgby1LAiR5uW8v+", + "80QPfHJl6TpnYDJQRUf/tY3GxasnJmPtfH9ZFniOBTwdwosb3M+OG3GiMofB1BrKcMSUKghdMO0UiVRG", + "Fr08OUdnF6+iUXTtyk7RdPx0PFVzswIoLkh0Gn0zno6nKoBgmWr9TcwRnPpzCYGY9y/9GsUpxB8jTYnr", + "Dq1XSXQa/QzSvI9aDXsnprmrScpi2NQtqxTS67ULmXdFdqIGGRhOKEvMPEGW9eEszjJkhgWYfmtfhHge", + "3JBWxaVhCaNuQFnPunWUbtNaJZtshTjIklNIvAXtJLCq0W7zWDXIt3C9nLYlfpipZFdiFb8/RFi9jWa1", + "Qia35ph53auZn0HqNSCN3j7FvHWH1X6rbY906yETe8atWLyTXrcp0XZQDFZcdTC+o95sD+i2sc/uQ8ej", + "qGAiVMTSJ/NIVAkWdj0ATdVeMHE43Wovcs6S1UHV2mg1WHd7kE+MOlo7AqtbJwG9+dQkEs/FZavHrHtl", + "341WmM1O1x1k+A0mHTv3e6xaSGhlqeivElztUTK0IJnLy+rem3/AeDlGf0SlAP4jnsd/lNPpyXe4KH4s", + "OEv+iP45Rv/WVFTOBzhOdeFO/ecaZyUIlJdCojmg9+9eI6AxSyAZq52H4kDPXwdn99/+5vXZ/caVdvfQ", + "3SJMV3sajdMhaJzeY2TycrsPs/XoNpitqY1lIO1qwrxe6QY3Z9rPcHXUpQ/BWjuXrsfzUX4Ut1V3L6+b", + "uwlbhmrh8HAXGhrTdl2if0JtKxEBd/iFgKrhPyde+8GOftQcZLnvNznVN9WYr771gL7Vb/A5tJttKvcx", + "mMcgtN9Wx/trA/UMZKAk9AvJstq7drD9Qn9WwfvSaxnYLY2smw0CSOpJ73xf9pFk2ePI7I4VH3u3dXVs", + "nK+QPj7r909HUuDhtnntnGqXrZ6o27i/GFz0Gv3EFft7ceNQY4v9A0Dz2ozcGzijYE1YeWMZ6OARSKZY", + "IpGyMktUHKuUTSjKSZYR22HdE9N0KboR0zqnWZsvNXWaMcwNN0Srw7NNXPZwlZGcNLmqW8yn0+muveLH", + "tEW/42kfQzTI+mqNyhq3JZ++QQ5JNCub7M0478+fH6KZdx94NXK2Lx5hhbsiE95I6xs0re6+DfvmCl/m", + "5s19p3t6Mc10T+9sYkyNj9VXho6peXsrf9vYHx4ZSjgsOIjUnmAGkfLODGmYGnySQBPdJS2Fjtbu5tNA", + "GL2r5r0rlPYr5jSPbpPSMBw4Irdv9AG56bX25VCH+Y9QSIT13a/6rpd/Df6b71Q033Jt2j5i8z8hloNr", + "3y3XaCR7TznwA0Cwsv1N8FXv9/B05sPPhM+Ne6LmBcmHW3C0bvnedul/Ux/tXT8NQ/wSpH+HtX35dIyu", + "whfD0CfnqLwyOqlbly14x+g5zjK9LUuJUGlWyhKUl5kkRQa2Y5BdA7/hRNrmwaur1yNThNQES2E+BxSX", + "nAOV/jUAe3fF7f0KRtR7hnLAouTQWJrz1OOBRnxVXev9/FGmcY243c2oFlcHjlofvrxsK1RvGOpe99vn", + "1zosl7ODRCNhoek4ddS/tBxdAs4HHDWYYYGN3pV9cZ81d93ydMfyulnQ/VXG2z1wm9TYOGJXz5yqzEHi", + "IHW5oUGV1S9bzidUCKoucPiVoL2aGWf3DRN78HpnqDh5PXy41LwOOKqmcLP5dNpHyjESxmB38qC08eTg", + "PPTljeb6isoacRxDIXffzX9mGDRcx+S2bh3feN5mDtQQ7geIGVFB5MpvSd8tp/G62YdXYRp3K8wq7pbQ", + "f35rxTJOu4s13d0bDFV9dhQ1HM/gmx3rgyx+OgAG9nrLY2gkOaaDfwfGaWE60L0/DtB8jRKfJUpMzE+5", + "TW7tPaP1hv2+vjrj34gZBDrzs2Ln1TWm/RE42jraXZYKBJqTsIcxqk29X1L522p2Ul+P6z38qxyukUtf", + "m/42NV+6q2r3ouzOEfkrmsCn6mckXIVn7i4V9p7om1+KaN3WDp2es6X4dbEQ0HOE/qDOz5s3Onc64qzE", + "8DDrJgexH02VXzuEljyzV9HE6WSCCzKGk/k4gevIo3Db/nlzoUHY/DH15kO901/P1v8LAAD//xCfnelO", + "XgAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/packages/api/internal/api/types.gen.go b/packages/api/internal/api/types.gen.go index ba6d27480..d23512d23 100644 --- a/packages/api/internal/api/types.gen.go +++ b/packages/api/internal/api/types.gen.go @@ -30,6 +30,7 @@ const ( TemplateBuildStatusBuilding TemplateBuildStatus = "building" TemplateBuildStatusError TemplateBuildStatus = "error" TemplateBuildStatusReady TemplateBuildStatus = "ready" + TemplateBuildStatusWaiting TemplateBuildStatus = "waiting" ) // CPUCount CPU cores for the sandbox diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 45140be47..28ba3b45e 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -149,7 +149,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem func getCorrespondingTemplateBuildStatus(s envbuild.Status) api.TemplateBuildStatus { switch s { case envbuild.StatusWaiting: - return api.TemplateBuildStatusBuilding + return api.TemplateBuildStatusWaiting case envbuild.StatusFailed: return api.TemplateBuildStatusError case envbuild.StatusUploaded: diff --git a/spec/openapi.yml b/spec/openapi.yml index 97b56bb74..2b57ddfce 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -446,6 +446,7 @@ components: description: Status of the template enum: - building + - waiting - ready - error From 25e3b138288674406a397f206920c3aff6a52ccf Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 16:00:37 +0100 Subject: [PATCH 57/78] missing build cache status set --- packages/api/internal/template-manager/template_manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/internal/template-manager/template_manager.go b/packages/api/internal/template-manager/template_manager.go index 7f424a1f6..573884f84 100644 --- a/packages/api/internal/template-manager/template_manager.go +++ b/packages/api/internal/template-manager/template_manager.go @@ -104,6 +104,7 @@ func (tm *TemplateManager) BuildStatusSync(ctx context.Context, buildID uuid.UUI if time.Since(envBuildDb.CreatedAt) > syncWaitingStateDeadline { logger.Error("Build is in waiting state for too long, failing it") + tm.buildCache.SetStatus(buildID, envbuild.StatusFailed) dbErr := tm.db.EnvBuildSetStatus(childCtx, templateID, buildID, envbuild.StatusFailed) if dbErr != nil { logger.Error("Error when setting build status", zap.Error(dbErr)) From 2afda5bfe6eef5ce0b6e49ce995c41b71fce42cd Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Fri, 14 Mar 2025 16:12:48 +0100 Subject: [PATCH 58/78] template build logs early return --- .../api/internal/handlers/template_build_logs.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 28ba3b45e..64f1525a1 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -85,6 +85,19 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem return } + // early return if still waiting for build start + if buildInfo.BuildStatus == envbuild.StatusWaiting { + result := api.TemplateBuild{ + Logs: make([]string, 0), + TemplateID: templateID, + BuildID: buildID, + Status: api.TemplateBuildStatusWaiting, + } + + c.JSON(http.StatusOK, result) + return + } + // Sanitize env ID // https://grafana.com/blog/2021/01/05/how-to-escape-special-characters-with-lokis-logql/ templateIdSanitized := strings.ReplaceAll(templateID, "`", "") From fb587a88741c09b5be72000dedb0ef7d74cf7c62 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 17:04:59 +0100 Subject: [PATCH 59/78] Add template manager integration tests --- .github/actions/start-services/action.yml | 4 ++ Makefile | 6 +- tests/integration/Makefile | 15 ++++- .../integration/internal/setup/api_client.go | 8 +++ tests/integration/internal/setup/constants.go | 4 +- .../templates/template_request_build_test.go | 64 +++++++++++++++++++ tests/integration/seed.go | 43 +++++++++---- 7 files changed, 130 insertions(+), 14 deletions(-) create mode 100644 tests/integration/internal/tests/api/templates/template_request_build_test.go diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 9f35d9e65..fb83998aa 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -7,6 +7,8 @@ runs: - name: Run PostgreSQL Database env: TESTS_E2B_API_KEY: "e2b_5ec17bd3933af21f80dc10bba686691c4fcd7057" + TESTS_E2B_ACCESS_TOKEN: "sk_e2b_acce55acce55acce55acce55acce55acce55acce" + TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD: "e2b00e2b-e4fe-47af-8ff6-187bca92f3f9" TESTS_SANDBOX_TEAM_ID: "834777bd-9956-45ca-b088-9bac9290e2ac" TESTS_SANDBOX_USER_ID: "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc" # Signed token using "supabasejwtsecretsupabasejwtsecret" with data @@ -34,6 +36,8 @@ runs: echo "extensions installed" echo "TESTS_E2B_API_KEY=${TESTS_E2B_API_KEY}" >> .env.test + echo "TESTS_E2B_ACCESS_TOKEN=${TESTS_E2B_ACCESS_TOKEN}" >> .env.test + echo "TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD=${TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD}" >> .env.test echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test echo "TESTS_SUPABASE_TOKEN=${TESTS_SUPABASE_TOKEN}" >> .env.test diff --git a/Makefile b/Makefile index 5e1b7de4e..de055bdf9 100644 --- a/Makefile +++ b/Makefile @@ -183,4 +183,8 @@ grafana-apply: .PHONY: connect-orchestrator connect-orchestrator: - $(MAKE) -C tests/integration connect-orchestrator \ No newline at end of file + $(MAKE) -C tests/integration connect-orchestrator + +.PHONY: connect-template-manager +connect-template-manager: + $(MAKE) -C tests/integration connect-template-manager \ No newline at end of file diff --git a/tests/integration/Makefile b/tests/integration/Makefile index 4d4fe400e..0c6c27f97 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -20,8 +20,10 @@ seed: @echo "Applying seeds" @POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING) \ TESTS_E2B_API_KEY=$(TESTS_E2B_API_KEY) \ + TESTS_E2B_ACCESS_TOKEN=$(TESTS_E2B_ACCESS_TOKEN) \ TESTS_SANDBOX_TEMPLATE_ID=$(TESTS_SANDBOX_TEMPLATE_ID) \ TESTS_SANDBOX_BUILD_ID=$(TESTS_SANDBOX_BUILD_ID) \ + TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD=$(TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD) \ go run seed.go @echo "Done" @@ -31,8 +33,10 @@ test: export TESTS_ORCHESTRATOR_HOST=$(TESTS_ORCHESTRATOR_HOST); \ export TESTS_SANDBOX_TEMPLATE_ID=$(TESTS_SANDBOX_TEMPLATE_ID); \ export TESTS_E2B_API_KEY=$(TESTS_E2B_API_KEY); \ + export TESTS_E2B_ACCESS_TOKEN=$(TESTS_E2B_ACCESS_TOKEN); \ export TESTS_SUPABASE_TOKEN=$(TESTS_SUPABASE_TOKEN); \ export TESTS_SANDBOX_TEAM_ID=$(TESTS_SANDBOX_TEAM_ID); \ + export TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD=$(TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD); \ go test -v ./internal/main_test.go -count=1 && \ go test -v ./internal/tests/... -count=1 @@ -43,4 +47,13 @@ connect-orchestrator: --format='value(name)' \ --zones=$(GCP_ZONE) | head -n1) && \ INSTANCE_ID=$$(gcloud compute instance-groups list-instances "$$CLIENT_IG" --zone=$(GCP_ZONE) --format='value(instance)' | head -n1) && \ - gcloud compute ssh "$$INSTANCE_ID" --zone=$(GCP_ZONE) -- -NL 5008:localhost:5008 -o PermitLocalCommand=yes -o LocalCommand="echo 'SSH tunnel established'" \ No newline at end of file + gcloud compute ssh "$$INSTANCE_ID" --zone=$(GCP_ZONE) -- -NL 5008:localhost:5008 -o PermitLocalCommand=yes -o LocalCommand="echo 'SSH tunnel established'" + +.PHONY: connect-template-manager +connect-template-manager: + VM_INSTANCE_ID=$$(gcloud compute instance-groups list \ + --filter="name~'^.*build.*'" \ + --format='value(name)' \ + --zones=$(GCP_ZONE) | head -n1) && \ + INSTANCE_ID=$$(gcloud compute instance-groups list-instances "$$VM_INSTANCE_ID" --zone=$(GCP_ZONE) --format='value(instance)' | head -n1) && \ + gcloud compute ssh "$$INSTANCE_ID" --zone=$(GCP_ZONE) -- -NL 5009:localhost:5009 -o PermitLocalCommand=yes -o LocalCommand="echo 'SSH tunnel established'" \ No newline at end of file diff --git a/tests/integration/internal/setup/api_client.go b/tests/integration/internal/setup/api_client.go index e07f813ad..021971c53 100644 --- a/tests/integration/internal/setup/api_client.go +++ b/tests/integration/internal/setup/api_client.go @@ -27,3 +27,11 @@ func WithAPIKey() func(ctx context.Context, req *http.Request) error { return nil } } + +func WithAccessToken() func(ctx context.Context, req *http.Request) error { + return func(ctx context.Context, req *http.Request) error { + req.Header.Set("Authorization", "Bearer "+AccessToken) + + return nil + } +} diff --git a/tests/integration/internal/setup/constants.go b/tests/integration/internal/setup/constants.go index b30881e26..c4f70bafb 100644 --- a/tests/integration/internal/setup/constants.go +++ b/tests/integration/internal/setup/constants.go @@ -15,9 +15,11 @@ var ( APIServerURL = utils.RequiredEnv("TESTS_API_SERVER_URL", "e.g. https://api.great-innovations.dev") SandboxTemplateID = utils.RequiredEnv("TESTS_SANDBOX_TEMPLATE_ID", "e.g. base") APIKey = utils.RequiredEnv("TESTS_E2B_API_KEY", "your Team API key") + AccessToken = utils.RequiredEnv("TESTS_E2B_ACCESS_TOKEN", "your access token") SupabaseToken = os.Getenv("TESTS_SUPABASE_TOKEN") SupabaseTeamID = os.Getenv("TESTS_SANDBOX_TEAM_ID") - OrchestratorHost = os.Getenv("TESTS_ORCHESTRATOR_HOST") + OrchestratorHost = os.Getenv("TESTS_ORCHESTRATOR_HOST") + TemplateManagerHost = os.Getenv("TESTS_TEMPLATE_MANAGER_HOST") ) diff --git a/tests/integration/internal/tests/api/templates/template_request_build_test.go b/tests/integration/internal/tests/api/templates/template_request_build_test.go new file mode 100644 index 000000000..e29ddb8eb --- /dev/null +++ b/tests/integration/internal/tests/api/templates/template_request_build_test.go @@ -0,0 +1,64 @@ +package templates + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/e2b-dev/infra/packages/shared/pkg/models/envbuild" + "github.com/e2b-dev/infra/tests/integration/internal/api" + "github.com/e2b-dev/infra/tests/integration/internal/setup" + + "github.com/stretchr/testify/assert" +) + +func TestTemplateRequestBuild(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c := setup.GetAPIClient() + + resp, err := c.PostTemplatesWithResponse(ctx, api.TemplateBuildRequest{ + Dockerfile: "FROM alpine:3.14\nRUN echo 'Hello, World!'", + }, setup.WithAccessToken()) + + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + if t.Failed() { + t.Logf("Response: %s", string(resp.Body)) + } + }) + + assert.Equal(t, http.StatusAccepted, resp.StatusCode()) + + var data api.Template + err = json.Unmarshal(resp.Body, &data) + if err != nil { + t.Fatal(err) + } + + templateID := data.TemplateID + + resp2, err := c.GetTemplatesTemplateIDBuildsBuildIDStatusWithResponse(ctx, templateID, data.BuildID, nil, setup.WithAccessToken()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if t.Failed() { + t.Logf("Response: %s", string(resp2.Body)) + } + }) + + assert.Equal(t, http.StatusOK, resp2.StatusCode()) + var statusData api.TemplateBuild + err = json.Unmarshal(resp2.Body, &statusData) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, envbuild.StatusWaiting, statusData.Status) +} diff --git a/tests/integration/seed.go b/tests/integration/seed.go index 78db2d7ab..f42ec76ff 100644 --- a/tests/integration/seed.go +++ b/tests/integration/seed.go @@ -14,11 +14,13 @@ import ( ) type SeedData struct { - APIKey string - EnvID string - BuildID uuid.UUID - TeamID uuid.UUID - UserID uuid.UUID + APIKey string + EnvID string + BuildID uuid.UUID + BuildIDToBeBuild uuid.UUID + TeamID uuid.UUID + UserID uuid.UUID + AccessToken string } func main() { @@ -34,11 +36,13 @@ func main() { defer database.Close() data := SeedData{ - APIKey: os.Getenv("TESTS_E2B_API_KEY"), - EnvID: os.Getenv("TESTS_SANDBOX_TEMPLATE_ID"), - BuildID: uuid.MustParse(os.Getenv("TESTS_SANDBOX_BUILD_ID")), - TeamID: uuid.MustParse(os.Getenv("TESTS_SANDBOX_TEAM_ID")), - UserID: uuid.MustParse(os.Getenv("TESTS_SANDBOX_USER_ID")), + APIKey: os.Getenv("TESTS_E2B_API_KEY"), + EnvID: os.Getenv("TESTS_SANDBOX_TEMPLATE_ID"), + BuildID: uuid.MustParse(os.Getenv("TESTS_SANDBOX_BUILD_ID")), + BuildIDToBeBuild: uuid.MustParse(os.Getenv("TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD")), + TeamID: uuid.MustParse(os.Getenv("TESTS_SANDBOX_TEAM_ID")), + UserID: uuid.MustParse(os.Getenv("TESTS_SANDBOX_USER_ID")), + AccessToken: os.Getenv("TESTS_E2B_ACCESS_TOKEN"), } err = seed(database, data) @@ -61,6 +65,15 @@ func seed(db *db.DB, data SeedData) error { return fmt.Errorf("failed to create user: %w", err) } + // Access Token + _, err = db.Client.AccessToken.Create(). + SetID(data.AccessToken). + SetUserID(data.UserID). + Save(ctx) + if err != nil { + return fmt.Errorf("failed to create user: %w", err) + } + // Team t, err := db.Client.Team.Create(). SetID(data.TeamID). @@ -107,6 +120,7 @@ func seed(db *db.DB, data SeedData) error { type buildData struct { id uuid.UUID createdAt *time.Time + status envbuild.Status } oldBuildTime := time.Now().Add(-time.Hour) @@ -114,11 +128,18 @@ func seed(db *db.DB, data SeedData) error { { id: data.BuildID, createdAt: nil, + status: envbuild.StatusUploaded, }, // An older build, so we have multiple builds { id: uuid.New(), createdAt: &oldBuildTime, + status: envbuild.StatusUploaded, + }, + { + id: data.BuildIDToBeBuild, + createdAt: nil, + status: envbuild.StatusWaiting, }, } @@ -127,7 +148,7 @@ func seed(db *db.DB, data SeedData) error { SetID(build.id). SetEnvID(data.EnvID). SetDockerfile("FROM e2bdev/base:latest"). - SetStatus(envbuild.StatusUploaded). + SetStatus(build.status). SetVcpu(2). SetRAMMB(512). SetFreeDiskSizeMB(512). From 5c6a09ef36a7a4fc38621901fca6c3e2204e8616 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 17:26:01 +0100 Subject: [PATCH 60/78] Add build test --- tests/integration/internal/setup/constants.go | 1 + .../api/templates/template_build_test.go | 66 +++++++++++++++++++ .../templates/template_request_build_test.go | 5 +- 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 tests/integration/internal/tests/api/templates/template_build_test.go diff --git a/tests/integration/internal/setup/constants.go b/tests/integration/internal/setup/constants.go index c4f70bafb..060a0612b 100644 --- a/tests/integration/internal/setup/constants.go +++ b/tests/integration/internal/setup/constants.go @@ -14,6 +14,7 @@ const ( var ( APIServerURL = utils.RequiredEnv("TESTS_API_SERVER_URL", "e.g. https://api.great-innovations.dev") SandboxTemplateID = utils.RequiredEnv("TESTS_SANDBOX_TEMPLATE_ID", "e.g. base") + BuildIDToBeBuild = utils.RequiredEnv("TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD", "e.g. base") APIKey = utils.RequiredEnv("TESTS_E2B_API_KEY", "your Team API key") AccessToken = utils.RequiredEnv("TESTS_E2B_ACCESS_TOKEN", "your access token") diff --git a/tests/integration/internal/tests/api/templates/template_build_test.go b/tests/integration/internal/tests/api/templates/template_build_test.go new file mode 100644 index 000000000..981c72165 --- /dev/null +++ b/tests/integration/internal/tests/api/templates/template_build_test.go @@ -0,0 +1,66 @@ +package templates + +import ( + "context" + "encoding/json" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/e2b-dev/infra/tests/integration/internal/api" + "github.com/e2b-dev/infra/tests/integration/internal/setup" + + "github.com/stretchr/testify/assert" +) + +func TestTemplateBuild(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + c := setup.GetAPIClient() + + resp, err := c.PostTemplatesTemplateIDBuildsBuildIDWithResponse(ctx, setup.SandboxTemplateID, setup.BuildIDToBeBuild, setup.WithAccessToken()) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + if t.Failed() { + t.Logf("Response: %s", string(resp.Body)) + } + }) + + assert.Equal(t, http.StatusAccepted, resp.StatusCode()) + + var finished atomic.Bool + for !finished.Load() { + resp2, err := c.GetTemplatesTemplateIDBuildsBuildIDStatusWithResponse(ctx, setup.SandboxTemplateID, setup.BuildIDToBeBuild, nil, setup.WithAccessToken()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if t.Failed() { + t.Logf("Response: %s", string(resp2.Body)) + } + }) + + if resp2.StatusCode() != http.StatusOK { + t.Fatal("Unexpected status code") + } + + var statusData api.TemplateBuild + err = json.Unmarshal(resp2.Body, &statusData) + if err != nil { + t.Fatal(err) + } + + switch statusData.Status { + case api.TemplateBuildStatusReady: + finished.Store(true) + + case api.TemplateBuildStatusError: + t.Fatal("Build failed") + } + } +} diff --git a/tests/integration/internal/tests/api/templates/template_request_build_test.go b/tests/integration/internal/tests/api/templates/template_request_build_test.go index e29ddb8eb..331224a6f 100644 --- a/tests/integration/internal/tests/api/templates/template_request_build_test.go +++ b/tests/integration/internal/tests/api/templates/template_request_build_test.go @@ -41,12 +41,11 @@ func TestTemplateRequestBuild(t *testing.T) { t.Fatal(err) } - templateID := data.TemplateID - - resp2, err := c.GetTemplatesTemplateIDBuildsBuildIDStatusWithResponse(ctx, templateID, data.BuildID, nil, setup.WithAccessToken()) + resp2, err := c.GetTemplatesTemplateIDBuildsBuildIDStatusWithResponse(ctx, data.TemplateID, data.BuildID, nil, setup.WithAccessToken()) if err != nil { t.Fatal(err) } + t.Cleanup(func() { if t.Failed() { t.Logf("Response: %s", string(resp2.Body)) From 73e3cec53d771fe6aff4fb8547b0016696f9e313 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 17:27:19 +0100 Subject: [PATCH 61/78] Small improvements --- .../tests/api/templates/template_build_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/internal/tests/api/templates/template_build_test.go b/tests/integration/internal/tests/api/templates/template_build_test.go index 981c72165..2abb90137 100644 --- a/tests/integration/internal/tests/api/templates/template_build_test.go +++ b/tests/integration/internal/tests/api/templates/template_build_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "net/http" - "sync/atomic" "testing" "time" @@ -33,8 +32,8 @@ func TestTemplateBuild(t *testing.T) { assert.Equal(t, http.StatusAccepted, resp.StatusCode()) - var finished atomic.Bool - for !finished.Load() { + var finished bool + for !finished { resp2, err := c.GetTemplatesTemplateIDBuildsBuildIDStatusWithResponse(ctx, setup.SandboxTemplateID, setup.BuildIDToBeBuild, nil, setup.WithAccessToken()) if err != nil { t.Fatal(err) @@ -57,10 +56,11 @@ func TestTemplateBuild(t *testing.T) { switch statusData.Status { case api.TemplateBuildStatusReady: - finished.Store(true) - + finished = true case api.TemplateBuildStatusError: t.Fatal("Build failed") } } + + assert.True(t, finished) } From 113df55296e3e2179edf12d86a219556179ef857 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 17:29:09 +0100 Subject: [PATCH 62/78] Run test --- .github/workflows/integration_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a513230c7..306f7ff7b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -2,7 +2,6 @@ name: Integration Tests on: pull_request: - branches: [ main ] jobs: integration_tests: From 703227e4762e2a57181e4df16a2badde4e2e0e29 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 17:39:09 +0100 Subject: [PATCH 63/78] Run build test only in pipeline --- .github/actions/start-services/action.yml | 2 ++ tests/integration/Makefile | 1 + tests/integration/internal/setup/constants.go | 1 + .../internal/tests/api/templates/template_build_test.go | 3 +++ 4 files changed, 7 insertions(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index fb83998aa..264724be5 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -11,6 +11,7 @@ runs: TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD: "e2b00e2b-e4fe-47af-8ff6-187bca92f3f9" TESTS_SANDBOX_TEAM_ID: "834777bd-9956-45ca-b088-9bac9290e2ac" TESTS_SANDBOX_USER_ID: "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc" + TESTS_RUN_BUILD_TEST: "true" # Signed token using "supabasejwtsecretsupabasejwtsecret" with data # { # "sub": "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc", @@ -41,6 +42,7 @@ runs: echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test echo "TESTS_SUPABASE_TOKEN=${TESTS_SUPABASE_TOKEN}" >> .env.test + echo "TESTS_RUN_BUILD_TEST=${TESTS_RUN_BUILD_TEST}" >> .env.test set -x make migrate make -C tests/integration seed diff --git a/tests/integration/Makefile b/tests/integration/Makefile index 0c6c27f97..c0db2388f 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -37,6 +37,7 @@ test: export TESTS_SUPABASE_TOKEN=$(TESTS_SUPABASE_TOKEN); \ export TESTS_SANDBOX_TEAM_ID=$(TESTS_SANDBOX_TEAM_ID); \ export TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD=$(TESTS_SANDBOX_BUILD_ID_TO_BE_BUILD); \ + export TESTS_RUN_BUILD_TEST=$(TESTS_RUN_BUILD_TEST); \ go test -v ./internal/main_test.go -count=1 && \ go test -v ./internal/tests/... -count=1 diff --git a/tests/integration/internal/setup/constants.go b/tests/integration/internal/setup/constants.go index 060a0612b..3da450d88 100644 --- a/tests/integration/internal/setup/constants.go +++ b/tests/integration/internal/setup/constants.go @@ -23,4 +23,5 @@ var ( OrchestratorHost = os.Getenv("TESTS_ORCHESTRATOR_HOST") TemplateManagerHost = os.Getenv("TESTS_TEMPLATE_MANAGER_HOST") + RunBuildTest = os.Getenv("TESTS_RUN_BUILD_TEST") ) diff --git a/tests/integration/internal/tests/api/templates/template_build_test.go b/tests/integration/internal/tests/api/templates/template_build_test.go index 2abb90137..64c227500 100644 --- a/tests/integration/internal/tests/api/templates/template_build_test.go +++ b/tests/integration/internal/tests/api/templates/template_build_test.go @@ -14,6 +14,9 @@ import ( ) func TestTemplateBuild(t *testing.T) { + if setup.RunBuildTest == "" { + t.Skip("Build test is disabled") + } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() From ea6b1f0cb6f730416a7e743f3fcaf6b469ba1043 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 17:56:42 +0100 Subject: [PATCH 64/78] Clean up --- Makefile | 6 +----- tests/integration/Makefile | 11 +---------- tests/integration/internal/setup/constants.go | 5 ++--- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index de055bdf9..5e1b7de4e 100644 --- a/Makefile +++ b/Makefile @@ -183,8 +183,4 @@ grafana-apply: .PHONY: connect-orchestrator connect-orchestrator: - $(MAKE) -C tests/integration connect-orchestrator - -.PHONY: connect-template-manager -connect-template-manager: - $(MAKE) -C tests/integration connect-template-manager \ No newline at end of file + $(MAKE) -C tests/integration connect-orchestrator \ No newline at end of file diff --git a/tests/integration/Makefile b/tests/integration/Makefile index c0db2388f..da455e169 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -48,13 +48,4 @@ connect-orchestrator: --format='value(name)' \ --zones=$(GCP_ZONE) | head -n1) && \ INSTANCE_ID=$$(gcloud compute instance-groups list-instances "$$CLIENT_IG" --zone=$(GCP_ZONE) --format='value(instance)' | head -n1) && \ - gcloud compute ssh "$$INSTANCE_ID" --zone=$(GCP_ZONE) -- -NL 5008:localhost:5008 -o PermitLocalCommand=yes -o LocalCommand="echo 'SSH tunnel established'" - -.PHONY: connect-template-manager -connect-template-manager: - VM_INSTANCE_ID=$$(gcloud compute instance-groups list \ - --filter="name~'^.*build.*'" \ - --format='value(name)' \ - --zones=$(GCP_ZONE) | head -n1) && \ - INSTANCE_ID=$$(gcloud compute instance-groups list-instances "$$VM_INSTANCE_ID" --zone=$(GCP_ZONE) --format='value(instance)' | head -n1) && \ - gcloud compute ssh "$$INSTANCE_ID" --zone=$(GCP_ZONE) -- -NL 5009:localhost:5009 -o PermitLocalCommand=yes -o LocalCommand="echo 'SSH tunnel established'" \ No newline at end of file + gcloud compute ssh "$$INSTANCE_ID" --zone=$(GCP_ZONE) -- -NL 5008:localhost:5008 -o PermitLocalCommand=yes -o LocalCommand="echo 'SSH tunnel established'" \ No newline at end of file diff --git a/tests/integration/internal/setup/constants.go b/tests/integration/internal/setup/constants.go index 3da450d88..e501bb884 100644 --- a/tests/integration/internal/setup/constants.go +++ b/tests/integration/internal/setup/constants.go @@ -21,7 +21,6 @@ var ( SupabaseToken = os.Getenv("TESTS_SUPABASE_TOKEN") SupabaseTeamID = os.Getenv("TESTS_SANDBOX_TEAM_ID") - OrchestratorHost = os.Getenv("TESTS_ORCHESTRATOR_HOST") - TemplateManagerHost = os.Getenv("TESTS_TEMPLATE_MANAGER_HOST") - RunBuildTest = os.Getenv("TESTS_RUN_BUILD_TEST") + OrchestratorHost = os.Getenv("TESTS_ORCHESTRATOR_HOST") + RunBuildTest = os.Getenv("TESTS_RUN_BUILD_TEST") ) From 82d2b0b3a08301bd3b4c913f92d0867f79b05e8c Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 18:18:43 +0100 Subject: [PATCH 65/78] Add template manager address --- .github/actions/start-services/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 264724be5..dc1b71e0a 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -52,6 +52,7 @@ runs: env: ENVD_TIMEOUT: "60s" SUPABASE_JWT_SECRETS: "supabasejwtsecretsupabasejwtsecret" + TEMPLATE_MANAGER_ADDRESS: "localhost:5009" run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRETS}" >> .env.test From c7b2d835126b0316ae5fec48f896f0216aa64b77 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 18:53:52 +0100 Subject: [PATCH 66/78] Start template manager --- .github/actions/start-services/action.yml | 1 + packages/template-manager/Makefile | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index dc1b71e0a..1f62f1487 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -59,6 +59,7 @@ runs: mkdir -p ~/logs + make -C packages/template-manager run-debug 2>&1 | tee ~/logs/template-manager.log & make -C packages/orchestrator run-debug 2>&1 | tee ~/logs/orchestrator.log & ORCH_PID=$! sleep 30 diff --git a/packages/template-manager/Makefile b/packages/template-manager/Makefile index 3cc1dde56..89f20b2c5 100644 --- a/packages/template-manager/Makefile +++ b/packages/template-manager/Makefile @@ -27,6 +27,12 @@ build: build-debug: CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -race -gcflags=all="-N -l" -o bin/template-manager . +.PHONY: run-debug +run-debug: + make build-debug + ./bin/template-manager + + .PHONY: upload upload: ./upload.sh $(GCP_PROJECT_ID) From a4f7a900edc2d45d8c773e80dbb1b71559f82f99 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 19:51:20 +0100 Subject: [PATCH 67/78] Pass variable --- .github/actions/start-services/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 1f62f1487..f06aebbe1 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -56,7 +56,8 @@ runs: run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRETS}" >> .env.test - + echo "TEMPLATE_MANAGER_ADDRESS=${TEMPLATE_MANAGER_ADDRESS}" >> .env.test + mkdir -p ~/logs make -C packages/template-manager run-debug 2>&1 | tee ~/logs/template-manager.log & From 818f863ff792a34809764e0e3fc7caae45ffa29a Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 19:51:34 +0100 Subject: [PATCH 68/78] Update spec --- tests/integration/internal/api/models.gen.go | 33 +++++++++++++++---- .../templates/template_request_build_test.go | 3 +- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/integration/internal/api/models.gen.go b/tests/integration/internal/api/models.gen.go index 258089061..d23512d23 100644 --- a/tests/integration/internal/api/models.gen.go +++ b/tests/integration/internal/api/models.gen.go @@ -10,15 +10,19 @@ import ( ) const ( - AccessTokenAuthScopes = "AccessTokenAuth.Scopes" - AdminTokenAuthScopes = "AdminTokenAuth.Scopes" - ApiKeyAuthScopes = "ApiKeyAuth.Scopes" + AccessTokenAuthScopes = "AccessTokenAuth.Scopes" + AdminTokenAuthScopes = "AdminTokenAuth.Scopes" + ApiKeyAuthScopes = "ApiKeyAuth.Scopes" + Supabase1TokenAuthScopes = "Supabase1TokenAuth.Scopes" + Supabase2TeamAuthScopes = "Supabase2TeamAuth.Scopes" ) // Defines values for NodeStatus. const ( - NodeStatusDraining NodeStatus = "draining" - NodeStatusReady NodeStatus = "ready" + NodeStatusConnecting NodeStatus = "connecting" + NodeStatusDraining NodeStatus = "draining" + NodeStatusReady NodeStatus = "ready" + NodeStatusUnhealthy NodeStatus = "unhealthy" ) // Defines values for TemplateBuildStatus. @@ -26,6 +30,7 @@ const ( TemplateBuildStatusBuilding TemplateBuildStatus = "building" TemplateBuildStatusError TemplateBuildStatus = "error" TemplateBuildStatusReady TemplateBuildStatus = "ready" + TemplateBuildStatusWaiting TemplateBuildStatus = "waiting" ) // CPUCount CPU cores for the sandbox @@ -48,8 +53,10 @@ type MemoryMB = int32 // NewSandbox defines model for NewSandbox. type NewSandbox struct { - EnvVars *EnvVars `json:"envVars,omitempty"` - Metadata *SandboxMetadata `json:"metadata,omitempty"` + // AutoPause Automatically pauses the sandbox after the timeout + AutoPause *bool `json:"autoPause,omitempty"` + EnvVars *EnvVars `json:"envVars,omitempty"` + Metadata *SandboxMetadata `json:"metadata,omitempty"` // TemplateID Identifier of the required template TemplateID string `json:"templateID"` @@ -66,12 +73,18 @@ type Node struct { // AllocatedMemoryMiB Amount of allocated memory in MiB AllocatedMemoryMiB int32 `json:"allocatedMemoryMiB"` + // CreateFails Number of sandbox create fails + CreateFails uint64 `json:"createFails"` + // NodeID Identifier of the node NodeID string `json:"nodeID"` // SandboxCount Number of sandboxes running on the node SandboxCount int32 `json:"sandboxCount"` + // SandboxStartingCount Number of starting Sandboxes + SandboxStartingCount int `json:"sandboxStartingCount"` + // Status Status of the node Status NodeStatus `json:"status"` } @@ -81,6 +94,9 @@ type NodeDetail struct { // CachedBuilds List of cached builds id on the node CachedBuilds []string `json:"cachedBuilds"` + // CreateFails Number of sandbox create fails + CreateFails uint64 `json:"createFails"` + // NodeID Identifier of the node NodeID string `json:"nodeID"` @@ -102,6 +118,9 @@ type NodeStatusChange struct { // ResumedSandbox defines model for ResumedSandbox. type ResumedSandbox struct { + // AutoPause Automatically pauses the sandbox after the timeout + AutoPause *bool `json:"autoPause,omitempty"` + // Timeout Time to live for the sandbox in seconds. Timeout *int32 `json:"timeout,omitempty"` } diff --git a/tests/integration/internal/tests/api/templates/template_request_build_test.go b/tests/integration/internal/tests/api/templates/template_request_build_test.go index 331224a6f..12fb28b49 100644 --- a/tests/integration/internal/tests/api/templates/template_request_build_test.go +++ b/tests/integration/internal/tests/api/templates/template_request_build_test.go @@ -6,7 +6,6 @@ import ( "net/http" "testing" - "github.com/e2b-dev/infra/packages/shared/pkg/models/envbuild" "github.com/e2b-dev/infra/tests/integration/internal/api" "github.com/e2b-dev/infra/tests/integration/internal/setup" @@ -59,5 +58,5 @@ func TestTemplateRequestBuild(t *testing.T) { t.Fatal(err) } - assert.Equal(t, envbuild.StatusWaiting, statusData.Status) + assert.Equal(t, api.TemplateBuildStatusWaiting, statusData.Status) } From e4c6353e6e59d24495629fde38380b221a85d790 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 20:07:08 +0100 Subject: [PATCH 69/78] temp fix --- .../api/internal/handlers/template_build_logs.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 64f1525a1..73b04eb83 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -4,11 +4,12 @@ import ( "encoding/json" "errors" "fmt" - "github.com/e2b-dev/infra/packages/shared/pkg/db" "net/http" "strings" "time" + "github.com/e2b-dev/infra/packages/shared/pkg/db" + "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "go.uber.org/zap" @@ -106,6 +107,18 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem end := time.Now() start := end.Add(-templateBuildOldestLogsLimit) + logs := make([]string, 0) + + if a.lokiClient == nil { + result := api.TemplateBuild{ + Logs: logs, + TemplateID: templateID, + BuildID: buildID, + Status: getCorrespondingTemplateBuildStatus(buildInfo.BuildStatus), + } + + c.JSON(http.StatusOK, result) + } res, err := a.lokiClient.QueryRange(query, templateBuildLogsLimit, start, end, logproto.FORWARD, time.Duration(0), time.Duration(0), true) if err != nil { errMsg := fmt.Errorf("error when returning logs for template build: %w", err) @@ -116,7 +129,6 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem return } - logs := make([]string, 0) logsCrawled := 0 offset := 0 From 0e7e851df3fe45f928df2b748f054c6b5d002800 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 20:15:59 +0100 Subject: [PATCH 70/78] Test --- .github/actions/start-services/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index f06aebbe1..2b385aff9 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -52,7 +52,7 @@ runs: env: ENVD_TIMEOUT: "60s" SUPABASE_JWT_SECRETS: "supabasejwtsecretsupabasejwtsecret" - TEMPLATE_MANAGER_ADDRESS: "localhost:5009" + TEMPLATE_MANAGER_ADDRESS: "http://127.0.0.1:5009" run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRETS}" >> .env.test From b7f338e50514a9b1a5c92d1a033deff2b99c6303 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 20:16:18 +0100 Subject: [PATCH 71/78] Fix --- packages/api/internal/handlers/template_build_logs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/internal/handlers/template_build_logs.go b/packages/api/internal/handlers/template_build_logs.go index 73b04eb83..c146c9e0c 100644 --- a/packages/api/internal/handlers/template_build_logs.go +++ b/packages/api/internal/handlers/template_build_logs.go @@ -118,6 +118,7 @@ func (a *APIStore) GetTemplatesTemplateIDBuildsBuildIDStatus(c *gin.Context, tem } c.JSON(http.StatusOK, result) + return } res, err := a.lokiClient.QueryRange(query, templateBuildLogsLimit, start, end, logproto.FORWARD, time.Duration(0), time.Duration(0), true) if err != nil { From a09f270899a065b9250cd09d8bb325f5293457e9 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 21:45:01 +0100 Subject: [PATCH 72/78] Test --- .github/actions/start-services/action.yml | 4 ++++ .../internal/tests/api/templates/template_build_test.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 2b385aff9..badfd0a5b 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -53,10 +53,14 @@ runs: ENVD_TIMEOUT: "60s" SUPABASE_JWT_SECRETS: "supabasejwtsecretsupabasejwtsecret" TEMPLATE_MANAGER_ADDRESS: "http://127.0.0.1:5009" + GCP_PROJECT_ID: "e2b-prod" + GCP_DOCKER_REPOSITORY_NAME: "custom-environments-tests" + GOOGLE_SERVICE_ACCOUNT_BASE64: "required-but-not-needed" run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRETS}" >> .env.test echo "TEMPLATE_MANAGER_ADDRESS=${TEMPLATE_MANAGER_ADDRESS}" >> .env.test + echo "GCP_PROJECT_ID=${GCP_PROJECT_ID}" >> .env.test mkdir -p ~/logs diff --git a/tests/integration/internal/tests/api/templates/template_build_test.go b/tests/integration/internal/tests/api/templates/template_build_test.go index 64c227500..2cf140e72 100644 --- a/tests/integration/internal/tests/api/templates/template_build_test.go +++ b/tests/integration/internal/tests/api/templates/template_build_test.go @@ -3,6 +3,7 @@ package templates import ( "context" "encoding/json" + "fmt" "net/http" "testing" "time" @@ -33,6 +34,7 @@ func TestTemplateBuild(t *testing.T) { } }) + fmt.Printf("Resp build test: %s\n", resp.HTTPResponse.Request.URL.String()) assert.Equal(t, http.StatusAccepted, resp.StatusCode()) var finished bool From 6a1607c9265f88184f43a902428af37c9a3cab26 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 21:45:11 +0100 Subject: [PATCH 73/78] Test --- .github/actions/start-services/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index badfd0a5b..08973896f 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -61,6 +61,8 @@ runs: echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRETS}" >> .env.test echo "TEMPLATE_MANAGER_ADDRESS=${TEMPLATE_MANAGER_ADDRESS}" >> .env.test echo "GCP_PROJECT_ID=${GCP_PROJECT_ID}" >> .env.test + echo "GCP_DOCKER_REPOSITORY_NAME=${GCP_DOCKER_REPOSITORY_NAME}" >> .env.test + echo "GOOGLE_SERVICE_ACCOUNT_BASE64=${GOOGLE_SERVICE_ACCOUNT_BASE64}" >> .env.test mkdir -p ~/logs From 62f4fdac760361e7e2d05ccfacc88e966f601aa0 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 21:53:31 +0100 Subject: [PATCH 74/78] Test --- .github/actions/start-services/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 08973896f..a15e8b105 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -56,6 +56,7 @@ runs: GCP_PROJECT_ID: "e2b-prod" GCP_DOCKER_REPOSITORY_NAME: "custom-environments-tests" GOOGLE_SERVICE_ACCOUNT_BASE64: "required-but-not-needed" + TEMPLATE_BUCKET_NAME: "e2b-tests-fc-templates" run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRETS}" >> .env.test @@ -63,6 +64,7 @@ runs: echo "GCP_PROJECT_ID=${GCP_PROJECT_ID}" >> .env.test echo "GCP_DOCKER_REPOSITORY_NAME=${GCP_DOCKER_REPOSITORY_NAME}" >> .env.test echo "GOOGLE_SERVICE_ACCOUNT_BASE64=${GOOGLE_SERVICE_ACCOUNT_BASE64}" >> .env.test + echo "TEMPLATE_BUCKET_NAME=${TEMPLATE_BUCKET_NAME}" >> .env.test mkdir -p ~/logs From 995233e7a85291ea1eb5cfb99f6e3b91c1607ce6 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 22:08:28 +0100 Subject: [PATCH 75/78] Add logs --- .github/workflows/integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 306f7ff7b..0026759f2 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -35,6 +35,7 @@ jobs: ls -l ~/logs tail -f ~/logs/orchestrator.log -n 0 & tail -f ~/logs/api.log -n 0 & + tail -f ~/logs/template-manager.log -n 0 & # Run the integration tests make test-integration From 52441ff3208864825d6f9b5e14618a68aba0d98b Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Fri, 14 Mar 2025 22:36:26 +0100 Subject: [PATCH 76/78] Use secret --- .github/actions/start-services/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index a15e8b105..12b08d4df 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -55,7 +55,7 @@ runs: TEMPLATE_MANAGER_ADDRESS: "http://127.0.0.1:5009" GCP_PROJECT_ID: "e2b-prod" GCP_DOCKER_REPOSITORY_NAME: "custom-environments-tests" - GOOGLE_SERVICE_ACCOUNT_BASE64: "required-but-not-needed" + GOOGLE_SERVICE_ACCOUNT_BASE64: ${{ secrets.TESTS_GOOGLE_SERVICE_ACCOUNT_BASE64 }} TEMPLATE_BUCKET_NAME: "e2b-tests-fc-templates" run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test From 18263b063f035eae894a1123a122adca8973ee48 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Sat, 15 Mar 2025 11:00:21 +0100 Subject: [PATCH 77/78] Pass secret --- .github/actions/start-services/action.yml | 2 +- .github/workflows/integration_tests.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 12b08d4df..b441e614e 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -55,7 +55,7 @@ runs: TEMPLATE_MANAGER_ADDRESS: "http://127.0.0.1:5009" GCP_PROJECT_ID: "e2b-prod" GCP_DOCKER_REPOSITORY_NAME: "custom-environments-tests" - GOOGLE_SERVICE_ACCOUNT_BASE64: ${{ secrets.TESTS_GOOGLE_SERVICE_ACCOUNT_BASE64 }} + GOOGLE_SERVICE_ACCOUNT_BASE64: ${{ env.GOOGLE_SERVICE_ACCOUNT_BASE64 }} TEMPLATE_BUCKET_NAME: "e2b-tests-fc-templates" run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 0026759f2..fdc37c5a7 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -24,6 +24,8 @@ jobs: uses: ./.github/actions/build-sandbox-template - name: Start Services + env: + GOOGLE_SERVICE_ACCOUNT_BASE64: ${{ secrets.TESTS_GOOGLE_SERVICE_ACCOUNT_BASE64 }} uses: ./.github/actions/start-services - name: Run Integration Tests From 65472f57d4e13493c7d0ebfc43c81f0ac511d8de Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Sat, 15 Mar 2025 11:07:37 +0100 Subject: [PATCH 78/78] Pass secret --- .github/actions/start-services/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index b441e614e..9ad6f1133 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -55,7 +55,6 @@ runs: TEMPLATE_MANAGER_ADDRESS: "http://127.0.0.1:5009" GCP_PROJECT_ID: "e2b-prod" GCP_DOCKER_REPOSITORY_NAME: "custom-environments-tests" - GOOGLE_SERVICE_ACCOUNT_BASE64: ${{ env.GOOGLE_SERVICE_ACCOUNT_BASE64 }} TEMPLATE_BUCKET_NAME: "e2b-tests-fc-templates" run: | echo "ENVD_TIMEOUT=${ENVD_TIMEOUT}" >> .env.test @@ -63,9 +62,13 @@ runs: echo "TEMPLATE_MANAGER_ADDRESS=${TEMPLATE_MANAGER_ADDRESS}" >> .env.test echo "GCP_PROJECT_ID=${GCP_PROJECT_ID}" >> .env.test echo "GCP_DOCKER_REPOSITORY_NAME=${GCP_DOCKER_REPOSITORY_NAME}" >> .env.test - echo "GOOGLE_SERVICE_ACCOUNT_BASE64=${GOOGLE_SERVICE_ACCOUNT_BASE64}" >> .env.test echo "TEMPLATE_BUCKET_NAME=${TEMPLATE_BUCKET_NAME}" >> .env.test + export ACCESS_TOKEN=$(gcloud auth print-access-token) + export DOCKER_AUTH_BASE64=$(echo -n "{\"username\":\"oauth2accesstoken\",\"password\":\"$ACCESS_TOKEN\"}" | base64 -w 0) + echo "::add-mask::$DOCKER_AUTH_BASE64" + echo "DOCKER_AUTH_BASE64=${DOCKER_AUTH_BASE64}" >> .env.test + mkdir -p ~/logs make -C packages/template-manager run-debug 2>&1 | tee ~/logs/template-manager.log &