diff --git a/.gitignore b/.gitignore index a558c8e..27a6a82 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ ant data conf container-volume -./bin/** \ No newline at end of file +./bin/** + +config.yaml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c40188c..67a0c2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ FROM golang:1.23.0-bookworm AS builder ARG TARGETOS=linux ARG TARGETARCH=amd64 -RUN apt-get update && \ - apt-get install -y --no-install-recommends make rsync && \ +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends make rsync openssh-server ssh && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -35,7 +35,6 @@ COPY --from=builder /go/src/github.com/cloud-barista/cm-ant/ant /app/ant COPY --from=builder /go/src/github.com/cloud-barista/cm-ant/config.yaml /app/config.yaml COPY --from=builder /go/src/github.com/cloud-barista/cm-ant/test_plan /app/test_plan COPY --from=builder /go/src/github.com/cloud-barista/cm-ant/script /app/script -COPY --from=builder /go/src/github.com/cloud-barista/cm-ant/meta /app/meta HEALTHCHECK --interval=10s --timeout=5s --start-period=10s \ CMD curl -f "http://localhost:8880/ant/api/v1/readyz" || exit 1 diff --git a/Makefile b/Makefile index 7daf09d..03fa44b 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,23 @@ +########################################################### +ANT_NETWORK=cm-ant-local-net +DB_CONTAINER_NAME=ant-local-postgres +DB_NAME=cm-ant-db +DB_USER=cm-ant-user +DB_PASSWORD=cm-ant-secret + +ANT_CONTAINER_NAME=cm-ant +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +ARCH := $(shell uname -m) + +ifeq ($(ARCH),x86_64) + ARCH := amd64 +else ifeq ($(ARCH),arm64) + ARCH := arm64 +else ifeq ($(ARCH),aarch64) + ARCH := arm64 +endif +########################################################### + ########################################################### .PHONY: swag swag: @@ -12,6 +32,66 @@ build: ########################################################### .PHONY: run -run: +run: run-db @go run cmd/cm-ant/main.go ########################################################### + +########################################################### +.PHONY: create-network +create-network: + @if [ -z "$$(docker network ls -q -f name=$(ANT_NETWORK))" ]; then \ + echo "Creating cm-ant network..."; \ + docker network create --driver bridge $(ANT_NETWORK); \ + echo "cm-ant network created!"; \ + else \ + echo "cm-ant network already exist..."; \ + fi +########################################################### + +########################################################### +.PHONY: run-db +run-db: create-network + @if [ -z "$$(docker container ps -q -f name=$(DB_CONTAINER_NAME))" ]; then \ + echo "Run database container...."; \ + docker container run \ + --name $(DB_CONTAINER_NAME) \ + --network $(ANT_NETWORK) \ + -p 5432:5432 \ + -e POSTGRES_USER=$(DB_USER) \ + -e POSTGRES_PASSWORD=$(DB_PASSWORD) \ + -e POSTGRES_DB=$(DB_NAME) \ + -d --rm \ + timescale/timescaledb:latest-pg16; \ + echo "Started Postgres database container!"; \ + echo "Waiting for database to be ready..."; \ + for i in $$(seq 1 10); do \ + docker container exec $(DB_CONTAINER_NAME) pg_isready -U $(DB_USER) -d $(DB_NAME); \ + if [ $$? -eq 0 ]; then \ + echo "Database is ready!"; \ + break; \ + fi; \ + echo "Database is not ready yet. Waiting..."; \ + sleep 5; \ + done; \ + if [ $$i -eq 10 ]; then \ + echo "Failed to start the database"; \ + exit 1; \ + fi; \ + echo "Database $(DB_NAME) successfully started!"; \ + else \ + echo "Database container is already running."; \ + fi +########################################################### + +########################################################### +.PHONY: down +down: + @echo "Checking if the database container is running..." + @if [ -n "$$(docker container ps -q -f name=$(DB_CONTAINER_NAME))" ]; then \ + echo "Stopping and removing the database container..."; \ + docker container stop $(DB_CONTAINER_NAME); \ + echo "Database container stopped!"; \ + else \ + echo "No running database container found."; \ + fi +########################################################### \ No newline at end of file diff --git a/cmd/cm-ant/main.go b/cmd/cm-ant/main.go index ceb3d4b..b6b4fd2 100644 --- a/cmd/cm-ant/main.go +++ b/cmd/cm-ant/main.go @@ -19,7 +19,7 @@ import ( // @basePath /ant func main() { - err := utils.Script(utils.JoinRootPathWith("/script/install_rsync.sh"), []string{}) + err := utils.Script(utils.JoinRootPathWith("/script/install_required_utils.sh"), []string{}) if err != nil { log.Fatal("required tool can not install") } diff --git a/docker-compose.yaml b/docker-compose.yaml index a627c31..0f13a7f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -50,7 +50,7 @@ services: restart: unless-stopped cb-tumblebug: - image: cloudbaristaorg/cb-tumblebug:0.9.11 + image: cloudbaristaorg/cb-tumblebug:0.9.13 container_name: cb-tumblebug platform: linux/amd64 ports: diff --git a/internal/core/load/authorized_key_checker.go b/internal/core/load/authorized_key_checker.go new file mode 100644 index 0000000..c61bd88 --- /dev/null +++ b/internal/core/load/authorized_key_checker.go @@ -0,0 +1,57 @@ +package load + +import ( + "fmt" + "os" + "strings" + + "github.com/cloud-barista/cm-ant/internal/utils" +) + +// getAddAuthorizedKeyCommand returns a command to add the authorized key. +func getAddAuthorizedKeyCommand(pkName, pubkName string) (string, error) { + pubKeyPath, _, err := validateKeyPair(pkName, pubkName) + if err != nil { + return "", err + } + + pub, err := utils.ReadToString(pubKeyPath) + if err != nil { + return "", err + } + + addAuthorizedKeyScript := utils.JoinRootPathWith("/script/add-authorized-key.sh") + + addAuthorizedKeyCommand, err := utils.ReadToString(addAuthorizedKeyScript) + if err != nil { + return "", err + } + + addAuthorizedKeyCommand = strings.Replace(addAuthorizedKeyCommand, `PUBLIC_KEY=""`, fmt.Sprintf(`PUBLIC_KEY="%s"`, pub), 1) + return addAuthorizedKeyCommand, nil +} + +// validateKeyPair checks and generates SSH key pair if it doesn't exist. +func validateKeyPair(pkName, pubkName string) (string, string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", "", err + } + + privKeyPath := fmt.Sprintf("%s/.ssh/%s", homeDir, pkName) + pubKeyPath := fmt.Sprintf("%s/.ssh/%s", homeDir, pubkName) + + err = utils.CreateFolderIfNotExist(fmt.Sprintf("%s/.ssh", homeDir)) + if err != nil { + return pubKeyPath, privKeyPath, err + } + + exist := utils.ExistCheck(privKeyPath) + if !exist { + err := utils.GenerateSSHKeyPair(4096, privKeyPath, pubKeyPath) + if err != nil { + return pubKeyPath, privKeyPath, err + } + } + return pubKeyPath, privKeyPath, nil +} diff --git a/internal/core/load/load_generator_install_service.go b/internal/core/load/load_generator_install_service.go new file mode 100644 index 0000000..058afc0 --- /dev/null +++ b/internal/core/load/load_generator_install_service.go @@ -0,0 +1,540 @@ +package load + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/cloud-barista/cm-ant/internal/config" + "github.com/cloud-barista/cm-ant/internal/core/common/constant" + "github.com/cloud-barista/cm-ant/internal/infra/outbound/tumblebug" + "github.com/cloud-barista/cm-ant/internal/utils" + "gorm.io/gorm" +) + +const ( + antNsId = "ant-default-ns" + antMciDescription = "Default MCI for Cloud Migration Verification" + antInstallMonAgent = "no" + antLabelKey = "ant-default-label" + antMciLabel = "DynamicMci,AntDefault" + antMciId = "ant-default-mci" + + antVmDescription = "Default VM for Cloud Migration Verification" + antVmLabel = "DynamicVm,AntDefault" + antVmName = "ant-default-vm" + antVmRootDiskSize = "default" + antVmRootDiskType = "default" + antVmSubGroupSize = "1" + antVmUserPassword = "" + + antPubKeyName = "id_rsa_ant.pub" + antPrivKeyName = "id_rsa_ant" + + defaultDelay = 20 * time.Second + imageOs = "ubuntu22.04" +) + +// InstallLoadGenerator installs the load generator either locally or remotely. +// Currently remote request is executing via cb-tumblebug. +func (l *LoadService) InstallLoadGenerator(param InstallLoadGeneratorParam) (LoadGeneratorInstallInfoResult, error) { + utils.LogInfo("Starting InstallLoadGenerator") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + var result LoadGeneratorInstallInfoResult + + loadGeneratorInstallInfo := &LoadGeneratorInstallInfo{ + InstallLocation: param.InstallLocation, + InstallType: "jmeter", + InstallPath: config.AppConfig.Load.JMeter.Dir, + InstallVersion: config.AppConfig.Load.JMeter.Version, + Status: "starting", + } + + err := l.loadRepo.GetOrInsertLoadGeneratorInstallInfoTx(ctx, loadGeneratorInstallInfo) + if err != nil { + utils.LogError("Failed to insert LoadGeneratorInstallInfo:", err) + return result, err + } + utils.LogInfo("LoadGeneratorInstallInfo fetched successfully") + + defer func() { + if loadGeneratorInstallInfo.Status == "starting" { + loadGeneratorInstallInfo.Status = "failed" + err = l.loadRepo.UpdateLoadGeneratorInstallInfoTx(ctx, loadGeneratorInstallInfo) + if err != nil { + utils.LogError("Error updating LoadGeneratorInstallInfo to failed status:", err) + } + } + }() + + log.Println("install info : ", loadGeneratorInstallInfo) + + if len(loadGeneratorInstallInfo.LoadGeneratorServers) == 0 { + installLocation := param.InstallLocation + installScriptPath := utils.JoinRootPathWith("/script/install-jmeter.sh") + + switch installLocation { + case constant.Local: + utils.LogInfo("Starting local installation of JMeter") + err := utils.Script(installScriptPath, []string{ + fmt.Sprintf("JMETER_WORK_DIR=%s", config.AppConfig.Load.JMeter.Dir), + fmt.Sprintf("JMETER_VERSION=%s", config.AppConfig.Load.JMeter.Version), + }) + if err != nil { + utils.LogError("Error while installing JMeter locally:", err) + return result, fmt.Errorf("error while installing jmeter; %s", err) + } + utils.LogInfo("Local installation of JMeter completed successfully") + case constant.Remote: + utils.LogInfo("Starting remote installation of JMeter") + // get the spec and image information + recommendVm, err := l.getRecommendVm(ctx, param.Coordinates) + if err != nil { + utils.LogError("Failed to get recommended VM:", err) + return result, err + } + antVmCommonSpec := recommendVm[0].Name + antVmConnectionName := recommendVm[0].ConnectionName + antVmCommonImage, err := utils.ReplaceAtIndex(antVmCommonSpec, imageOs, "+", 2) + + if err != nil { + utils.LogError("Error replacing VM spec index:", err) + return result, err + } + + // check namespace is valid or not + err = l.validDefaultNs(ctx, antNsId) + if err != nil { + utils.LogError("Error validating default namespace:", err) + return result, err + } + + // get the ant default mci + antMci, err := l.getAndDefaultMci(ctx, antVmCommonSpec, antVmCommonImage, antVmConnectionName) + if err != nil { + utils.LogError("Error getting or creating default mci:", err) + return result, err + } + + // if server is not running state, try to resume and get mci information + retryCount := config.AppConfig.Load.Retry + for retryCount > 0 && antMci.StatusCount.CountRunning < 1 { + utils.LogInfof("Attempting to resume MCI, retry count: %d", retryCount) + + err = l.tumblebugClient.ControlLifecycleWithContext(ctx, antNsId, antMci.ID, "resume") + if err != nil { + utils.LogError("Error resuming MCI:", err) + return result, err + } + time.Sleep(defaultDelay) + antMci, err = l.getAndDefaultMci(ctx, antVmCommonSpec, antVmCommonImage, antVmConnectionName) + if err != nil { + utils.LogError("Error getting MCI after resume attempt:", err) + return result, err + } + + retryCount = retryCount - 1 + } + + if antMci.StatusCount.CountRunning < 1 { + utils.LogError("No running VM on ant default MCI") + return result, errors.New("there is no running vm on ant default mci") + } + + addAuthorizedKeyCommand, err := getAddAuthorizedKeyCommand(antPrivKeyName, antPubKeyName) + if err != nil { + utils.LogError("Error getting add authorized key command:", err) + return result, err + } + + installationCommand, err := utils.ReadToString(installScriptPath) + if err != nil { + utils.LogError("Error reading installation script:", err) + return result, err + } + + commandReq := tumblebug.SendCommandReq{ + Command: []string{installationCommand, addAuthorizedKeyCommand}, + } + + _, err = l.tumblebugClient.CommandToMciWithContext(ctx, antNsId, antMci.ID, commandReq) + if err != nil { + utils.LogError("Error sending command to MCI:", err) + return result, err + } + + utils.LogInfo("Commands sent to MCI successfully") + + loadGeneratorServers := make([]LoadGeneratorServer, 0) + + for i, vm := range antMci.VMs { + loadGeneratorServer := LoadGeneratorServer{ + Csp: vm.ConnectionConfig.ProviderName, + Region: vm.Region.Region, + Zone: vm.Region.Zone, + PublicIp: vm.PublicIP, + PrivateIp: vm.PrivateIP, + PublicDns: vm.PublicDNS, + MachineType: vm.CspViewVMDetail.VMSpecName, + Status: vm.Status, + SshPort: vm.SSHPort, + Lat: fmt.Sprintf("%f", vm.Location.Latitude), + Lon: fmt.Sprintf("%f", vm.Location.Longitude), + Username: vm.CspViewVMDetail.VMUserID, + VmId: vm.CspViewVMDetail.IID.SystemID, + StartTime: vm.CspViewVMDetail.StartTime, + AdditionalVmKey: vm.ID, + Label: "temp-label", + IsCluster: false, + IsMaster: i == 0, + ClusterSize: uint64(len(antMci.VMs)), + } + + loadGeneratorServers = append(loadGeneratorServers, loadGeneratorServer) + } + + loadGeneratorInstallInfo.LoadGeneratorServers = loadGeneratorServers + loadGeneratorInstallInfo.PublicKeyName = antPubKeyName + loadGeneratorInstallInfo.PrivateKeyName = antPrivKeyName + } + + loadGeneratorInstallInfo.Status = "installed" + err = l.loadRepo.UpdateLoadGeneratorInstallInfoTx(ctx, loadGeneratorInstallInfo) + if err != nil { + utils.LogError("Error updating LoadGeneratorInstallInfo after installed:", err) + return result, err + } + + utils.LogInfo("LoadGeneratorInstallInfo updated successfully") + } + + loadGeneratorServerResults := make([]LoadGeneratorServerResult, 0) + for _, l := range loadGeneratorInstallInfo.LoadGeneratorServers { + lr := LoadGeneratorServerResult{ + ID: l.ID, + Csp: l.Csp, + Region: l.Region, + Zone: l.Zone, + PublicIp: l.PublicIp, + PrivateIp: l.PrivateIp, + PublicDns: l.PublicDns, + MachineType: l.MachineType, + Status: l.Status, + SshPort: l.SshPort, + Lat: l.Lat, + Lon: l.Lon, + Username: l.Username, + VmId: l.VmId, + StartTime: l.StartTime, + AdditionalVmKey: l.AdditionalVmKey, + Label: l.Label, + CreatedAt: l.CreatedAt, + UpdatedAt: l.UpdatedAt, + } + loadGeneratorServerResults = append(loadGeneratorServerResults, lr) + } + + result.ID = loadGeneratorInstallInfo.ID + result.InstallLocation = loadGeneratorInstallInfo.InstallLocation + result.InstallType = loadGeneratorInstallInfo.InstallType + result.InstallPath = loadGeneratorInstallInfo.InstallPath + result.InstallVersion = loadGeneratorInstallInfo.InstallVersion + result.Status = loadGeneratorInstallInfo.Status + result.PublicKeyName = loadGeneratorInstallInfo.PublicKeyName + result.PrivateKeyName = loadGeneratorInstallInfo.PrivateKeyName + result.CreatedAt = loadGeneratorInstallInfo.CreatedAt + result.UpdatedAt = loadGeneratorInstallInfo.UpdatedAt + result.LoadGeneratorServers = loadGeneratorServerResults + + utils.LogInfo("InstallLoadGenerator completed successfully") + + return result, nil +} + +// getAndDefaultMci retrieves or creates the default MCI. +func (l *LoadService) getAndDefaultMci(ctx context.Context, antVmCommonSpec, antVmCommonImage, antVmConnectionName string) (tumblebug.MciRes, error) { + var antMci tumblebug.MciRes + var err error + antMci, err = l.tumblebugClient.GetMciWithContext(ctx, antNsId, antMciId) + if err != nil { + if errors.Is(err, tumblebug.ErrNotFound) { + dynamicMciArg := tumblebug.DynamicMciReq{ + Description: antMciDescription, + InstallMonAgent: antInstallMonAgent, + Label: map[string]string{antLabelKey: antMciLabel}, + Name: antMciId, + SystemLabel: "", + VM: []tumblebug.DynamicVmReq{ + { + CommonImage: antVmCommonImage, + CommonSpec: antVmCommonSpec, + ConnectionName: antVmConnectionName, + Description: antVmDescription, + Label: map[string]string{antLabelKey: antVmLabel}, + Name: antVmName, + RootDiskSize: antVmRootDiskSize, + RootDiskType: antVmRootDiskType, + SubGroupSize: antVmSubGroupSize, + VMUserPassword: antVmUserPassword, + }, + }, + } + antMci, err = l.tumblebugClient.DynamicMciWithContext(ctx, antNsId, dynamicMciArg) + time.Sleep(defaultDelay) + if err != nil { + return antMci, err + } + } else { + return antMci, err + } + } else if antMci.VMs != nil && len(antMci.VMs) == 0 { + + dynamicVmArg := tumblebug.DynamicVmReq{ + CommonImage: antVmCommonImage, + CommonSpec: antVmCommonSpec, + ConnectionName: antVmConnectionName, + Description: antVmDescription, + Label: map[string]string{antLabelKey: antVmLabel}, + Name: antVmName, + RootDiskSize: antVmRootDiskSize, + RootDiskType: antVmRootDiskType, + SubGroupSize: antVmSubGroupSize, + VMUserPassword: antVmUserPassword, + } + + antMci, err = l.tumblebugClient.DynamicVmWithContext(ctx, antNsId, antMciId, dynamicVmArg) + time.Sleep(defaultDelay) + if err != nil { + return antMci, err + } + } + return antMci, nil +} + +// getRecommendVm retrieves recommendVm to specify the location of provisioning. +func (l *LoadService) getRecommendVm(ctx context.Context, coordinates []string) (tumblebug.RecommendVmResList, error) { + recommendVmArg := tumblebug.RecommendVmReq{ + Filter: tumblebug.Filter{ + Policy: []tumblebug.FilterPolicy{ + { + Condition: []tumblebug.Condition{ + { + Operand: "2", + Operator: ">=", + }, + { + Operand: "8", + Operator: "<=", + }, + }, + Metric: "vCPU", + }, + { + Condition: []tumblebug.Condition{ + { + Operand: "4", + Operator: ">=", + }, + { + Operand: "8", + Operator: "<=", + }, + }, + Metric: "memoryGiB", + }, + { + Condition: []tumblebug.Condition{ + { + Operand: "aws", + }, + }, + Metric: "providerName", + }, + }, + }, + Limit: "1", + Priority: tumblebug.Priority{ + Policy: []tumblebug.Policy{ + { + Metric: "location", + Parameter: []tumblebug.Parameter{ + { + Key: "coordinateClose", + Val: coordinates, + }, + }, + }, + }, + }, + } + + recommendRes, err := l.tumblebugClient.GetRecommendVmWithContext(ctx, recommendVmArg) + + if err != nil { + return nil, err + } + + if len(recommendRes) == 0 { + return nil, errors.New("there is no recommended vm list") + } + + return recommendRes, nil +} + +// validDefaultNs checks if the default namespace exists, and creates it if not. +func (l *LoadService) validDefaultNs(ctx context.Context, antNsId string) error { + _, err := l.tumblebugClient.GetNsWithContext(ctx, antNsId) + if err != nil && errors.Is(err, tumblebug.ErrNotFound) { + + arg := tumblebug.CreateNsReq{ + Name: antNsId, + Description: "cm-ant default ns for validating migration", + } + + err = l.tumblebugClient.CreateNsWithContext(ctx, arg) + if err != nil { + return err + } + } else if err != nil { + return err + } + + return nil +} + +func (l *LoadService) UninstallLoadGenerator(param UninstallLoadGeneratorParam) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + loadGeneratorInstallInfo, err := l.loadRepo.GetValidLoadGeneratorInstallInfoByIdTx(ctx, param.LoadGeneratorInstallInfoId) + + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + utils.LogError("Cannot find valid load generator install info:", err) + return errors.New("cannot find valid load generator install info") + } + utils.LogError("Error retrieving load generator install info:", err) + return err + } + + uninstallScriptPath := utils.JoinRootPathWith("/script/uninstall-jmeter.sh") + + switch loadGeneratorInstallInfo.InstallLocation { + case constant.Local: + err := utils.Script(uninstallScriptPath, []string{ + fmt.Sprintf("JMETER_WORK_DIR=%s", config.AppConfig.Load.JMeter.Dir), + fmt.Sprintf("JMETER_VERSION=%s", config.AppConfig.Load.JMeter.Version), + }) + if err != nil { + utils.LogErrorf("Error while uninstalling load generator: %s", err) + return fmt.Errorf("error while uninstalling load generator: %s", err) + } + case constant.Remote: + + uninstallCommand, err := utils.ReadToString(uninstallScriptPath) + if err != nil { + utils.LogError("Error reading uninstall script:", err) + return err + } + + commandReq := tumblebug.SendCommandReq{ + Command: []string{uninstallCommand}, + } + + _, err = l.tumblebugClient.CommandToMciWithContext(ctx, antNsId, antMciId, commandReq) + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + utils.LogError("VM is not in running state. Cannot connect to the VMs.") + return errors.New("vm is not running state. cannot connect to the vms") + } + utils.LogError("Error sending uninstall command to MCI:", err) + return err + } + + // err = l.tumblebugClient.ControlLifecycleWithContext(ctx, antNsId, antMciId, "suspend") + // if err != nil { + // return err + // } + } + + loadGeneratorInstallInfo.Status = "deleted" + for i := range loadGeneratorInstallInfo.LoadGeneratorServers { + loadGeneratorInstallInfo.LoadGeneratorServers[i].Status = "deleted" + } + + err = l.loadRepo.UpdateLoadGeneratorInstallInfoTx(ctx, &loadGeneratorInstallInfo) + if err != nil { + utils.LogError("Error updating load generator install info:", err) + return err + } + + utils.LogInfo("Successfully uninstalled load generator.") + return nil +} + +func (l *LoadService) GetAllLoadGeneratorInstallInfo(param GetAllLoadGeneratorInstallInfoParam) (GetAllLoadGeneratorInstallInfoResult, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var result GetAllLoadGeneratorInstallInfoResult + var infos []LoadGeneratorInstallInfoResult + pagedResult, totalRows, err := l.loadRepo.GetPagingLoadGeneratorInstallInfosTx(ctx, param) + + if err != nil { + utils.LogError("Error fetching paged load generator install infos:", err) + return result, err + } + + for _, l := range pagedResult { + loadGeneratorServerResults := make([]LoadGeneratorServerResult, 0) + for _, s := range l.LoadGeneratorServers { + lsr := LoadGeneratorServerResult{ + ID: s.ID, + Csp: s.Csp, + Region: s.Region, + Zone: s.Zone, + PublicIp: s.PublicIp, + PrivateIp: s.PrivateIp, + PublicDns: s.PublicDns, + MachineType: s.MachineType, + Status: s.Status, + SshPort: s.SshPort, + Lat: s.Lat, + Lon: s.Lon, + Username: s.Username, + VmId: s.VmId, + StartTime: s.StartTime, + AdditionalVmKey: s.AdditionalVmKey, + Label: s.Label, + CreatedAt: s.CreatedAt, + UpdatedAt: s.UpdatedAt, + } + loadGeneratorServerResults = append(loadGeneratorServerResults, lsr) + } + lr := LoadGeneratorInstallInfoResult{ + ID: l.ID, + InstallLocation: l.InstallLocation, + InstallType: l.InstallType, + InstallPath: l.InstallPath, + InstallVersion: l.InstallVersion, + Status: l.Status, + PublicKeyName: l.PublicKeyName, + PrivateKeyName: l.PrivateKeyName, + CreatedAt: l.CreatedAt, + UpdatedAt: l.UpdatedAt, + LoadGeneratorServers: loadGeneratorServerResults, + } + + infos = append(infos, lr) + } + + result.LoadGeneratorInstallInfoResults = infos + result.TotalRows = totalRows + utils.LogInfof("Fetched %d load generator install info results.", len(infos)) + + return result, nil +} diff --git a/internal/core/load/load_test_execution_service.go b/internal/core/load/load_test_execution_service.go new file mode 100644 index 0000000..5233818 --- /dev/null +++ b/internal/core/load/load_test_execution_service.go @@ -0,0 +1,360 @@ +package load + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/cloud-barista/cm-ant/internal/core/common/constant" + "github.com/cloud-barista/cm-ant/internal/infra/outbound/tumblebug" + "github.com/cloud-barista/cm-ant/internal/utils" +) + +// RunLoadTest initiates the load test and performs necessary initializations. +// Generates a load test key, installs the load generator or retrieves existing installation information, +// saves the load test execution state, and then asynchronously runs the load test. +func (l *LoadService) RunLoadTest(param RunLoadTestParam) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + loadTestKey := utils.CreateUniqIdBaseOnUnixTime() + param.LoadTestKey = loadTestKey + + utils.LogInfof("Starting load test with key: %s", loadTestKey) + + if param.LoadGeneratorInstallInfoId == uint(0) { + utils.LogInfo("No LoadGeneratorInstallInfoId provided, installing load generator...") + result, err := l.InstallLoadGenerator(param.InstallLoadGenerator) + if err != nil { + utils.LogErrorf("Error installing load generator: %v", err) + return "", err + } + + param.LoadGeneratorInstallInfoId = result.ID + utils.LogInfof("Load generator installed with ID: %d", result.ID) + } + + if param.LoadGeneratorInstallInfoId == uint(0) { + utils.LogErrorf("LoadGeneratorInstallInfoId is still 0 after installation.") + return "", nil + } + + utils.LogInfof("Retrieving load generator installation info with ID: %d", param.LoadGeneratorInstallInfoId) + loadGeneratorInstallInfo, err := l.loadRepo.GetValidLoadGeneratorInstallInfoByIdTx(ctx, param.LoadGeneratorInstallInfoId) + if err != nil { + utils.LogErrorf("Error retrieving load generator installation info: %v", err) + return "", err + } + + duration, err := strconv.Atoi(param.Duration) + if err != nil { + return "", err + } + + rampUpTime, err := strconv.Atoi(param.RampUpTime) + + if err != nil { + return "", err + } + + stateArg := LoadTestExecutionState{ + LoadGeneratorInstallInfoId: loadGeneratorInstallInfo.ID, + LoadTestKey: loadTestKey, + ExecutionStatus: constant.OnPreparing, + StartAt: time.Now(), + TotalExpectedExcutionSecond: uint64(duration + rampUpTime), + } + + go l.processLoadTest(param, &loadGeneratorInstallInfo, &stateArg) + + var hs []LoadTestExecutionHttpInfo + + for _, h := range param.HttpReqs { + hh := LoadTestExecutionHttpInfo{ + Method: h.Method, + Protocol: h.Protocol, + Hostname: h.Hostname, + Port: h.Port, + Path: h.Path, + BodyData: h.BodyData, + } + + hs = append(hs, hh) + } + + loadArg := LoadTestExecutionInfo{ + LoadTestKey: loadTestKey, + TestName: param.TestName, + VirtualUsers: param.VirtualUsers, + Duration: param.Duration, + RampUpTime: param.RampUpTime, + RampUpSteps: param.RampUpSteps, + Hostname: param.Hostname, + Port: param.Port, + AgentInstalled: param.AgentInstalled, + AgentHostname: param.AgentHostname, + LoadGeneratorInstallInfoId: loadGeneratorInstallInfo.ID, + LoadTestExecutionHttpInfos: hs, + } + + utils.LogInfof("Saving load test execution info for key: %s", loadTestKey) + err = l.loadRepo.SaveForLoadTestExecutionTx(ctx, &loadArg, &stateArg) + if err != nil { + utils.LogErrorf("Error saving load test execution info: %v", err) + return "", err + } + + utils.LogInfof("Load test started successfully with key: %s", loadTestKey) + + return loadTestKey, nil + +} + +// processLoadTest executes the load test. +// Depending on whether the installation location is local or remote, it creates the test plan and runs test commands. +// Fetches and saves test results from the local or remote system. +func (l *LoadService) processLoadTest(param RunLoadTestParam, loadGeneratorInstallInfo *LoadGeneratorInstallInfo, loadTestExecutionState *LoadTestExecutionState) { + + loadTestDone := make(chan bool) + + var username string + var publicIp string + var port string + for _, s := range loadGeneratorInstallInfo.LoadGeneratorServers { + if s.IsMaster { + username = s.Username + publicIp = s.PublicIp + port = s.SshPort + } + } + + home, err := os.UserHomeDir() + if err != nil { + return + } + + dataParam := &fetchDataParam{ + LoadTestDone: loadTestDone, + LoadTestKey: param.LoadTestKey, + InstallLocation: loadGeneratorInstallInfo.InstallLocation, + InstallPath: loadGeneratorInstallInfo.InstallPath, + PublicKeyName: loadGeneratorInstallInfo.PublicKeyName, + PrivateKeyName: loadGeneratorInstallInfo.PrivateKeyName, + Username: username, + PublicIp: publicIp, + Port: port, + AgentInstalled: param.AgentInstalled, + Home: home, + } + + go l.fetchData(dataParam) + + defer func() { + loadTestDone <- true + close(loadTestDone) + updateErr := l.loadRepo.UpdateLoadTestExecutionStateTx(context.Background(), loadTestExecutionState) + if updateErr != nil { + utils.LogErrorf("Error updating load test execution state: %v", updateErr) + return + } + }() + + compileDuration, executionDuration, loadTestErr := l.executeLoadTest(param, loadGeneratorInstallInfo) + + loadTestExecutionState.CompileDuration = compileDuration + loadTestExecutionState.ExecutionDuration = executionDuration + + if loadTestErr != nil { + loadTestExecutionState.ExecutionStatus = constant.TestFailed + loadTestExecutionState.FailureMessage = loadTestErr.Error() + finishAt := time.Now() + loadTestExecutionState.FinishAt = &finishAt + return + } + + updateErr := l.updateLoadTestExecution(loadTestExecutionState) + if updateErr != nil { + loadTestExecutionState.ExecutionStatus = constant.UpdateFailed + loadTestExecutionState.FailureMessage = updateErr.Error() + finishAt := time.Now() + loadTestExecutionState.FinishAt = &finishAt + return + } + + loadTestExecutionState.ExecutionStatus = constant.Successed +} + +func (l *LoadService) executeLoadTest(param RunLoadTestParam, loadGeneratorInstallInfo *LoadGeneratorInstallInfo) (string, string, error) { + installLocation := loadGeneratorInstallInfo.InstallLocation + loadTestKey := param.LoadTestKey + loadGeneratorInstallPath := loadGeneratorInstallInfo.InstallPath + testPlanName := fmt.Sprintf("%s.jmx", loadTestKey) + resultFileName := fmt.Sprintf("%s_result.csv", loadTestKey) + loadGeneratorInstallVersion := loadGeneratorInstallInfo.InstallVersion + + utils.LogInfof("Running load test with key: %s", loadTestKey) + compileDuration := "0" + executionDuration := "0" + start := time.Now() + + if installLocation == constant.Remote { + utils.LogInfo("Remote execute detected.") + var buf bytes.Buffer + err := parseTestPlanStructToString(&buf, param, loadGeneratorInstallInfo) + if err != nil { + return compileDuration, executionDuration, err + } + + testPlan := buf.String() + + createFileCmd := fmt.Sprintf("cat << 'EOF' > %s/test_plan/%s \n%s\nEOF", loadGeneratorInstallPath, testPlanName, testPlan) + + commandReq := tumblebug.SendCommandReq{ + Command: []string{createFileCmd}, + } + + compileDuration = utils.DurationString(start) + _, err = l.tumblebugClient.CommandToMciWithContext(context.Background(), antNsId, antMciId, commandReq) + if err != nil { + return compileDuration, executionDuration, err + } + + jmeterTestCommand := generateJmeterExecutionCmd(loadGeneratorInstallPath, loadGeneratorInstallVersion, testPlanName, resultFileName) + + commandReq = tumblebug.SendCommandReq{ + Command: []string{jmeterTestCommand}, + } + + stdout, err := l.tumblebugClient.CommandToMciWithContext(context.Background(), antNsId, antMciId, commandReq) + if err != nil { + return compileDuration, executionDuration, err + } + executionDuration = utils.DurationString(start) + + if strings.Contains(stdout, "exited with status 1") { + return compileDuration, executionDuration, errors.New("jmeter test stopped unexpectedly") + } + + } else if installLocation == constant.Local { + utils.LogInfo("Local execute detected.") + + exist := utils.ExistCheck(loadGeneratorInstallPath) + + if !exist { + return compileDuration, executionDuration, errors.New("load generator installaion is not validated") + } + + outputFile, err := os.Create(fmt.Sprintf("%s/test_plan/%s.jmx", loadGeneratorInstallPath, loadTestKey)) + if err != nil { + return compileDuration, executionDuration, err + } + + err = parseTestPlanStructToString(outputFile, param, loadGeneratorInstallInfo) + + if err != nil { + return compileDuration, executionDuration, err + } + + jmeterTestCommand := generateJmeterExecutionCmd(loadGeneratorInstallPath, loadGeneratorInstallVersion, testPlanName, resultFileName) + compileDuration = utils.DurationString(start) + + err = utils.InlineCmd(jmeterTestCommand) + executionDuration = utils.DurationString(start) + if err != nil { + return compileDuration, executionDuration, fmt.Errorf("jmeter test stopped unexpectedly; %w", err) + } + } + + return compileDuration, executionDuration, nil +} + +func (l *LoadService) updateLoadTestExecution(loadTestExecutionState *LoadTestExecutionState) error { + err := l.loadRepo.UpdateLoadTestExecutionInfoDuration(context.Background(), loadTestExecutionState.LoadTestKey, loadTestExecutionState.CompileDuration, loadTestExecutionState.ExecutionDuration) + if err != nil { + utils.LogErrorf("Error updating load test execution info: %v", err) + return err + } + + loadTestExecutionState.ExecutionStatus = constant.OnFetching + err = l.loadRepo.UpdateLoadTestExecutionStateTx(context.Background(), loadTestExecutionState) + if err != nil { + utils.LogErrorf("Error updating load test execution state: %v", err) + return err + } + return nil +} + +// generateJmeterExecutionCmd generates the JMeter execution command. +// Constructs a JMeter command string that includes the test plan path and result file path. +func generateJmeterExecutionCmd(loadGeneratorInstallPath, loadGeneratorInstallVersion, testPlanName, resultFileName string) string { + utils.LogInfof("Generating JMeter execution command for test plan: %s, result file: %s", testPlanName, resultFileName) + + var builder strings.Builder + testPath := fmt.Sprintf("%s/test_plan/%s", loadGeneratorInstallPath, testPlanName) + resultPath := fmt.Sprintf("%s/result/%s", loadGeneratorInstallPath, resultFileName) + + builder.WriteString(fmt.Sprintf("%s/apache-jmeter-%s/bin/jmeter.sh", loadGeneratorInstallPath, loadGeneratorInstallVersion)) + builder.WriteString(" -n -f") + builder.WriteString(fmt.Sprintf(" -t=%s", testPath)) + builder.WriteString(fmt.Sprintf(" -l=%s", resultPath)) + + builder.WriteString(fmt.Sprintf(" && sudo rm %s", testPath)) + utils.LogInfof("JMeter execution command generated: %s", builder.String()) + return builder.String() +} + +func (l *LoadService) StopLoadTest(param StopLoadTestParam) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + state, err := l.loadRepo.GetLoadTestExecutionStateTx(ctx, GetLoadTestExecutionStateParam{ + LoadTestKey: param.LoadTestKey, + }) + + if err != nil { + return err + } + + if state.ExecutionStatus == constant.Successed { + return nil + } + + installInfo := state.LoadGeneratorInstallInfo + + killCmd := killCmdGen(param.LoadTestKey) + + if installInfo.InstallLocation == constant.Remote { + + commandReq := tumblebug.SendCommandReq{ + Command: []string{killCmd}, + } + _, err := l.tumblebugClient.CommandToMciWithContext(ctx, antNsId, antMciId, commandReq) + + if err != nil { + return err + } + + } else if installInfo.InstallLocation == constant.Local { + err := utils.InlineCmd(killCmd) + + if err != nil { + log.Println(err) + return err + } + } + + return nil + +} + +func killCmdGen(loadTestKey string) string { + grepRegex := fmt.Sprintf("'\\/bin\\/ApacheJMeter\\.jar.*%s'", loadTestKey) + utils.LogInfof("Generating kill command for load test key: %s", loadTestKey) + return fmt.Sprintf("kill -15 $(ps -ef | grep -E %s | awk '{print $2}')", grepRegex) +} diff --git a/internal/core/load/metrics_agent_service.go b/internal/core/load/metrics_agent_service.go new file mode 100644 index 0000000..4dd7e9c --- /dev/null +++ b/internal/core/load/metrics_agent_service.go @@ -0,0 +1,223 @@ +package load + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/cloud-barista/cm-ant/internal/infra/outbound/tumblebug" + "github.com/cloud-barista/cm-ant/internal/utils" +) + +// InstallMonitoringAgent installs a monitoring agent on specified VMs or all VM on mci. +func (l *LoadService) InstallMonitoringAgent(param MonitoringAgentInstallationParams) ([]MonitoringAgentInstallationResult, error) { + utils.LogInfo("Starting installation of monitoring agent...") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var res []MonitoringAgentInstallationResult + + scriptPath := utils.JoinRootPathWith("/script/install-server-agent.sh") + utils.LogInfof("Reading installation script from %s", scriptPath) + installScript, err := os.ReadFile(scriptPath) + if err != nil { + utils.LogErrorf("Failed to read installation script: %v", err) + return res, err + } + username := "cb-user" + + utils.LogInfof("Fetching mci object for NS: %s, msi id: %s", param.NsId, param.MciId) + mci, err := l.tumblebugClient.GetMciWithContext(ctx, param.NsId, param.MciId) + if err != nil { + utils.LogErrorf("Failed to fetch mci : %v", err) + return res, err + } + + if len(mci.VMs) == 0 { + utils.LogErrorf("No VMs found on mci. Provision VM first.") + return res, errors.New("there is no vm on mci. provision vm first") + } + + var mapSet map[string]struct{} + if len(param.VmIds) > 0 { + mapSet = utils.SliceToMap(param.VmIds) + } + + var errorCollection []error + + for _, vm := range mci.VMs { + if mapSet != nil && !utils.Contains(mapSet, vm.ID) { + continue + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + m := MonitoringAgentInfo{ + Username: username, + Status: "installing", + AgentType: "perfmon", + NsId: param.NsId, + MciId: param.MciId, + VmId: vm.ID, + VmCount: len(mci.VMs), + } + utils.LogInfof("Inserting monitoring agent installation info into database vm id : %s", vm.ID) + err = l.loadRepo.InsertMonitoringAgentInfoTx(ctx, &m) + if err != nil { + utils.LogErrorf("Failed to insert monitoring agent info for vm id %s : %v", vm.ID, err) + errorCollection = append(errorCollection, err) + continue + } + + commandReq := tumblebug.SendCommandReq{ + Command: []string{string(installScript)}, + UserName: username, + } + + utils.LogInfof("Sending install command to mci. NS: %s, mci: %s, VMID: %s", param.NsId, param.MciId, vm.ID) + _, err = l.tumblebugClient.CommandToVmWithContext(ctx, param.NsId, param.MciId, vm.ID, commandReq) + + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + m.Status = "timeout" + utils.LogErrorf("Timeout of context. Already 15 seconds has been passed. vm id : %s", vm.ID) + } else { + m.Status = "failed" + utils.LogErrorf("Error occurred during command execution: %v", err) + } + errorCollection = append(errorCollection, err) + } else { + m.Status = "completed" + } + + l.loadRepo.UpdateAgentInstallInfoStatusTx(ctx, &m) + + r := MonitoringAgentInstallationResult{ + ID: m.ID, + NsId: m.NsId, + MciId: m.MciId, + VmId: m.VmId, + VmCount: m.VmCount, + Status: m.Status, + Username: m.Username, + AgentType: m.AgentType, + CreatedAt: m.CreatedAt, + UpdatedAt: m.UpdatedAt, + } + + res = append(res, r) + utils.LogInfof( + "Complete installing monitoring agent on mics: %s, vm: %s", + m.MciId, + m.VmId, + ) + + time.Sleep(time.Second) + } + + if len(errorCollection) > 0 { + return res, fmt.Errorf("multiple errors: %v", errorCollection) + } + + return res, nil +} + +func (l *LoadService) GetAllMonitoringAgentInfos(param GetAllMonitoringAgentInfosParam) (GetAllMonitoringAgentInfoResult, error) { + var res GetAllMonitoringAgentInfoResult + var monitoringAgentInfos []MonitoringAgentInstallationResult + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + utils.LogInfof("GetAllMonitoringAgentInfos called with param: %+v", param) + result, totalRows, err := l.loadRepo.GetPagingMonitoringAgentInfosTx(ctx, param) + + if err != nil { + utils.LogErrorf("Error fetching monitoring agent infos: %v", err) + return res, err + } + + utils.LogInfof("Fetched %d monitoring agent infos", len(result)) + + for _, monitoringAgentInfo := range result { + var r MonitoringAgentInstallationResult + r.ID = monitoringAgentInfo.ID + r.NsId = monitoringAgentInfo.NsId + r.MciId = monitoringAgentInfo.MciId + r.VmId = monitoringAgentInfo.VmId + r.VmCount = monitoringAgentInfo.VmCount + r.Status = monitoringAgentInfo.Status + r.Username = monitoringAgentInfo.Username + r.AgentType = monitoringAgentInfo.AgentType + r.CreatedAt = monitoringAgentInfo.CreatedAt + r.UpdatedAt = monitoringAgentInfo.UpdatedAt + monitoringAgentInfos = append(monitoringAgentInfos, r) + } + + res.MonitoringAgentInfos = monitoringAgentInfos + res.TotalRow = totalRows + + return res, nil +} + +// UninstallMonitoringAgent uninstalls a monitoring agent on specified VMs or all VM on Mci. +// It takes MonitoringAgentInstallationParams as input and returns the number of affected results and any encountered error. +func (l *LoadService) UninstallMonitoringAgent(param MonitoringAgentInstallationParams) (int64, error) { + + ctx := context.Background() + var effectedResults int64 + + utils.LogInfo("Starting uninstallation of monitoring agent...") + result, err := l.loadRepo.GetAllMonitoringAgentInfosTx(ctx, param) + + if err != nil { + utils.LogErrorf("Failed to fetch monitoring agent information: %v", err) + return effectedResults, err + } + + scriptPath := utils.JoinRootPathWith("/script/remove-server-agent.sh") + utils.LogInfof("Reading uninstallation script from %s", scriptPath) + + uninstallPath, err := os.ReadFile(scriptPath) + if err != nil { + utils.LogErrorf("Failed to read uninstallation script: %v", err) + return effectedResults, err + } + + username := "cb-user" + + var errorCollection []error + for _, monitoringAgentInfo := range result { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + commandReq := tumblebug.SendCommandReq{ + Command: []string{string(uninstallPath)}, + UserName: username, + } + + _, err = l.tumblebugClient.CommandToVmWithContext(ctx, monitoringAgentInfo.NsId, monitoringAgentInfo.MciId, monitoringAgentInfo.VmId, commandReq) + if err != nil { + errorCollection = append(errorCollection, err) + utils.LogErrorf("Failed to uninstall monitoring agent on mci: %s, VM: %s - Error: %v", monitoringAgentInfo.MciId, monitoringAgentInfo.VmId, err) + continue + } + + err = l.loadRepo.DeleteAgentInstallInfoStatusTx(ctx, &monitoringAgentInfo) + + if err != nil { + utils.LogErrorf("Failed to delete agent installation status for mci: %s, VM: %s - Error: %v", monitoringAgentInfo.MciId, monitoringAgentInfo.VmId, err) + errorCollection = append(errorCollection, err) + continue + } + + utils.LogInfof("Successfully uninstalled monitoring agent on mci: %s, VM: %s", monitoringAgentInfo.MciId, monitoringAgentInfo.VmId) + + time.Sleep(time.Second) + } + + if len(errorCollection) > 0 { + return effectedResults, fmt.Errorf("multiple errors: %v", errorCollection) + } + + return effectedResults, nil +} diff --git a/internal/core/load/performace_evaluation_service.go b/internal/core/load/performace_evaluation_service.go new file mode 100644 index 0000000..0bc81f1 --- /dev/null +++ b/internal/core/load/performace_evaluation_service.go @@ -0,0 +1,234 @@ +package load + +import ( + "context" + "time" + + "github.com/cloud-barista/cm-ant/internal/infra/outbound/tumblebug" + "github.com/cloud-barista/cm-ant/internal/utils" +) + +// LoadService represents a service for managing load operations. +type LoadService struct { + loadRepo *LoadRepository + tumblebugClient *tumblebug.TumblebugClient +} + +// NewLoadService creates a new instance of LoadService. +func NewLoadService(loadRepo *LoadRepository, client *tumblebug.TumblebugClient) *LoadService { + return &LoadService{ + loadRepo: loadRepo, + tumblebugClient: client, + } +} + +func (l *LoadService) Readyz() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + sqlDB, err := l.loadRepo.db.DB() + if err != nil { + return err + } + + err = sqlDB.Ping() + if err != nil { + return err + } + + err = l.tumblebugClient.ReadyzWithContext(ctx) + if err != nil { + return err + } + + return nil +} + +func (l *LoadService) GetAllLoadTestExecutionState(param GetAllLoadTestExecutionStateParam) (GetAllLoadTestExecutionStateResult, error) { + var res GetAllLoadTestExecutionStateResult + var states []LoadTestExecutionStateResult + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + utils.LogInfof("GetAllLoadExecutionStates called with param: %+v", param) + result, totalRows, err := l.loadRepo.GetPagingLoadTestExecutionStateTx(ctx, param) + + if err != nil { + utils.LogErrorf("Error fetching load test execution state infos: %v", err) + return res, err + } + + utils.LogInfof("Fetched %d monitoring agent infos", len(result)) + + for _, loadTestExecutionState := range result { + state := mapLoadTestExecutionStateResult(loadTestExecutionState) + state.LoadGeneratorInstallInfo = mapLoadGeneratorInstallInfoResult(loadTestExecutionState.LoadGeneratorInstallInfo) + states = append(states, state) + } + + res.LoadTestExecutionStates = states + res.TotalRow = totalRows + + return res, nil +} + +func (l *LoadService) GetLoadTestExecutionState(param GetLoadTestExecutionStateParam) (LoadTestExecutionStateResult, error) { + var res LoadTestExecutionStateResult + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + utils.LogInfof("GetLoadTestExecutionState called with param: %+v", param) + state, err := l.loadRepo.GetLoadTestExecutionStateTx(ctx, param) + + if err != nil { + utils.LogErrorf("Error fetching load test execution state infos: %v", err) + return res, err + } + + res = mapLoadTestExecutionStateResult(state) + res.LoadGeneratorInstallInfo = mapLoadGeneratorInstallInfoResult(state.LoadGeneratorInstallInfo) + return res, nil +} + +func (l *LoadService) GetAllLoadTestExecutionInfos(param GetAllLoadTestExecutionInfosParam) (GetAllLoadTestExecutionInfosResult, error) { + var res GetAllLoadTestExecutionInfosResult + var rs []LoadTestExecutionInfoResult + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + utils.LogInfof("GetAllLoadTestExecutionInfos called with param: %+v", param) + result, totalRows, err := l.loadRepo.GetPagingLoadTestExecutionHistoryTx(ctx, param) + + if err != nil { + utils.LogErrorf("Error fetching load test execution infos: %v", err) + return res, err + } + + utils.LogInfof("Fetched %d load test execution infos:", len(result)) + + for _, r := range result { + rs = append(rs, mapLoadTestExecutionInfoResult(r)) + } + + res.TotalRow = totalRows + res.LoadTestExecutionInfos = rs + + return res, nil +} + +func (l *LoadService) GetLoadTestExecutionInfo(param GetLoadTestExecutionInfoParam) (LoadTestExecutionInfoResult, error) { + var res LoadTestExecutionInfoResult + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + utils.LogInfof("GetLoadTestExecutionInfo called with param: %+v", param) + executionInfo, err := l.loadRepo.GetLoadTestExecutionInfoTx(ctx, param) + + if err != nil { + utils.LogErrorf("Error fetching load test execution state infos: %v", err) + return res, err + } + + return mapLoadTestExecutionInfoResult(executionInfo), nil +} + +func mapLoadTestExecutionHttpInfoResult(h LoadTestExecutionHttpInfo) LoadTestExecutionHttpInfoResult { + return LoadTestExecutionHttpInfoResult{ + ID: h.ID, + Method: h.Method, + Protocol: h.Protocol, + Hostname: h.Hostname, + Port: h.Port, + Path: h.Path, + BodyData: h.BodyData, + } +} + +func mapLoadTestExecutionStateResult(state LoadTestExecutionState) LoadTestExecutionStateResult { + return LoadTestExecutionStateResult{ + ID: state.ID, + LoadTestKey: state.LoadTestKey, + ExecutionStatus: state.ExecutionStatus, + StartAt: state.StartAt, + FinishAt: state.FinishAt, + TotalExpectedExcutionSecond: state.TotalExpectedExcutionSecond, + FailureMessage: state.FailureMessage, + CompileDuration: state.CompileDuration, + ExecutionDuration: state.ExecutionDuration, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +func mapLoadGeneratorServerResult(s LoadGeneratorServer) LoadGeneratorServerResult { + return LoadGeneratorServerResult{ + ID: s.ID, + Csp: s.Csp, + Region: s.Region, + Zone: s.Zone, + PublicIp: s.PublicIp, + PrivateIp: s.PrivateIp, + PublicDns: s.PublicDns, + MachineType: s.MachineType, + Status: s.Status, + SshPort: s.SshPort, + Lat: s.Lat, + Lon: s.Lon, + Username: s.Username, + VmId: s.VmId, + StartTime: s.StartTime, + AdditionalVmKey: s.AdditionalVmKey, + Label: s.Label, + CreatedAt: s.CreatedAt, + } +} + +func mapLoadGeneratorInstallInfoResult(install LoadGeneratorInstallInfo) LoadGeneratorInstallInfoResult { + var servers []LoadGeneratorServerResult + for _, s := range install.LoadGeneratorServers { + servers = append(servers, mapLoadGeneratorServerResult(s)) + } + + return LoadGeneratorInstallInfoResult{ + ID: install.ID, + InstallLocation: install.InstallLocation, + InstallType: install.InstallType, + InstallPath: install.InstallPath, + InstallVersion: install.InstallVersion, + Status: install.Status, + CreatedAt: install.CreatedAt, + UpdatedAt: install.UpdatedAt, + PublicKeyName: install.PublicKeyName, + PrivateKeyName: install.PrivateKeyName, + LoadGeneratorServers: servers, + } +} + +func mapLoadTestExecutionInfoResult(executionInfo LoadTestExecutionInfo) LoadTestExecutionInfoResult { + var httpResults []LoadTestExecutionHttpInfoResult + for _, h := range executionInfo.LoadTestExecutionHttpInfos { + httpResults = append(httpResults, mapLoadTestExecutionHttpInfoResult(h)) + } + + executionState := mapLoadTestExecutionStateResult(executionInfo.LoadTestExecutionState) + installInfo := mapLoadGeneratorInstallInfoResult(executionInfo.LoadGeneratorInstallInfo) + + return LoadTestExecutionInfoResult{ + ID: executionInfo.ID, + LoadTestKey: executionInfo.LoadTestKey, + TestName: executionInfo.TestName, + VirtualUsers: executionInfo.VirtualUsers, + Duration: executionInfo.Duration, + RampUpTime: executionInfo.RampUpTime, + RampUpSteps: executionInfo.RampUpSteps, + Hostname: executionInfo.Hostname, + Port: executionInfo.Port, + AgentHostname: executionInfo.AgentHostname, + AgentInstalled: executionInfo.AgentInstalled, + CompileDuration: executionInfo.CompileDuration, + ExecutionDuration: executionInfo.ExecutionDuration, + LoadTestExecutionHttpInfos: httpResults, + LoadTestExecutionState: executionState, + LoadGeneratorInstallInfo: installInfo, + } +} diff --git a/internal/core/load/performance_evaluation_result_service.go b/internal/core/load/performance_evaluation_result_service.go new file mode 100644 index 0000000..aaeba11 --- /dev/null +++ b/internal/core/load/performance_evaluation_result_service.go @@ -0,0 +1,414 @@ +package load + +import ( + "errors" + "fmt" + "log" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/cloud-barista/cm-ant/internal/core/common/constant" + "github.com/cloud-barista/cm-ant/internal/utils" +) + +type metricsUnits struct { + Multiple float64 + Unit string +} + +var tags = map[string]metricsUnits{ + "cpu_all_combined": { + Multiple: 0.001, + Unit: "%", + }, + "cpu_all_idle": { + Multiple: 0.001, + Unit: "%", + }, + "memory_all_used": { + Multiple: 0.001, + Unit: "%", + }, + "memory_all_free": { + Multiple: 0.001, + Unit: "%", + }, + "memory_all_used_kb": { + Multiple: 0.000001, + Unit: "mb", + }, + "memory_all_free_kb": { + Multiple: 0.000001, + Unit: "mb", + }, + "disk_read_kb": { + Multiple: 0.001, + Unit: "kb", + }, + "disk_write_kb": { + Multiple: 0.001, + Unit: "kb", + }, + "disk_use": { + Multiple: 0.001, + Unit: "%", + }, + "disk_total": { + Multiple: 0.000001, + Unit: "mb", + }, + "network_recv_kb": { + Multiple: 0.000001, + Unit: "kb", + }, + "network_sent_kb": { + Multiple: 0.001, + Unit: "kb", + }, +} + +func (l *LoadService) GetLoadTestResult(param GetLoadTestResultParam) (interface{}, error) { + loadTestKey := param.LoadTestKey + fileName := fmt.Sprintf("%s_result.csv", loadTestKey) + resultFolderPath := utils.JoinRootPathWith("/result/" + loadTestKey) + toFilePath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) + resultMap, err := appendResultRawData(toFilePath) + if err != nil { + return nil, err + } + + var resultSummaries []ResultSummary + + for label, results := range resultMap { + resultSummaries = append(resultSummaries, ResultSummary{ + Label: label, + Results: results, + }) + } + + formattedDate, err := resultFormat(param.Format, resultSummaries) + + if err != nil { + return nil, err + } + + return formattedDate, nil +} + +func (l *LoadService) GetLoadTestMetrics(param GetLoadTestResultParam) ([]MetricsSummary, error) { + loadTestKey := param.LoadTestKey + metrics := []string{"cpu", "disk", "memory", "network"} + resultFolderPath := utils.JoinRootPathWith("/result/" + loadTestKey) + + metricsMap := make(map[string][]*MetricsRawData) + var err error + for _, v := range metrics { + + fileName := fmt.Sprintf("%s_%s_result.csv", loadTestKey, v) + toPath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) + + metricsMap, err = appendMetricsRawData(metricsMap, toPath) + if err != nil { + return nil, err + } + + } + + var metricsSummaries []MetricsSummary + + for label, metrics := range metricsMap { + metricsSummaries = append(metricsSummaries, MetricsSummary{ + Label: label, + Metrics: metrics, + }) + } + + if err != nil { + return nil, err + } + + return metricsSummaries, nil +} + +func calculatePercentile(elapsedList []int, percentile float64) float64 { + index := int(math.Ceil(float64(len(elapsedList))*percentile)) - 1 + + return float64(elapsedList[index]) +} + +func calculateMedian(data []int) float64 { + n := len(data) + if n%2 == 0 { + return float64(data[n/2-1]+data[n/2]) / 2 + } + return float64(data[n/2]) +} + +func findMin(elapsedList []int) float64 { + if len(elapsedList) == 0 { + return 0 + } + + return float64(elapsedList[0]) +} + +func findMax(elapsedList []int) float64 { + if len(elapsedList) == 0 { + return 0 + } + + return float64(elapsedList[len(elapsedList)-1]) +} + +func calculateErrorPercent(errorCount, requestCount int) float64 { + if requestCount == 0 { + return 0 + } + errorPercent := float64(errorCount) / float64(requestCount) * 100 + return errorPercent +} + +func calculateThroughput(totalRequests int, totalMillTime int) float64 { + return float64(totalRequests) / (float64(totalMillTime)) * 1000 +} + +func calculateReceivedKBPerSec(totalBytes int, totalMillTime int) float64 { + return (float64(totalBytes) / 1024) / (float64(totalMillTime)) * 1000 +} + +func calculateSentKBPerSec(totalBytes int, totalMillTime int) float64 { + return (float64(totalBytes) / 1024) / (float64(totalMillTime)) * 1000 +} + +func appendResultRawData(filePath string) (map[string][]*ResultRawData, error) { + var resultMap = make(map[string][]*ResultRawData) + + csvRows, err := utils.ReadCSV(filePath) + if err != nil || csvRows == nil { + return nil, err + } + + if len(*csvRows) <= 1 { + return nil, errors.New("result data file is empty") + } + // every time is basically millisecond + for i, row := range (*csvRows)[1:] { + label := row[2] + + elapsed, err := strconv.Atoi(row[1]) + if err != nil { + log.Printf("[%d] elapsed has error %s\n", i, err) + continue + } + bytes, err := strconv.Atoi(row[9]) + if err != nil { + log.Printf("[%d] bytes has error %s\n", i, err) + continue + } + sentBytes, err := strconv.Atoi(row[10]) + if err != nil { + log.Printf("[%d] sentBytes has error %s\n", i, err) + continue + } + latency, err := strconv.Atoi(row[14]) + if err != nil { + log.Printf("[%d] latency has error %s\n", i, err) + continue + } + idleTime, err := strconv.Atoi(row[15]) + if err != nil { + log.Printf("[%d] idleTime has error %s\n", i, err) + continue + } + connection, err := strconv.Atoi(row[16]) + if err != nil { + log.Printf("[%d] connection has error %s\n", i, err) + continue + } + unixMilliseconds, err := strconv.ParseInt(row[0], 10, 64) + if err != nil { + log.Printf("[%d] time has error %s\n", i, err) + continue + } + + isError := row[7] == "false" + url := row[13] + t := time.UnixMilli(unixMilliseconds) + + tr := &ResultRawData{ + No: i, + Elapsed: elapsed, + Bytes: bytes, + SentBytes: sentBytes, + IsError: isError, + URL: url, + Latency: latency, + IdleTime: idleTime, + Connection: connection, + Timestamp: t, + } + if _, ok := resultMap[label]; !ok { + resultMap[label] = []*ResultRawData{tr} + } else { + resultMap[label] = append(resultMap[label], tr) + } + } + + return resultMap, nil +} + +func appendMetricsRawData(mrds map[string][]*MetricsRawData, filePath string) (map[string][]*MetricsRawData, error) { + csvRows, err := utils.ReadCSV(filePath) + if err != nil || csvRows == nil { + return nil, err + } + + if len(*csvRows) <= 1 { + return nil, errors.New("metrics data file is empty") + } + + for i, row := range (*csvRows)[1:] { + isError := row[7] == "false" + intValue, err := strconv.Atoi(row[1]) + if err != nil { + log.Printf("[%d] value has error %s\n", i, err) + continue + } + + var label string + var value string + var u string + + if isError { + label = row[2] + } else { + words := strings.Split(row[2], " ") + label = words[len(words)-1] + + unit, ok := tags[label] + if !ok { + continue + } + + floatValue := float64(intValue) * unit.Multiple + value = strconv.FormatFloat(floatValue, 'f', 3, 64) + u = unit.Unit + } + + unixMilliseconds, err := strconv.ParseInt(row[0], 10, 64) + if err != nil { + log.Printf("[%d] time has error %s\n", i, err) + continue + } + + t := time.UnixMilli(unixMilliseconds) + + rd := &MetricsRawData{ + Value: value, + Unit: u, + IsError: isError, + Timestamp: t, + } + + if _, ok := mrds[label]; !ok { + mrds[label] = []*MetricsRawData{rd} + } else { + mrds[label] = append(mrds[label], rd) + } + } + + return mrds, nil +} + +func aggregate(resultRawDatas []ResultSummary) []*LoadTestStatistics { + var statistics []*LoadTestStatistics + + for i := range resultRawDatas { + + record := resultRawDatas[i] + var requestCount, totalElapsed, totalBytes, totalSentBytes, errorCount int + var elapsedList []int + if len(record.Results) < 1 { + continue + } + + startTime := record.Results[0].Timestamp + endTime := record.Results[0].Timestamp + for _, record := range record.Results { + requestCount++ + if !record.IsError { + totalElapsed += record.Elapsed + } + + totalBytes += record.Bytes + totalSentBytes += record.SentBytes + + if record.IsError { + errorCount++ + } + + if record.Timestamp.Before(startTime) { + startTime = record.Timestamp + } + if record.Timestamp.After(endTime) { + endTime = record.Timestamp + } + + elapsedList = append(elapsedList, record.Elapsed) + } + + // total Elapsed time and running time is different + runningTime := endTime.Sub(startTime).Milliseconds() + + // for percentile calculation purpose + sort.Ints(elapsedList) + + average := float64(totalElapsed) / float64(requestCount) + median := calculateMedian(elapsedList) + ninetyPercent := calculatePercentile(elapsedList, 0.9) + ninetyFive := calculatePercentile(elapsedList, 0.95) + ninetyNine := calculatePercentile(elapsedList, 0.99) + calcMin := findMin(elapsedList) + calcMax := findMax(elapsedList) + errorPercent := calculateErrorPercent(errorCount, requestCount) + throughput := calculateThroughput(requestCount, int(runningTime)) + receivedKB := calculateReceivedKBPerSec(totalBytes, int(runningTime)) + sentKB := calculateSentKBPerSec(totalSentBytes, int(runningTime)) + + labelStat := LoadTestStatistics{ + Label: record.Label, + RequestCount: requestCount, + Average: average, + Median: median, + NinetyPercent: ninetyPercent, + NinetyFive: ninetyFive, + NinetyNine: ninetyNine, + MinTime: calcMin, + MaxTime: calcMax, + ErrorPercent: errorPercent, + Throughput: throughput, + ReceivedKB: receivedKB, + SentKB: sentKB, + } + + statistics = append(statistics, &labelStat) + } + + return statistics +} + +func resultFormat(format constant.ResultFormat, resultSummaries []ResultSummary) (any, error) { + if resultSummaries == nil { + return nil, nil + } + + switch format { + case constant.Aggregate: + return aggregate(resultSummaries), nil + } + + return resultSummaries, nil +} diff --git a/internal/core/load/repository.go b/internal/core/load/repository.go index 903f01f..e4e94f8 100644 --- a/internal/core/load/repository.go +++ b/internal/core/load/repository.go @@ -138,25 +138,35 @@ func (r *LoadRepository) GetAllMonitoringAgentInfosTx(ctx context.Context, param return monitoringAgentInfos, err } -func (r *LoadRepository) InsertLoadGeneratorInstallInfoTx(ctx context.Context, param *LoadGeneratorInstallInfo) error { +func (r *LoadRepository) GetOrInsertLoadGeneratorInstallInfoTx(ctx context.Context, param *LoadGeneratorInstallInfo) error { err := r.execInTransaction(ctx, func(d *gorm.DB) error { + var existing LoadGeneratorInstallInfo + err := d. - Where( - "install_location = ? AND install_type = ? AND install_path = ? AND install_version = ? AND status = ?", - param.InstallLocation, param.InstallType, param.InstallPath, param.InstallVersion, "installed", - ). - // Omit("LoadGeneratorServers"). - FirstOrCreate(param).Error + Preload("LoadGeneratorServers"). + Where("install_location = ? AND status = ?", param.InstallLocation, "installed"). + First(&existing). + Error - if err != nil { + if err == gorm.ErrRecordNotFound { + err = d.Create(param).Error + if err != nil { + return err + } + } else if err != nil { return err + } else { + *param = existing } return nil }) - return err + if err != nil { + return err + } + return nil } func (r *LoadRepository) UpdateLoadGeneratorInstallInfoTx(ctx context.Context, param *LoadGeneratorInstallInfo) error { diff --git a/internal/core/load/rsync_fetch_service.go b/internal/core/load/rsync_fetch_service.go new file mode 100644 index 0000000..e199cc9 --- /dev/null +++ b/internal/core/load/rsync_fetch_service.go @@ -0,0 +1,150 @@ +package load + +import ( + "fmt" + "log" + "sync" + "time" + + "github.com/cloud-barista/cm-ant/internal/core/common/constant" + "github.com/cloud-barista/cm-ant/internal/utils" +) + +type fetchDataParam struct { + LoadTestDone <-chan bool + LoadTestKey string + InstallLocation constant.InstallLocation + InstallPath string + PublicKeyName string + PrivateKeyName string + Username string + PublicIp string + Port string + AgentInstalled bool + fetchMx sync.Mutex + fetchRunning bool + Home string +} + +func (f *fetchDataParam) setFetchRunning(running bool) { + f.fetchMx.Lock() + defer f.fetchMx.Unlock() + f.fetchRunning = running +} + +func (f *fetchDataParam) isRunning() bool { + f.fetchMx.Lock() + defer f.fetchMx.Unlock() + return f.fetchRunning +} + +const ( + defaultFetchIntervalSec = 30 +) + +func (l *LoadService) fetchData(f *fetchDataParam) { + ticker := time.NewTicker(defaultFetchIntervalSec * time.Second) + defer ticker.Stop() + + done := f.LoadTestDone + for { + select { + case <-ticker.C: + if !f.isRunning() { + f.setFetchRunning(true) + if err := rsyncFiles(f); err != nil { + log.Println(err) + } + f.setFetchRunning(false) + } + case <-done: + retry := 3 + for retry > 0 { + if !f.isRunning() { + if err := rsyncFiles(f); err != nil { + log.Println(err) + } + break + } + time.Sleep(time.Duration(1<<4-retry) * time.Second) + retry-- + } + + return + } + } +} + +func rsyncFiles(f *fetchDataParam) error { + loadTestKey := f.LoadTestKey + installLocation := f.InstallLocation + loadGeneratorInstallPath := f.InstallPath + + var wg sync.WaitGroup + resultsPrefix := []string{""} + + if f.AgentInstalled { + resultsPrefix = append(resultsPrefix, "_cpu", "_disk", "_memory", "_network") + } + + errorChan := make(chan error, len(resultsPrefix)) + + resultFolderPath := utils.JoinRootPathWith("/result/" + loadTestKey) + + err := utils.CreateFolderIfNotExist(utils.JoinRootPathWith("/result")) + if err != nil { + return err + } + + err = utils.CreateFolderIfNotExist(resultFolderPath) + if err != nil { + return err + } + + if installLocation == constant.Local { + for _, p := range resultsPrefix { + wg.Add(1) + go func(prefix string) { + defer wg.Done() + fileName := fmt.Sprintf("%s%s_result.csv", loadTestKey, prefix) + fromFilePath := fmt.Sprintf("%s/result/%s", loadGeneratorInstallPath, fileName) + toFilePath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) + + cmd := fmt.Sprintf(`rsync -avz %s %s`, fromFilePath, toFilePath) + err := utils.InlineCmd(cmd) + errorChan <- err + }(p) + } + } else if installLocation == constant.Remote { + for _, p := range resultsPrefix { + wg.Add(1) + go func(prefix string) { + defer wg.Done() + fileName := fmt.Sprintf("%s%s_result.csv", loadTestKey, prefix) + fromFilePath := fmt.Sprintf("%s/result/%s", loadGeneratorInstallPath, fileName) + toFilePath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) + + cmd := fmt.Sprintf(`rsync -avz -e "ssh -i %s -o StrictHostKeyChecking=no" %s@%s:%s %s`, + fmt.Sprintf("%s/.ssh/%s", f.Home, f.PrivateKeyName), + f.Username, + f.PublicIp, + fromFilePath, + toFilePath) + + err := utils.InlineCmd(cmd) + errorChan <- err + }(p) + } + } + + wg.Wait() + close(errorChan) + + for err := range errorChan { + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/core/load/service.go b/internal/core/load/service.go deleted file mode 100644 index 9e423fe..0000000 --- a/internal/core/load/service.go +++ /dev/null @@ -1,1892 +0,0 @@ -package load - -import ( - "bytes" - "context" - "errors" - "fmt" - "log" - "math" - "os" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/cloud-barista/cm-ant/internal/config" - "github.com/cloud-barista/cm-ant/internal/core/common/constant" - "github.com/cloud-barista/cm-ant/internal/infra/outbound/tumblebug" - "github.com/cloud-barista/cm-ant/internal/utils" - "gorm.io/gorm" -) - -// LoadService represents a service for managing load operations. -type LoadService struct { - loadRepo *LoadRepository - tumblebugClient *tumblebug.TumblebugClient -} - -// NewLoadService creates a new instance of LoadService. -func NewLoadService(loadRepo *LoadRepository, client *tumblebug.TumblebugClient) *LoadService { - return &LoadService{ - loadRepo: loadRepo, - tumblebugClient: client, - } -} - -func (l *LoadService) Readyz() error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - sqlDB, err := l.loadRepo.db.DB() - if err != nil { - return err - } - - err = sqlDB.Ping() - if err != nil { - return err - } - - err = l.tumblebugClient.ReadyzWithContext(ctx) - if err != nil { - return err - } - - return nil -} - -// InstallMonitoringAgent installs a monitoring agent on specified VMs or all VM on mci. -func (l *LoadService) InstallMonitoringAgent(param MonitoringAgentInstallationParams) ([]MonitoringAgentInstallationResult, error) { - utils.LogInfo("Starting installation of monitoring agent...") - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - var res []MonitoringAgentInstallationResult - - scriptPath := utils.JoinRootPathWith("/script/install-server-agent.sh") - utils.LogInfof("Reading installation script from %s", scriptPath) - installScript, err := os.ReadFile(scriptPath) - if err != nil { - utils.LogErrorf("Failed to read installation script: %v", err) - return res, err - } - username := "cb-user" - - utils.LogInfof("Fetching mci object for NS: %s, msi id: %s", param.NsId, param.MciId) - mci, err := l.tumblebugClient.GetMciWithContext(ctx, param.NsId, param.MciId) - if err != nil { - utils.LogErrorf("Failed to fetch mci : %v", err) - return res, err - } - - if len(mci.VMs) == 0 { - utils.LogErrorf("No VMs found on mci. Provision VM first.") - return res, errors.New("there is no vm on mci. provision vm first") - } - - var mapSet map[string]struct{} - if len(param.VmIds) > 0 { - mapSet = utils.SliceToMap(param.VmIds) - } - - var errorCollection []error - - for _, vm := range mci.VMs { - if mapSet != nil && !utils.Contains(mapSet, vm.ID) { - continue - } - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - m := MonitoringAgentInfo{ - Username: username, - Status: "installing", - AgentType: "perfmon", - NsId: param.NsId, - MciId: param.MciId, - VmId: vm.ID, - VmCount: len(mci.VMs), - } - utils.LogInfof("Inserting monitoring agent installation info into database vm id : %s", vm.ID) - err = l.loadRepo.InsertMonitoringAgentInfoTx(ctx, &m) - if err != nil { - utils.LogErrorf("Failed to insert monitoring agent info for vm id %s : %v", vm.ID, err) - errorCollection = append(errorCollection, err) - continue - } - - commandReq := tumblebug.SendCommandReq{ - Command: []string{string(installScript)}, - UserName: username, - } - - utils.LogInfof("Sending install command to mci. NS: %s, mci: %s, VMID: %s", param.NsId, param.MciId, vm.ID) - _, err = l.tumblebugClient.CommandToVmWithContext(ctx, param.NsId, param.MciId, vm.ID, commandReq) - - if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - m.Status = "timeout" - utils.LogErrorf("Timeout of context. Already 15 seconds has been passed. vm id : %s", vm.ID) - } else { - m.Status = "failed" - utils.LogErrorf("Error occurred during command execution: %v", err) - } - errorCollection = append(errorCollection, err) - } else { - m.Status = "completed" - } - - l.loadRepo.UpdateAgentInstallInfoStatusTx(ctx, &m) - - r := MonitoringAgentInstallationResult{ - ID: m.ID, - NsId: m.NsId, - MciId: m.MciId, - VmId: m.VmId, - VmCount: m.VmCount, - Status: m.Status, - Username: m.Username, - AgentType: m.AgentType, - CreatedAt: m.CreatedAt, - UpdatedAt: m.UpdatedAt, - } - - res = append(res, r) - utils.LogInfof( - "Complete installing monitoring agent on mics: %s, vm: %s", - m.MciId, - m.VmId, - ) - - time.Sleep(time.Second) - } - - if len(errorCollection) > 0 { - return res, fmt.Errorf("multiple errors: %v", errorCollection) - } - - return res, nil -} - -func (l *LoadService) GetAllMonitoringAgentInfos(param GetAllMonitoringAgentInfosParam) (GetAllMonitoringAgentInfoResult, error) { - var res GetAllMonitoringAgentInfoResult - var monitoringAgentInfos []MonitoringAgentInstallationResult - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - utils.LogInfof("GetAllMonitoringAgentInfos called with param: %+v", param) - result, totalRows, err := l.loadRepo.GetPagingMonitoringAgentInfosTx(ctx, param) - - if err != nil { - utils.LogErrorf("Error fetching monitoring agent infos: %v", err) - return res, err - } - - utils.LogInfof("Fetched %d monitoring agent infos", len(result)) - - for _, monitoringAgentInfo := range result { - var r MonitoringAgentInstallationResult - r.ID = monitoringAgentInfo.ID - r.NsId = monitoringAgentInfo.NsId - r.MciId = monitoringAgentInfo.MciId - r.VmId = monitoringAgentInfo.VmId - r.VmCount = monitoringAgentInfo.VmCount - r.Status = monitoringAgentInfo.Status - r.Username = monitoringAgentInfo.Username - r.AgentType = monitoringAgentInfo.AgentType - r.CreatedAt = monitoringAgentInfo.CreatedAt - r.UpdatedAt = monitoringAgentInfo.UpdatedAt - monitoringAgentInfos = append(monitoringAgentInfos, r) - } - - res.MonitoringAgentInfos = monitoringAgentInfos - res.TotalRow = totalRows - - return res, nil -} - -// UninstallMonitoringAgent uninstalls a monitoring agent on specified VMs or all VM on Mci. -// It takes MonitoringAgentInstallationParams as input and returns the number of affected results and any encountered error. -func (l *LoadService) UninstallMonitoringAgent(param MonitoringAgentInstallationParams) (int64, error) { - - ctx := context.Background() - var effectedResults int64 - - utils.LogInfo("Starting uninstallation of monitoring agent...") - result, err := l.loadRepo.GetAllMonitoringAgentInfosTx(ctx, param) - - if err != nil { - utils.LogErrorf("Failed to fetch monitoring agent information: %v", err) - return effectedResults, err - } - - scriptPath := utils.JoinRootPathWith("/script/remove-server-agent.sh") - utils.LogInfof("Reading uninstallation script from %s", scriptPath) - - uninstallPath, err := os.ReadFile(scriptPath) - if err != nil { - utils.LogErrorf("Failed to read uninstallation script: %v", err) - return effectedResults, err - } - - username := "cb-user" - - var errorCollection []error - for _, monitoringAgentInfo := range result { - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - - commandReq := tumblebug.SendCommandReq{ - Command: []string{string(uninstallPath)}, - UserName: username, - } - - _, err = l.tumblebugClient.CommandToVmWithContext(ctx, monitoringAgentInfo.NsId, monitoringAgentInfo.MciId, monitoringAgentInfo.VmId, commandReq) - if err != nil { - errorCollection = append(errorCollection, err) - utils.LogErrorf("Failed to uninstall monitoring agent on mci: %s, VM: %s - Error: %v", monitoringAgentInfo.MciId, monitoringAgentInfo.VmId, err) - continue - } - - err = l.loadRepo.DeleteAgentInstallInfoStatusTx(ctx, &monitoringAgentInfo) - - if err != nil { - utils.LogErrorf("Failed to delete agent installation status for mci: %s, VM: %s - Error: %v", monitoringAgentInfo.MciId, monitoringAgentInfo.VmId, err) - errorCollection = append(errorCollection, err) - continue - } - - utils.LogInfof("Successfully uninstalled monitoring agent on mci: %s, VM: %s", monitoringAgentInfo.MciId, monitoringAgentInfo.VmId) - - time.Sleep(time.Second) - } - - if len(errorCollection) > 0 { - return effectedResults, fmt.Errorf("multiple errors: %v", errorCollection) - } - - return effectedResults, nil -} - -const ( - antNsId = "ant-default-ns" - antMciDescription = "Default MCI for Cloud Migration Verification" - antInstallMonAgent = "no" - antMciLabel = "DynamicMci,AntDefault" - antMciId = "ant-default-mci" - - antVmDescription = "Default VM for Cloud Migration Verification" - antVmLabel = "DynamicVm,AntDefault" - antVmName = "ant-default-vm" - antVmRootDiskSize = "default" - antVmRootDiskType = "default" - antVmSubGroupSize = "1" - antVmUserPassword = "" - - antPubKeyName = "id_rsa_ant.pub" - antPrivKeyName = "id_rsa_ant" - - defaultDelay = 20 * time.Second - imageOs = "ubuntu22.04" -) - -// InstallLoadGenerator installs the load generator either locally or remotely. -// Currently remote request is executing via cb-tumblebug. -func (l *LoadService) InstallLoadGenerator(param InstallLoadGeneratorParam) (LoadGeneratorInstallInfoResult, error) { - utils.LogInfo("Starting InstallLoadGenerator") - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - var result LoadGeneratorInstallInfoResult - - loadGeneratorInstallInfo := LoadGeneratorInstallInfo{ - InstallLocation: param.InstallLocation, - InstallType: "jmeter", - InstallPath: config.AppConfig.Load.JMeter.Dir, - InstallVersion: config.AppConfig.Load.JMeter.Version, - Status: "starting", - } - - err := l.loadRepo.InsertLoadGeneratorInstallInfoTx(ctx, &loadGeneratorInstallInfo) - if err != nil { - utils.LogError("Failed to insert LoadGeneratorInstallInfo:", err) - return result, err - } - - utils.LogInfo("LoadGeneratorInstallInfo inserted successfully") - - installLocation := param.InstallLocation - installScriptPath := utils.JoinRootPathWith("/script/install-jmeter.sh") - - switch installLocation { - case constant.Local: - utils.LogInfo("Starting local installation of JMeter") - err := utils.Script(installScriptPath, []string{ - fmt.Sprintf("JMETER_WORK_DIR=%s", config.AppConfig.Load.JMeter.Dir), - fmt.Sprintf("JMETER_VERSION=%s", config.AppConfig.Load.JMeter.Version), - }) - if err != nil { - utils.LogError("Error while installing JMeter locally:", err) - return result, fmt.Errorf("error while installing jmeter; %s", err) - } - utils.LogInfo("Local installation of JMeter completed successfully") - case constant.Remote: - utils.LogInfo("Starting remote installation of JMeter") - // get the spec and image information - recommendVm, err := l.getRecommendVm(ctx, param.Coordinates) - if err != nil { - utils.LogError("Failed to get recommended VM:", err) - return result, err - } - antVmCommonSpec := recommendVm[0].Name - antVmConnectionName := recommendVm[0].ConnectionName - antVmCommonImage, err := utils.ReplaceAtIndex(antVmCommonSpec, imageOs, "+", 2) - - if err != nil { - utils.LogError("Error replacing VM spec index:", err) - return result, err - } - - // check namespace is valid or not - err = l.validDefaultNs(ctx, antNsId) - if err != nil { - utils.LogError("Error validating default namespace:", err) - return result, err - } - - // get the ant default mci - antMci, err := l.getAndDefaultMci(ctx, antVmCommonSpec, antVmCommonImage, antVmConnectionName) - if err != nil { - utils.LogError("Error getting or creating default mci:", err) - return result, err - } - - // if server is not running state, try to resume and get mci information - retryCount := config.AppConfig.Load.Retry - for retryCount > 0 && antMci.StatusCount.CountRunning < 1 { - utils.LogInfof("Attempting to resume MCI, retry count: %d", retryCount) - - err = l.tumblebugClient.ControlLifecycleWithContext(ctx, antNsId, antMci.ID, "resume") - if err != nil { - utils.LogError("Error resuming MCI:", err) - return result, err - } - time.Sleep(defaultDelay) - antMci, err = l.getAndDefaultMci(ctx, antVmCommonSpec, antVmCommonImage, antVmConnectionName) - if err != nil { - utils.LogError("Error getting MCI after resume attempt:", err) - return result, err - } - - retryCount = retryCount - 1 - } - - if antMci.StatusCount.CountRunning < 1 { - utils.LogError("No running VM on ant default MCI") - return result, errors.New("there is no running vm on ant default mci") - } - - addAuthorizedKeyCommand, err := getAddAuthorizedKeyCommand() - if err != nil { - utils.LogError("Error getting add authorized key command:", err) - return result, err - } - - installationCommand, err := utils.ReadToString(installScriptPath) - if err != nil { - utils.LogError("Error reading installation script:", err) - return result, err - } - - commandReq := tumblebug.SendCommandReq{ - Command: []string{installationCommand, addAuthorizedKeyCommand}, - } - - _, err = l.tumblebugClient.CommandToMciWithContext(ctx, antNsId, antMci.ID, commandReq) - if err != nil { - utils.LogError("Error sending command to MCI:", err) - return result, err - } - - utils.LogInfo("Commands sent to MCI successfully") - - loadGeneratorServers := make([]LoadGeneratorServer, 0) - - for i, vm := range antMci.VMs { - loadGeneratorServer := LoadGeneratorServer{ - Csp: vm.ConnectionConfig.ProviderName, - Region: vm.Region.Region, - Zone: vm.Region.Zone, - PublicIp: vm.PublicIP, - PrivateIp: vm.PrivateIP, - PublicDns: vm.PublicDNS, - MachineType: vm.CspViewVMDetail.VMSpecName, - Status: vm.Status, - SshPort: vm.SSHPort, - Lat: fmt.Sprintf("%f", vm.Location.Latitude), - Lon: fmt.Sprintf("%f", vm.Location.Longitude), - Username: vm.CspViewVMDetail.VMUserID, - VmId: vm.CspViewVMDetail.IID.SystemID, - StartTime: vm.CspViewVMDetail.StartTime, - AdditionalVmKey: vm.ID, - Label: vm.Label, - IsCluster: false, - IsMaster: i == 0, - ClusterSize: uint64(len(antMci.VMs)), - } - - loadGeneratorServers = append(loadGeneratorServers, loadGeneratorServer) - } - - loadGeneratorInstallInfo.LoadGeneratorServers = loadGeneratorServers - loadGeneratorInstallInfo.PublicKeyName = antPubKeyName - loadGeneratorInstallInfo.PrivateKeyName = antPrivKeyName - } - - loadGeneratorInstallInfo.Status = "installed" - err = l.loadRepo.UpdateLoadGeneratorInstallInfoTx(ctx, &loadGeneratorInstallInfo) - if err != nil { - utils.LogError("Error updating LoadGeneratorInstallInfo:", err) - return result, err - } - - utils.LogInfo("LoadGeneratorInstallInfo updated successfully") - - loadGeneratorServerResults := make([]LoadGeneratorServerResult, 0) - for _, l := range loadGeneratorInstallInfo.LoadGeneratorServers { - lr := LoadGeneratorServerResult{ - ID: l.ID, - Csp: l.Csp, - Region: l.Region, - Zone: l.Zone, - PublicIp: l.PublicIp, - PrivateIp: l.PrivateIp, - PublicDns: l.PublicDns, - MachineType: l.MachineType, - Status: l.Status, - SshPort: l.SshPort, - Lat: l.Lat, - Lon: l.Lon, - Username: l.Username, - VmId: l.VmId, - StartTime: l.StartTime, - AdditionalVmKey: l.AdditionalVmKey, - Label: l.Label, - CreatedAt: l.CreatedAt, - UpdatedAt: l.UpdatedAt, - } - loadGeneratorServerResults = append(loadGeneratorServerResults, lr) - } - - result.ID = loadGeneratorInstallInfo.ID - result.InstallLocation = loadGeneratorInstallInfo.InstallLocation - result.InstallType = loadGeneratorInstallInfo.InstallType - result.InstallPath = loadGeneratorInstallInfo.InstallPath - result.InstallVersion = loadGeneratorInstallInfo.InstallVersion - result.Status = loadGeneratorInstallInfo.Status - result.PublicKeyName = loadGeneratorInstallInfo.PublicKeyName - result.PrivateKeyName = loadGeneratorInstallInfo.PrivateKeyName - result.CreatedAt = loadGeneratorInstallInfo.CreatedAt - result.UpdatedAt = loadGeneratorInstallInfo.UpdatedAt - result.LoadGeneratorServers = loadGeneratorServerResults - - utils.LogInfo("InstallLoadGenerator completed successfully") - - return result, nil -} - -// getAddAuthorizedKeyCommand returns a command to add the authorized key. -func getAddAuthorizedKeyCommand() (string, error) { - pubKeyPath, _, err := validateKeyPair() - if err != nil { - return "", err - } - - pub, err := utils.ReadToString(pubKeyPath) - if err != nil { - return "", err - } - - addAuthorizedKeyScript := utils.JoinRootPathWith("/script/add-authorized-key.sh") - - addAuthorizedKeyCommand, err := utils.ReadToString(addAuthorizedKeyScript) - if err != nil { - return "", err - } - - addAuthorizedKeyCommand = strings.Replace(addAuthorizedKeyCommand, `PUBLIC_KEY=""`, fmt.Sprintf(`PUBLIC_KEY="%s"`, pub), 1) - return addAuthorizedKeyCommand, nil -} - -// validateKeyPair checks and generates SSH key pair if it doesn't exist. -func validateKeyPair() (string, string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", "", err - } - - privKeyPath := fmt.Sprintf("%s/.ssh/%s", homeDir, antPrivKeyName) - pubKeyPath := fmt.Sprintf("%s/.ssh/%s", homeDir, antPubKeyName) - - err = utils.CreateFolderIfNotExist(fmt.Sprintf("%s/.ssh", homeDir)) - if err != nil { - return pubKeyPath, privKeyPath, err - } - - exist := utils.ExistCheck(privKeyPath) - if !exist { - err := utils.GenerateSSHKeyPair(4096, privKeyPath, pubKeyPath) - if err != nil { - return pubKeyPath, privKeyPath, err - } - } - return pubKeyPath, privKeyPath, nil -} - -// getAndDefaultMci retrieves or creates the default MCI. -func (l *LoadService) getAndDefaultMci(ctx context.Context, antVmCommonSpec, antVmCommonImage, antVmConnectionName string) (tumblebug.MciRes, error) { - var antMci tumblebug.MciRes - var err error - antMci, err = l.tumblebugClient.GetMciWithContext(ctx, antNsId, antMciId) - if err != nil { - if errors.Is(err, tumblebug.ErrNotFound) { - dynamicMciArg := tumblebug.DynamicMciReq{ - Description: antMciDescription, - InstallMonAgent: antInstallMonAgent, - Label: antMciLabel, - Name: antMciId, - SystemLabel: "", - VM: []tumblebug.DynamicVmReq{ - { - CommonImage: antVmCommonImage, - CommonSpec: antVmCommonSpec, - ConnectionName: antVmConnectionName, - Description: antVmDescription, - Label: antVmLabel, - Name: antVmName, - RootDiskSize: antVmRootDiskSize, - RootDiskType: antVmRootDiskType, - SubGroupSize: antVmSubGroupSize, - VMUserPassword: antVmUserPassword, - }, - }, - } - antMci, err = l.tumblebugClient.DynamicMciWithContext(ctx, antNsId, dynamicMciArg) - time.Sleep(defaultDelay) - if err != nil { - return antMci, err - } - } else { - return antMci, err - } - } else if antMci.VMs != nil && len(antMci.VMs) == 0 { - - dynamicVmArg := tumblebug.DynamicVmReq{ - CommonImage: antVmCommonImage, - CommonSpec: antVmCommonSpec, - ConnectionName: antVmConnectionName, - Description: antVmDescription, - Label: antVmLabel, - Name: antVmName, - RootDiskSize: antVmRootDiskSize, - RootDiskType: antVmRootDiskType, - SubGroupSize: antVmSubGroupSize, - VMUserPassword: antVmUserPassword, - } - - antMci, err = l.tumblebugClient.DynamicVmWithContext(ctx, antNsId, antMciId, dynamicVmArg) - time.Sleep(defaultDelay) - if err != nil { - return antMci, err - } - } - return antMci, nil -} - -// getRecommendVm retrieves recommendVm to specify the location of provisioning. -func (l *LoadService) getRecommendVm(ctx context.Context, coordinates []string) (tumblebug.RecommendVmResList, error) { - recommendVmArg := tumblebug.RecommendVmReq{ - Filter: tumblebug.Filter{ - Policy: []tumblebug.FilterPolicy{ - { - Condition: []tumblebug.Condition{ - { - Operand: "2", - Operator: ">=", - }, - { - Operand: "8", - Operator: "<=", - }, - }, - Metric: "vCPU", - }, - { - Condition: []tumblebug.Condition{ - { - Operand: "4", - Operator: ">=", - }, - { - Operand: "8", - Operator: "<=", - }, - }, - Metric: "memoryGiB", - }, - { - Condition: []tumblebug.Condition{ - { - Operand: "aws", - }, - }, - Metric: "providerName", - }, - }, - }, - Limit: "1", - Priority: tumblebug.Priority{ - Policy: []tumblebug.Policy{ - { - Metric: "location", - Parameter: []tumblebug.Parameter{ - { - Key: "coordinateClose", - Val: coordinates, - }, - }, - }, - }, - }, - } - - recommendRes, err := l.tumblebugClient.GetRecommendVmWithContext(ctx, recommendVmArg) - - if err != nil { - return nil, err - } - - if len(recommendRes) == 0 { - return nil, errors.New("there is no recommended vm list") - } - - return recommendRes, nil -} - -// validDefaultNs checks if the default namespace exists, and creates it if not. -func (l *LoadService) validDefaultNs(ctx context.Context, antNsId string) error { - _, err := l.tumblebugClient.GetNsWithContext(ctx, antNsId) - if err != nil && errors.Is(err, tumblebug.ErrNotFound) { - - arg := tumblebug.CreateNsReq{ - Name: antNsId, - Description: "cm-ant default ns for validating migration", - } - - err = l.tumblebugClient.CreateNsWithContext(ctx, arg) - if err != nil { - return err - } - } else if err != nil { - return err - } - - return nil -} - -func (l *LoadService) UninstallLoadGenerator(param UninstallLoadGeneratorParam) error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - loadGeneratorInstallInfo, err := l.loadRepo.GetValidLoadGeneratorInstallInfoByIdTx(ctx, param.LoadGeneratorInstallInfoId) - - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - utils.LogError("Cannot find valid load generator install info:", err) - return errors.New("cannot find valid load generator install info") - } - utils.LogError("Error retrieving load generator install info:", err) - return err - } - - uninstallScriptPath := utils.JoinRootPathWith("/script/uninstall-jmeter.sh") - - switch loadGeneratorInstallInfo.InstallLocation { - case constant.Local: - err := utils.Script(uninstallScriptPath, []string{ - fmt.Sprintf("JMETER_WORK_DIR=%s", config.AppConfig.Load.JMeter.Dir), - fmt.Sprintf("JMETER_VERSION=%s", config.AppConfig.Load.JMeter.Version), - }) - if err != nil { - utils.LogErrorf("Error while uninstalling load generator: %s", err) - return fmt.Errorf("error while uninstalling load generator: %s", err) - } - case constant.Remote: - - uninstallCommand, err := utils.ReadToString(uninstallScriptPath) - if err != nil { - utils.LogError("Error reading uninstall script:", err) - return err - } - - commandReq := tumblebug.SendCommandReq{ - Command: []string{uninstallCommand}, - } - - _, err = l.tumblebugClient.CommandToMciWithContext(ctx, antNsId, antMciId, commandReq) - if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - utils.LogError("VM is not in running state. Cannot connect to the VMs.") - return errors.New("vm is not running state. cannot connect to the vms") - } - utils.LogError("Error sending uninstall command to MCI:", err) - return err - } - - // err = l.tumblebugClient.ControlLifecycleWithContext(ctx, antNsId, antMciId, "suspend") - // if err != nil { - // return err - // } - } - - loadGeneratorInstallInfo.Status = "deleted" - for i := range loadGeneratorInstallInfo.LoadGeneratorServers { - loadGeneratorInstallInfo.LoadGeneratorServers[i].Status = "deleted" - } - - err = l.loadRepo.UpdateLoadGeneratorInstallInfoTx(ctx, &loadGeneratorInstallInfo) - if err != nil { - utils.LogError("Error updating load generator install info:", err) - return err - } - - utils.LogInfo("Successfully uninstalled load generator.") - return nil -} - -func (l *LoadService) GetAllLoadGeneratorInstallInfo(param GetAllLoadGeneratorInstallInfoParam) (GetAllLoadGeneratorInstallInfoResult, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - var result GetAllLoadGeneratorInstallInfoResult - var infos []LoadGeneratorInstallInfoResult - pagedResult, totalRows, err := l.loadRepo.GetPagingLoadGeneratorInstallInfosTx(ctx, param) - - if err != nil { - utils.LogError("Error fetching paged load generator install infos:", err) - return result, err - } - - for _, l := range pagedResult { - loadGeneratorServerResults := make([]LoadGeneratorServerResult, 0) - for _, s := range l.LoadGeneratorServers { - lsr := LoadGeneratorServerResult{ - ID: s.ID, - Csp: s.Csp, - Region: s.Region, - Zone: s.Zone, - PublicIp: s.PublicIp, - PrivateIp: s.PrivateIp, - PublicDns: s.PublicDns, - MachineType: s.MachineType, - Status: s.Status, - SshPort: s.SshPort, - Lat: s.Lat, - Lon: s.Lon, - Username: s.Username, - VmId: s.VmId, - StartTime: s.StartTime, - AdditionalVmKey: s.AdditionalVmKey, - Label: s.Label, - CreatedAt: s.CreatedAt, - UpdatedAt: s.UpdatedAt, - } - loadGeneratorServerResults = append(loadGeneratorServerResults, lsr) - } - lr := LoadGeneratorInstallInfoResult{ - ID: l.ID, - InstallLocation: l.InstallLocation, - InstallType: l.InstallType, - InstallPath: l.InstallPath, - InstallVersion: l.InstallVersion, - Status: l.Status, - PublicKeyName: l.PublicKeyName, - PrivateKeyName: l.PrivateKeyName, - CreatedAt: l.CreatedAt, - UpdatedAt: l.UpdatedAt, - LoadGeneratorServers: loadGeneratorServerResults, - } - - infos = append(infos, lr) - } - - result.LoadGeneratorInstallInfoResults = infos - result.TotalRows = totalRows - utils.LogInfof("Fetched %d load generator install info results.", len(infos)) - - return result, nil -} - -// RunLoadTest initiates the load test and performs necessary initializations. -// Generates a load test key, installs the load generator or retrieves existing installation information, -// saves the load test execution state, and then asynchronously runs the load test. -func (l *LoadService) RunLoadTest(param RunLoadTestParam) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - - loadTestKey := utils.CreateUniqIdBaseOnUnixTime() - param.LoadTestKey = loadTestKey - - utils.LogInfof("Starting load test with key: %s", loadTestKey) - - if param.LoadGeneratorInstallInfoId == uint(0) { - utils.LogInfo("No LoadGeneratorInstallInfoId provided, installing load generator...") - result, err := l.InstallLoadGenerator(param.InstallLoadGenerator) - if err != nil { - utils.LogErrorf("Error installing load generator: %v", err) - return "", err - } - - param.LoadGeneratorInstallInfoId = result.ID - utils.LogInfof("Load generator installed with ID: %d", result.ID) - } - - if param.LoadGeneratorInstallInfoId == uint(0) { - utils.LogErrorf("LoadGeneratorInstallInfoId is still 0 after installation.") - return "", nil - } - - utils.LogInfof("Retrieving load generator installation info with ID: %d", param.LoadGeneratorInstallInfoId) - loadGeneratorInstallInfo, err := l.loadRepo.GetValidLoadGeneratorInstallInfoByIdTx(ctx, param.LoadGeneratorInstallInfoId) - if err != nil { - utils.LogErrorf("Error retrieving load generator installation info: %v", err) - return "", err - } - - duration, err := strconv.Atoi(param.Duration) - if err != nil { - return "", err - } - - rampUpTime, err := strconv.Atoi(param.RampUpTime) - - if err != nil { - return "", err - } - - stateArg := LoadTestExecutionState{ - LoadGeneratorInstallInfoId: loadGeneratorInstallInfo.ID, - LoadTestKey: loadTestKey, - ExecutionStatus: constant.OnPreparing, - StartAt: time.Now(), - TotalExpectedExcutionSecond: uint64(duration + rampUpTime), - } - - go l.processLoadTest(param, &loadGeneratorInstallInfo, &stateArg) - - var hs []LoadTestExecutionHttpInfo - - for _, h := range param.HttpReqs { - hh := LoadTestExecutionHttpInfo{ - Method: h.Method, - Protocol: h.Protocol, - Hostname: h.Hostname, - Port: h.Port, - Path: h.Path, - BodyData: h.BodyData, - } - - hs = append(hs, hh) - } - - loadArg := LoadTestExecutionInfo{ - LoadTestKey: loadTestKey, - TestName: param.TestName, - VirtualUsers: param.VirtualUsers, - Duration: param.Duration, - RampUpTime: param.RampUpTime, - RampUpSteps: param.RampUpSteps, - Hostname: param.Hostname, - Port: param.Port, - AgentInstalled: param.AgentInstalled, - AgentHostname: param.AgentHostname, - LoadGeneratorInstallInfoId: loadGeneratorInstallInfo.ID, - LoadTestExecutionHttpInfos: hs, - } - - utils.LogInfof("Saving load test execution info for key: %s", loadTestKey) - err = l.loadRepo.SaveForLoadTestExecutionTx(ctx, &loadArg, &stateArg) - if err != nil { - utils.LogErrorf("Error saving load test execution info: %v", err) - return "", err - } - - utils.LogInfof("Load test started successfully with key: %s", loadTestKey) - - return loadTestKey, nil - -} - -// processLoadTest executes the load test. -// Depending on whether the installation location is local or remote, it creates the test plan and runs test commands. -// Fetches and saves test results from the local or remote system. -func (l *LoadService) processLoadTest(param RunLoadTestParam, loadGeneratorInstallInfo *LoadGeneratorInstallInfo, loadTestExecutionState *LoadTestExecutionState) { - - loadTestDone := make(chan bool) - - var username string - var publicIp string - var port string - for _, s := range loadGeneratorInstallInfo.LoadGeneratorServers { - if s.IsMaster { - username = s.Username - publicIp = s.PublicIp - port = s.SshPort - } - } - - home, err := os.UserHomeDir() - if err != nil { - return - } - - dataParam := &fetchDataParam{ - LoadTestDone: loadTestDone, - LoadTestKey: param.LoadTestKey, - InstallLocation: loadGeneratorInstallInfo.InstallLocation, - InstallPath: loadGeneratorInstallInfo.InstallPath, - PublicKeyName: loadGeneratorInstallInfo.PublicKeyName, - PrivateKeyName: loadGeneratorInstallInfo.PrivateKeyName, - Username: username, - PublicIp: publicIp, - Port: port, - AgentInstalled: param.AgentInstalled, - Home: home, - } - - go l.fetchData(dataParam) - - defer func() { - loadTestDone <- true - close(loadTestDone) - updateErr := l.loadRepo.UpdateLoadTestExecutionStateTx(context.Background(), loadTestExecutionState) - if updateErr != nil { - utils.LogErrorf("Error updating load test execution state: %v", updateErr) - return - } - }() - - compileDuration, executionDuration, loadTestErr := l.executeLoadTest(param, loadGeneratorInstallInfo) - - loadTestExecutionState.CompileDuration = compileDuration - loadTestExecutionState.ExecutionDuration = executionDuration - - if loadTestErr != nil { - loadTestExecutionState.ExecutionStatus = constant.TestFailed - loadTestExecutionState.FailureMessage = loadTestErr.Error() - finishAt := time.Now() - loadTestExecutionState.FinishAt = &finishAt - return - } - - updateErr := l.updateLoadTestExecution(loadTestExecutionState) - if updateErr != nil { - loadTestExecutionState.ExecutionStatus = constant.UpdateFailed - loadTestExecutionState.FailureMessage = updateErr.Error() - finishAt := time.Now() - loadTestExecutionState.FinishAt = &finishAt - return - } - - loadTestExecutionState.ExecutionStatus = constant.Successed -} - -type fetchDataParam struct { - LoadTestDone <-chan bool - LoadTestKey string - InstallLocation constant.InstallLocation - InstallPath string - PublicKeyName string - PrivateKeyName string - Username string - PublicIp string - Port string - AgentInstalled bool - fetchMx sync.Mutex - fetchRunning bool - Home string -} - -func (f *fetchDataParam) setFetchRunning(running bool) { - f.fetchMx.Lock() - defer f.fetchMx.Unlock() - f.fetchRunning = running -} - -func (f *fetchDataParam) isRunning() bool { - f.fetchMx.Lock() - defer f.fetchMx.Unlock() - return f.fetchRunning -} - -const ( - defaultFetchIntervalSec = 30 -) - -func (l *LoadService) fetchData(f *fetchDataParam) { - ticker := time.NewTicker(defaultFetchIntervalSec * time.Second) - defer ticker.Stop() - - done := f.LoadTestDone - for { - select { - case <-ticker.C: - if !f.isRunning() { - f.setFetchRunning(true) - if err := rsyncFiles(f); err != nil { - log.Println(err) - } - f.setFetchRunning(false) - } - case <-done: - retry := 3 - for retry > 0 { - if !f.isRunning() { - if err := rsyncFiles(f); err != nil { - log.Println(err) - } - break - } - time.Sleep(time.Duration(1<<4-retry) * time.Second) - retry-- - } - - return - } - } -} - -func rsyncFiles(f *fetchDataParam) error { - loadTestKey := f.LoadTestKey - installLocation := f.InstallLocation - loadGeneratorInstallPath := f.InstallPath - - var wg sync.WaitGroup - resultsPrefix := []string{""} - - if f.AgentInstalled { - resultsPrefix = append(resultsPrefix, "_cpu", "_disk", "_memory", "_network") - } - - errorChan := make(chan error, len(resultsPrefix)) - - resultFolderPath := utils.JoinRootPathWith("/result/" + loadTestKey) - - err := utils.CreateFolderIfNotExist(utils.JoinRootPathWith("/result")) - if err != nil { - return err - } - - err = utils.CreateFolderIfNotExist(resultFolderPath) - if err != nil { - return err - } - - if installLocation == constant.Local { - for _, p := range resultsPrefix { - wg.Add(1) - go func(prefix string) { - defer wg.Done() - fileName := fmt.Sprintf("%s%s_result.csv", loadTestKey, prefix) - fromFilePath := fmt.Sprintf("%s/result/%s", loadGeneratorInstallPath, fileName) - toFilePath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) - - cmd := fmt.Sprintf(`rsync -avz %s %s`, fromFilePath, toFilePath) - err := utils.InlineCmd(cmd) - errorChan <- err - }(p) - } - } else if installLocation == constant.Remote { - for _, p := range resultsPrefix { - wg.Add(1) - go func(prefix string) { - defer wg.Done() - fileName := fmt.Sprintf("%s%s_result.csv", loadTestKey, prefix) - fromFilePath := fmt.Sprintf("%s/result/%s", loadGeneratorInstallPath, fileName) - toFilePath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) - - cmd := fmt.Sprintf(`rsync -avz -e "ssh -i %s" %s@%s:%s %s`, fmt.Sprintf("%s/.ssh/%s", f.Home, f.PrivateKeyName), f.Username, f.PublicIp, fromFilePath, toFilePath) - err := utils.InlineCmd(cmd) - errorChan <- err - }(p) - } - } - - wg.Wait() - close(errorChan) - - for err := range errorChan { - if err != nil { - return err - } - } - - return nil -} - -func (l *LoadService) executeLoadTest(param RunLoadTestParam, loadGeneratorInstallInfo *LoadGeneratorInstallInfo) (string, string, error) { - installLocation := loadGeneratorInstallInfo.InstallLocation - loadTestKey := param.LoadTestKey - loadGeneratorInstallPath := loadGeneratorInstallInfo.InstallPath - testPlanName := fmt.Sprintf("%s.jmx", loadTestKey) - resultFileName := fmt.Sprintf("%s_result.csv", loadTestKey) - loadGeneratorInstallVersion := loadGeneratorInstallInfo.InstallVersion - - utils.LogInfof("Running load test with key: %s", loadTestKey) - compileDuration := "0" - executionDuration := "0" - start := time.Now() - - if installLocation == constant.Remote { - utils.LogInfo("Remote execute detected.") - var buf bytes.Buffer - err := parseTestPlanStructToString(&buf, param, loadGeneratorInstallInfo) - if err != nil { - return compileDuration, executionDuration, err - } - - testPlan := buf.String() - - createFileCmd := fmt.Sprintf("cat << 'EOF' > %s/test_plan/%s \n%s\nEOF", loadGeneratorInstallPath, testPlanName, testPlan) - - commandReq := tumblebug.SendCommandReq{ - Command: []string{createFileCmd}, - } - - compileDuration = utils.DurationString(start) - _, err = l.tumblebugClient.CommandToMciWithContext(context.Background(), antNsId, antMciId, commandReq) - if err != nil { - return compileDuration, executionDuration, err - } - - jmeterTestCommand := generateJmeterExecutionCmd(loadGeneratorInstallPath, loadGeneratorInstallVersion, testPlanName, resultFileName) - - commandReq = tumblebug.SendCommandReq{ - Command: []string{jmeterTestCommand}, - } - - stdout, err := l.tumblebugClient.CommandToMciWithContext(context.Background(), antNsId, antMciId, commandReq) - if err != nil { - return compileDuration, executionDuration, err - } - executionDuration = utils.DurationString(start) - - if strings.Contains(stdout, "exited with status 1") { - return compileDuration, executionDuration, errors.New("jmeter test stopped unexpectedly") - } - - } else if installLocation == constant.Local { - utils.LogInfo("Local execute detected.") - - exist := utils.ExistCheck(loadGeneratorInstallPath) - - if !exist { - return compileDuration, executionDuration, errors.New("load generator installaion is not validated") - } - - outputFile, err := os.Create(fmt.Sprintf("%s/test_plan/%s.jmx", loadGeneratorInstallPath, loadTestKey)) - if err != nil { - return compileDuration, executionDuration, err - } - - err = parseTestPlanStructToString(outputFile, param, loadGeneratorInstallInfo) - - if err != nil { - return compileDuration, executionDuration, err - } - - jmeterTestCommand := generateJmeterExecutionCmd(loadGeneratorInstallPath, loadGeneratorInstallVersion, testPlanName, resultFileName) - compileDuration = utils.DurationString(start) - - err = utils.InlineCmd(jmeterTestCommand) - executionDuration = utils.DurationString(start) - if err != nil { - return compileDuration, executionDuration, fmt.Errorf("jmeter test stopped unexpectedly; %w", err) - } - } - - return compileDuration, executionDuration, nil -} - -func (l *LoadService) updateLoadTestExecution(loadTestExecutionState *LoadTestExecutionState) error { - err := l.loadRepo.UpdateLoadTestExecutionInfoDuration(context.Background(), loadTestExecutionState.LoadTestKey, loadTestExecutionState.CompileDuration, loadTestExecutionState.ExecutionDuration) - if err != nil { - utils.LogErrorf("Error updating load test execution info: %v", err) - return err - } - - loadTestExecutionState.ExecutionStatus = constant.OnFetching - err = l.loadRepo.UpdateLoadTestExecutionStateTx(context.Background(), loadTestExecutionState) - if err != nil { - utils.LogErrorf("Error updating load test execution state: %v", err) - return err - } - return nil -} - -// generateJmeterExecutionCmd generates the JMeter execution command. -// Constructs a JMeter command string that includes the test plan path and result file path. -func generateJmeterExecutionCmd(loadGeneratorInstallPath, loadGeneratorInstallVersion, testPlanName, resultFileName string) string { - utils.LogInfof("Generating JMeter execution command for test plan: %s, result file: %s", testPlanName, resultFileName) - - var builder strings.Builder - testPath := fmt.Sprintf("%s/test_plan/%s", loadGeneratorInstallPath, testPlanName) - resultPath := fmt.Sprintf("%s/result/%s", loadGeneratorInstallPath, resultFileName) - - builder.WriteString(fmt.Sprintf("%s/apache-jmeter-%s/bin/jmeter.sh", loadGeneratorInstallPath, loadGeneratorInstallVersion)) - builder.WriteString(" -n -f") - builder.WriteString(fmt.Sprintf(" -t=%s", testPath)) - builder.WriteString(fmt.Sprintf(" -l=%s", resultPath)) - - builder.WriteString(fmt.Sprintf(" && sudo rm %s", testPath)) - utils.LogInfof("JMeter execution command generated: %s", builder.String()) - return builder.String() -} - -func (l *LoadService) GetAllLoadTestExecutionState(param GetAllLoadTestExecutionStateParam) (GetAllLoadTestExecutionStateResult, error) { - var res GetAllLoadTestExecutionStateResult - var states []LoadTestExecutionStateResult - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - utils.LogInfof("GetAllLoadExecutionStates called with param: %+v", param) - result, totalRows, err := l.loadRepo.GetPagingLoadTestExecutionStateTx(ctx, param) - - if err != nil { - utils.LogErrorf("Error fetching load test execution state infos: %v", err) - return res, err - } - - utils.LogInfof("Fetched %d monitoring agent infos", len(result)) - - for _, loadTestExecutionState := range result { - state := mapLoadTestExecutionStateResult(loadTestExecutionState) - state.LoadGeneratorInstallInfo = mapLoadGeneratorInstallInfoResult(loadTestExecutionState.LoadGeneratorInstallInfo) - states = append(states, state) - } - - res.LoadTestExecutionStates = states - res.TotalRow = totalRows - - return res, nil -} - -func (l *LoadService) GetLoadTestExecutionState(param GetLoadTestExecutionStateParam) (LoadTestExecutionStateResult, error) { - var res LoadTestExecutionStateResult - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - utils.LogInfof("GetLoadTestExecutionState called with param: %+v", param) - state, err := l.loadRepo.GetLoadTestExecutionStateTx(ctx, param) - - if err != nil { - utils.LogErrorf("Error fetching load test execution state infos: %v", err) - return res, err - } - - res = mapLoadTestExecutionStateResult(state) - res.LoadGeneratorInstallInfo = mapLoadGeneratorInstallInfoResult(state.LoadGeneratorInstallInfo) - return res, nil -} - -func (l *LoadService) GetAllLoadTestExecutionInfos(param GetAllLoadTestExecutionInfosParam) (GetAllLoadTestExecutionInfosResult, error) { - var res GetAllLoadTestExecutionInfosResult - var rs []LoadTestExecutionInfoResult - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - utils.LogInfof("GetAllLoadTestExecutionInfos called with param: %+v", param) - result, totalRows, err := l.loadRepo.GetPagingLoadTestExecutionHistoryTx(ctx, param) - - if err != nil { - utils.LogErrorf("Error fetching load test execution infos: %v", err) - return res, err - } - - utils.LogInfof("Fetched %d load test execution infos:", len(result)) - - for _, r := range result { - rs = append(rs, mapLoadTestExecutionInfoResult(r)) - } - - res.TotalRow = totalRows - res.LoadTestExecutionInfos = rs - - return res, nil -} - -func (l *LoadService) GetLoadTestExecutionInfo(param GetLoadTestExecutionInfoParam) (LoadTestExecutionInfoResult, error) { - var res LoadTestExecutionInfoResult - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - utils.LogInfof("GetLoadTestExecutionInfo called with param: %+v", param) - executionInfo, err := l.loadRepo.GetLoadTestExecutionInfoTx(ctx, param) - - if err != nil { - utils.LogErrorf("Error fetching load test execution state infos: %v", err) - return res, err - } - - return mapLoadTestExecutionInfoResult(executionInfo), nil -} - -func mapLoadTestExecutionHttpInfoResult(h LoadTestExecutionHttpInfo) LoadTestExecutionHttpInfoResult { - return LoadTestExecutionHttpInfoResult{ - ID: h.ID, - Method: h.Method, - Protocol: h.Protocol, - Hostname: h.Hostname, - Port: h.Port, - Path: h.Path, - BodyData: h.BodyData, - } -} - -func mapLoadTestExecutionStateResult(state LoadTestExecutionState) LoadTestExecutionStateResult { - return LoadTestExecutionStateResult{ - ID: state.ID, - LoadTestKey: state.LoadTestKey, - ExecutionStatus: state.ExecutionStatus, - StartAt: state.StartAt, - FinishAt: state.FinishAt, - TotalExpectedExcutionSecond: state.TotalExpectedExcutionSecond, - FailureMessage: state.FailureMessage, - CompileDuration: state.CompileDuration, - ExecutionDuration: state.ExecutionDuration, - CreatedAt: state.CreatedAt, - UpdatedAt: state.UpdatedAt, - } -} - -func mapLoadGeneratorServerResult(s LoadGeneratorServer) LoadGeneratorServerResult { - return LoadGeneratorServerResult{ - ID: s.ID, - Csp: s.Csp, - Region: s.Region, - Zone: s.Zone, - PublicIp: s.PublicIp, - PrivateIp: s.PrivateIp, - PublicDns: s.PublicDns, - MachineType: s.MachineType, - Status: s.Status, - SshPort: s.SshPort, - Lat: s.Lat, - Lon: s.Lon, - Username: s.Username, - VmId: s.VmId, - StartTime: s.StartTime, - AdditionalVmKey: s.AdditionalVmKey, - Label: s.Label, - CreatedAt: s.CreatedAt, - } -} - -func mapLoadGeneratorInstallInfoResult(install LoadGeneratorInstallInfo) LoadGeneratorInstallInfoResult { - var servers []LoadGeneratorServerResult - for _, s := range install.LoadGeneratorServers { - servers = append(servers, mapLoadGeneratorServerResult(s)) - } - - return LoadGeneratorInstallInfoResult{ - ID: install.ID, - InstallLocation: install.InstallLocation, - InstallType: install.InstallType, - InstallPath: install.InstallPath, - InstallVersion: install.InstallVersion, - Status: install.Status, - CreatedAt: install.CreatedAt, - UpdatedAt: install.UpdatedAt, - PublicKeyName: install.PublicKeyName, - PrivateKeyName: install.PrivateKeyName, - LoadGeneratorServers: servers, - } -} - -func mapLoadTestExecutionInfoResult(executionInfo LoadTestExecutionInfo) LoadTestExecutionInfoResult { - var httpResults []LoadTestExecutionHttpInfoResult - for _, h := range executionInfo.LoadTestExecutionHttpInfos { - httpResults = append(httpResults, mapLoadTestExecutionHttpInfoResult(h)) - } - - executionState := mapLoadTestExecutionStateResult(executionInfo.LoadTestExecutionState) - installInfo := mapLoadGeneratorInstallInfoResult(executionInfo.LoadGeneratorInstallInfo) - - return LoadTestExecutionInfoResult{ - ID: executionInfo.ID, - LoadTestKey: executionInfo.LoadTestKey, - TestName: executionInfo.TestName, - VirtualUsers: executionInfo.VirtualUsers, - Duration: executionInfo.Duration, - RampUpTime: executionInfo.RampUpTime, - RampUpSteps: executionInfo.RampUpSteps, - Hostname: executionInfo.Hostname, - Port: executionInfo.Port, - AgentHostname: executionInfo.AgentHostname, - AgentInstalled: executionInfo.AgentInstalled, - CompileDuration: executionInfo.CompileDuration, - ExecutionDuration: executionInfo.ExecutionDuration, - LoadTestExecutionHttpInfos: httpResults, - LoadTestExecutionState: executionState, - LoadGeneratorInstallInfo: installInfo, - } -} - -func (l *LoadService) StopLoadTest(param StopLoadTestParam) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - state, err := l.loadRepo.GetLoadTestExecutionStateTx(ctx, GetLoadTestExecutionStateParam{ - LoadTestKey: param.LoadTestKey, - }) - - if err != nil { - return err - } - - if state.ExecutionStatus == constant.Successed { - return nil - } - - installInfo := state.LoadGeneratorInstallInfo - - killCmd := killCmdGen(param.LoadTestKey) - - if installInfo.InstallLocation == constant.Remote { - - commandReq := tumblebug.SendCommandReq{ - Command: []string{killCmd}, - } - _, err := l.tumblebugClient.CommandToMciWithContext(ctx, antNsId, antMciId, commandReq) - - if err != nil { - return err - } - - } else if installInfo.InstallLocation == constant.Local { - err := utils.InlineCmd(killCmd) - - if err != nil { - log.Println(err) - return err - } - } - - return nil - -} - -func killCmdGen(loadTestKey string) string { - grepRegex := fmt.Sprintf("'\\/bin\\/ApacheJMeter\\.jar.*%s'", loadTestKey) - utils.LogInfof("Generating kill command for load test key: %s", loadTestKey) - return fmt.Sprintf("kill -15 $(ps -ef | grep -E %s | awk '{print $2}')", grepRegex) -} - -type metricsUnits struct { - Multiple float64 - Unit string -} - -var tags = map[string]metricsUnits{ - "cpu_all_combined": { - Multiple: 0.001, - Unit: "%", - }, - "cpu_all_idle": { - Multiple: 0.001, - Unit: "%", - }, - "memory_all_used": { - Multiple: 0.001, - Unit: "%", - }, - "memory_all_free": { - Multiple: 0.001, - Unit: "%", - }, - "memory_all_used_kb": { - Multiple: 0.000001, - Unit: "mb", - }, - "memory_all_free_kb": { - Multiple: 0.000001, - Unit: "mb", - }, - "disk_read_kb": { - Multiple: 0.001, - Unit: "kb", - }, - "disk_write_kb": { - Multiple: 0.001, - Unit: "kb", - }, - "disk_use": { - Multiple: 0.001, - Unit: "%", - }, - "disk_total": { - Multiple: 0.000001, - Unit: "mb", - }, - "network_recv_kb": { - Multiple: 0.000001, - Unit: "kb", - }, - "network_sent_kb": { - Multiple: 0.001, - Unit: "kb", - }, -} - -func (l *LoadService) GetLoadTestResult(param GetLoadTestResultParam) (interface{}, error) { - loadTestKey := param.LoadTestKey - fileName := fmt.Sprintf("%s_result.csv", loadTestKey) - resultFolderPath := utils.JoinRootPathWith("/result/" + loadTestKey) - toFilePath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) - resultMap, err := appendResultRawData(toFilePath) - if err != nil { - return nil, err - } - - var resultSummaries []ResultSummary - - for label, results := range resultMap { - resultSummaries = append(resultSummaries, ResultSummary{ - Label: label, - Results: results, - }) - } - - formattedDate, err := resultFormat(param.Format, resultSummaries) - - if err != nil { - return nil, err - } - - return formattedDate, nil -} - -func (l *LoadService) GetLoadTestMetrics(param GetLoadTestResultParam) ([]MetricsSummary, error) { - loadTestKey := param.LoadTestKey - metrics := []string{"cpu", "disk", "memory", "network"} - resultFolderPath := utils.JoinRootPathWith("/result/" + loadTestKey) - - metricsMap := make(map[string][]*MetricsRawData) - var err error - for _, v := range metrics { - - fileName := fmt.Sprintf("%s_%s_result.csv", loadTestKey, v) - toPath := fmt.Sprintf("%s/%s", resultFolderPath, fileName) - - metricsMap, err = appendMetricsRawData(metricsMap, toPath) - if err != nil { - return nil, err - } - - } - - var metricsSummaries []MetricsSummary - - for label, metrics := range metricsMap { - metricsSummaries = append(metricsSummaries, MetricsSummary{ - Label: label, - Metrics: metrics, - }) - } - - if err != nil { - return nil, err - } - - return metricsSummaries, nil -} - -func calculatePercentile(elapsedList []int, percentile float64) float64 { - index := int(math.Ceil(float64(len(elapsedList))*percentile)) - 1 - - return float64(elapsedList[index]) -} - -func calculateMedian(data []int) float64 { - n := len(data) - if n%2 == 0 { - return float64(data[n/2-1]+data[n/2]) / 2 - } - return float64(data[n/2]) -} - -func findMin(elapsedList []int) float64 { - if len(elapsedList) == 0 { - return 0 - } - - return float64(elapsedList[0]) -} - -func findMax(elapsedList []int) float64 { - if len(elapsedList) == 0 { - return 0 - } - - return float64(elapsedList[len(elapsedList)-1]) -} - -func calculateErrorPercent(errorCount, requestCount int) float64 { - if requestCount == 0 { - return 0 - } - errorPercent := float64(errorCount) / float64(requestCount) * 100 - return errorPercent -} - -func calculateThroughput(totalRequests int, totalMillTime int) float64 { - return float64(totalRequests) / (float64(totalMillTime)) * 1000 -} - -func calculateReceivedKBPerSec(totalBytes int, totalMillTime int) float64 { - return (float64(totalBytes) / 1024) / (float64(totalMillTime)) * 1000 -} - -func calculateSentKBPerSec(totalBytes int, totalMillTime int) float64 { - return (float64(totalBytes) / 1024) / (float64(totalMillTime)) * 1000 -} - -func appendResultRawData(filePath string) (map[string][]*ResultRawData, error) { - var resultMap = make(map[string][]*ResultRawData) - - csvRows, err := utils.ReadCSV(filePath) - if err != nil || csvRows == nil { - return nil, err - } - - if len(*csvRows) <= 1 { - return nil, errors.New("result data file is empty") - } - // every time is basically millisecond - for i, row := range (*csvRows)[1:] { - label := row[2] - - elapsed, err := strconv.Atoi(row[1]) - if err != nil { - log.Printf("[%d] elapsed has error %s\n", i, err) - continue - } - bytes, err := strconv.Atoi(row[9]) - if err != nil { - log.Printf("[%d] bytes has error %s\n", i, err) - continue - } - sentBytes, err := strconv.Atoi(row[10]) - if err != nil { - log.Printf("[%d] sentBytes has error %s\n", i, err) - continue - } - latency, err := strconv.Atoi(row[14]) - if err != nil { - log.Printf("[%d] latency has error %s\n", i, err) - continue - } - idleTime, err := strconv.Atoi(row[15]) - if err != nil { - log.Printf("[%d] idleTime has error %s\n", i, err) - continue - } - connection, err := strconv.Atoi(row[16]) - if err != nil { - log.Printf("[%d] connection has error %s\n", i, err) - continue - } - unixMilliseconds, err := strconv.ParseInt(row[0], 10, 64) - if err != nil { - log.Printf("[%d] time has error %s\n", i, err) - continue - } - - isError := row[7] == "false" - url := row[13] - t := time.UnixMilli(unixMilliseconds) - - tr := &ResultRawData{ - No: i, - Elapsed: elapsed, - Bytes: bytes, - SentBytes: sentBytes, - IsError: isError, - URL: url, - Latency: latency, - IdleTime: idleTime, - Connection: connection, - Timestamp: t, - } - if _, ok := resultMap[label]; !ok { - resultMap[label] = []*ResultRawData{tr} - } else { - resultMap[label] = append(resultMap[label], tr) - } - } - - return resultMap, nil -} - -func appendMetricsRawData(mrds map[string][]*MetricsRawData, filePath string) (map[string][]*MetricsRawData, error) { - csvRows, err := utils.ReadCSV(filePath) - if err != nil || csvRows == nil { - return nil, err - } - - if len(*csvRows) <= 1 { - return nil, errors.New("metrics data file is empty") - } - - for i, row := range (*csvRows)[1:] { - isError := row[7] == "false" - intValue, err := strconv.Atoi(row[1]) - if err != nil { - log.Printf("[%d] value has error %s\n", i, err) - continue - } - - var label string - var value string - var u string - - if isError { - label = row[2] - } else { - words := strings.Split(row[2], " ") - label = words[len(words)-1] - - unit, ok := tags[label] - if !ok { - continue - } - - floatValue := float64(intValue) * unit.Multiple - value = strconv.FormatFloat(floatValue, 'f', 3, 64) - u = unit.Unit - } - - unixMilliseconds, err := strconv.ParseInt(row[0], 10, 64) - if err != nil { - log.Printf("[%d] time has error %s\n", i, err) - continue - } - - t := time.UnixMilli(unixMilliseconds) - - rd := &MetricsRawData{ - Value: value, - Unit: u, - IsError: isError, - Timestamp: t, - } - - if _, ok := mrds[label]; !ok { - mrds[label] = []*MetricsRawData{rd} - } else { - mrds[label] = append(mrds[label], rd) - } - } - - return mrds, nil -} - -func aggregate(resultRawDatas []ResultSummary) []*LoadTestStatistics { - var statistics []*LoadTestStatistics - - for i := range resultRawDatas { - - record := resultRawDatas[i] - var requestCount, totalElapsed, totalBytes, totalSentBytes, errorCount int - var elapsedList []int - if len(record.Results) < 1 { - continue - } - - startTime := record.Results[0].Timestamp - endTime := record.Results[0].Timestamp - for _, record := range record.Results { - requestCount++ - if !record.IsError { - totalElapsed += record.Elapsed - } - - totalBytes += record.Bytes - totalSentBytes += record.SentBytes - - if record.IsError { - errorCount++ - } - - if record.Timestamp.Before(startTime) { - startTime = record.Timestamp - } - if record.Timestamp.After(endTime) { - endTime = record.Timestamp - } - - elapsedList = append(elapsedList, record.Elapsed) - } - - // total Elapsed time and running time is different - runningTime := endTime.Sub(startTime).Milliseconds() - - // for percentile calculation purpose - sort.Ints(elapsedList) - - average := float64(totalElapsed) / float64(requestCount) - median := calculateMedian(elapsedList) - ninetyPercent := calculatePercentile(elapsedList, 0.9) - ninetyFive := calculatePercentile(elapsedList, 0.95) - ninetyNine := calculatePercentile(elapsedList, 0.99) - calcMin := findMin(elapsedList) - calcMax := findMax(elapsedList) - errorPercent := calculateErrorPercent(errorCount, requestCount) - throughput := calculateThroughput(requestCount, int(runningTime)) - receivedKB := calculateReceivedKBPerSec(totalBytes, int(runningTime)) - sentKB := calculateSentKBPerSec(totalSentBytes, int(runningTime)) - - labelStat := LoadTestStatistics{ - Label: record.Label, - RequestCount: requestCount, - Average: average, - Median: median, - NinetyPercent: ninetyPercent, - NinetyFive: ninetyFive, - NinetyNine: ninetyNine, - MinTime: calcMin, - MaxTime: calcMax, - ErrorPercent: errorPercent, - Throughput: throughput, - ReceivedKB: receivedKB, - SentKB: sentKB, - } - - statistics = append(statistics, &labelStat) - } - - return statistics -} - -func resultFormat(format constant.ResultFormat, resultSummaries []ResultSummary) (any, error) { - if resultSummaries == nil { - return nil, nil - } - - switch format { - case constant.Aggregate: - return aggregate(resultSummaries), nil - } - - return resultSummaries, nil -} diff --git a/internal/infra/outbound/tumblebug/req.go b/internal/infra/outbound/tumblebug/req.go index a1ec66e..6d12b3c 100644 --- a/internal/infra/outbound/tumblebug/req.go +++ b/internal/infra/outbound/tumblebug/req.go @@ -129,23 +129,23 @@ type CreateNsReq struct { } type DynamicVmReq struct { - CommonImage string `json:"commonImage"` - CommonSpec string `json:"commonSpec"` - ConnectionName string `json:"connectionName"` - Description string `json:"description"` - Label string `json:"label"` - Name string `json:"name"` - RootDiskSize string `json:"rootDiskSize"` - RootDiskType string `json:"rootDiskType"` - SubGroupSize string `json:"subGroupSize"` - VMUserPassword string `json:"vmUserPassword"` + CommonImage string `json:"commonImage"` + CommonSpec string `json:"commonSpec"` + ConnectionName string `json:"connectionName"` + Description string `json:"description"` + Label map[string]string `json:"label"` + Name string `json:"name"` + RootDiskSize string `json:"rootDiskSize"` + RootDiskType string `json:"rootDiskType"` + SubGroupSize string `json:"subGroupSize"` + VMUserPassword string `json:"vmUserPassword"` } type DynamicMciReq struct { - Description string `json:"description"` - InstallMonAgent string `json:"installMonAgent"` - Label string `json:"label"` - Name string `json:"name"` - SystemLabel string `json:"systemLabel"` - VM []DynamicVmReq `json:"vm"` + Description string `json:"description"` + InstallMonAgent string `json:"installMonAgent"` + Label map[string]string `json:"label"` + Name string `json:"name"` + SystemLabel string `json:"systemLabel"` + VM []DynamicVmReq `json:"vm"` } diff --git a/internal/infra/outbound/tumblebug/res.go b/internal/infra/outbound/tumblebug/res.go index 544f9ae..ae9f367 100644 --- a/internal/infra/outbound/tumblebug/res.go +++ b/internal/infra/outbound/tumblebug/res.go @@ -5,21 +5,21 @@ import ( ) type MciRes struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Status string `json:"status,omitempty"` - StatusCount StatusCountRes `json:"statusCount,omitempty"` - TargetStatus string `json:"targetStatus,omitempty"` - TargetAction string `json:"targetAction,omitempty"` - InstallMonAgent string `json:"installMonAgent,omitempty"` - ConfigureCloudAdaptiveNetwork string `json:"configureCloudAdaptiveNetwork,omitempty"` - Label string `json:"label,omitempty"` - SystemLabel string `json:"systemLabel,omitempty"` - SystemMessage string `json:"systemMessage,omitempty"` - Description string `json:"description,omitempty"` - VMs []VmRes `json:"vm,omitempty"` - NewVMList []string `json:"newVmList,omitempty"` - PlacementAlgo string `json:"placementAlgo"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + StatusCount StatusCountRes `json:"statusCount,omitempty"` + TargetStatus string `json:"targetStatus,omitempty"` + TargetAction string `json:"targetAction,omitempty"` + InstallMonAgent string `json:"installMonAgent,omitempty"` + ConfigureCloudAdaptiveNetwork string `json:"configureCloudAdaptiveNetwork,omitempty"` + Label map[string]string `json:"label,omitempty"` + SystemLabel string `json:"systemLabel,omitempty"` + SystemMessage string `json:"systemMessage,omitempty"` + Description string `json:"description,omitempty"` + VMs []VmRes `json:"vm,omitempty"` + NewVMList []string `json:"newVmList,omitempty"` + PlacementAlgo string `json:"placementAlgo"` } type StatusCountRes struct { @@ -106,7 +106,7 @@ type VmRes struct { NetworkAgentStatus string `json:"networkAgentStatus,omitempty"` SystemMessage string `json:"systemMessage,omitempty"` CreatedTime string `json:"createdTime,omitempty"` - Label string `json:"label,omitempty"` + Label map[string]string `json:"label,omitempty"` Description string `json:"description,omitempty"` Region RegionRes `json:"region,omitempty"` PublicIP string `json:"publicIP,omitempty"` diff --git a/meta/.gitkeep b/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/script/add-authorized-key.sh b/script/add-authorized-key.sh index eedd500..b0c0c0a 100755 --- a/script/add-authorized-key.sh +++ b/script/add-authorized-key.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + PUBLIC_KEY="" AUTHORIZED_KEYS_FILE="$HOME/.ssh/authorized_keys" SSH_DIR="$HOME/.ssh" diff --git a/script/install-jmeter.sh b/script/install-jmeter.sh index 0bd53f3..b11bd2c 100755 --- a/script/install-jmeter.sh +++ b/script/install-jmeter.sh @@ -35,16 +35,20 @@ fi echo "[CM-ANT] [Step 1/6] Installing default required tools..." -sudo apt-get update -y | sudo apt-get install -y wget openjdk-17-jre -unzip_jmeter() { - sudo tar -xf "${JMETER_FULL_PATH}.tgz" -C "${JMETER_WORK_DIR}" && sudo rm "${JMETER_FULL_PATH}.tgz" - sudo rm -rf "${JMETER_FULL_PATH}/docs" "${JMETER_FULL_PATH}/printable_docs" -} +sudo add-apt-repository universe -y +sudo apt-get update -y +sudo apt-get install -y wget openjdk-17-jre + +sleep 1s echo echo +unzip_jmeter() { + sudo tar -xf "${JMETER_FULL_PATH}.tgz" -C "${JMETER_WORK_DIR}" && sudo rm "${JMETER_FULL_PATH}.tgz" + sudo rm -rf "${JMETER_FULL_PATH}/docs" "${JMETER_FULL_PATH}/printable_docs" +} echo "[CM-ANT] [Step 2/6] Downloading and Extracting Apache JMeter..." if [ -d "${JMETER_FULL_PATH}" ]; then diff --git a/script/install_required_utils.sh b/script/install_required_utils.sh new file mode 100755 index 0000000..81c58f4 --- /dev/null +++ b/script/install_required_utils.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Function to check and install a package +check_and_install() { + PACKAGE_NAME=$1 + if command -v $PACKAGE_NAME &>/dev/null; then + echo "$PACKAGE_NAME is already installed." + else + echo "Installing $PACKAGE_NAME..." + apt-get update -y + apt-get install -y $PACKAGE_NAME + fi +} + +check_and_install rsync +sleep 1 +check_and_install ssh diff --git a/script/install_rsync.sh b/script/install_rsync.sh deleted file mode 100755 index 49bd493..0000000 --- a/script/install_rsync.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Check if rsync is already installed -if command -v rsync &>/dev/null; then - echo "rsync is already installed." - exit 0 -fi - -# Install rsync -echo "Installing rsync..." -apt-get update -apt-get install -y rsync \ No newline at end of file