From 4a66d9d9d356273b15ab98407d14a0e95e6f60dc Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Tue, 26 Nov 2019 22:15:37 +0100 Subject: [PATCH 01/17] [UPDATE] bump version to 1.2.0, move v2 operations to v2 package --- CHANGELOG.md | 6 +++++ README.md | 2 +- cf/v2/app.go | 30 ++++++++++++++++++++++++ cf/v2/resources.go | 6 +++++ puppeteer.go | 58 +++++++++++----------------------------------- 5 files changed, 57 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2112531..72958da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - multi manifest works correct. +## [1.2.0] - XXXX-XX-XX + +### Changed +- move all v2 operations into the v2 package + + ## [1.1.3] - 2019-11-26 ### Changed diff --git a/README.md b/README.md index f4f1b03..11aea47 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ To get an overview of the changes between versions, read the [changelog](CHANGEL ## Version -The latest version of *CF-Puppeteer* is *1.1.2*. It works with and is based on Cloud Foundry CLI version 6.43.0. +The latest version of *CF-Puppeteer* is *1.2.0*. It works with and is based on Cloud Foundry CLI version 6.43.0. For more details on the most recent release, check out the [changelog](CHANGELOG.md). diff --git a/cf/v2/app.go b/cf/v2/app.go index 4496fd4..ea43f0a 100644 --- a/cf/v2/app.go +++ b/cf/v2/app.go @@ -105,3 +105,33 @@ func (resource *ResourcesData) GetAppMetadata(appName string) (*AppResourcesEnti return &metaDataResponseEntity.AppResourcesEntity[0], nil } + +func (resource *ResourcesData) RenameApplication(oldName, newName string) error { + _, err := resource.connection.CliCommand("rename", oldName, newName) + return err +} + +func (resource *ResourcesData) StopApplication(appName string) error { + _, err := resource.connection.CliCommand("stop", appName) + return err +} + +func (resource *ResourcesData) StartApplication(appName string) error { + _, err := resource.connection.CliCommand("start", appName) + return err +} + +func (resource *ResourcesData) DeleteApplication(appName string) error { + _, err := resource.connection.CliCommand("delete", appName, "-f") + return err +} + +func (resource *ResourcesData) ShowCrashLogs(appName string) error { + _, err := resource.connection.CliCommand("logs", "--recent", appName) + return err +} + +func (resource *ResourcesData) ListApplications() error { + _, err := resource.connection.CliCommand("apps") + return err +} diff --git a/cf/v2/resources.go b/cf/v2/resources.go index 1380557..301bb4d 100644 --- a/cf/v2/resources.go +++ b/cf/v2/resources.go @@ -7,6 +7,12 @@ import ( type Resources interface { GetAppMetadata(appName string) (*AppResourcesEntity, error) + RenameApplication(oldName string, newName string) (err error) + StopApplication(appName string) (err error) + StartApplication(appName string) (err error) + DeleteApplication(appName string) (err error) + ShowCrashLogs(appName string) (err error) + ListApplications() (err error) } //ResourcesData internal struct with connection an tracing options etc diff --git a/puppeteer.go b/puppeteer.go index 38eaa2d..e3aad92 100644 --- a/puppeteer.go +++ b/puppeteer.go @@ -79,20 +79,20 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse // If current app isn't started, then we'll just delete it, and we're done => only if route switching was not used if curApp.Entity.State != "STARTED" && parsedArguments.AddRoutes == false { - return appRepo.DeleteApplication(parsedArguments.AppName) + return appRepo.v2Resources.DeleteApplication(parsedArguments.AppName) } // Do we have a ven app that will stop a rename? -> normal workflow only if we dont run the add routes mode if venApp != nil && parsedArguments.AddRoutes == false { // Finally, since the current app claims to be healthy, we'll delete the venerable app, and rename the current over the top - err = appRepo.DeleteApplication(venName) + err = appRepo.v2Resources.DeleteApplication(venName) if err != nil { return err } } if parsedArguments.AddRoutes == false { - return appRepo.RenameApplication(parsedArguments.AppName, venName) + return appRepo.v2Resources.RenameApplication(parsedArguments.AppName, venName) } return nil }, @@ -111,8 +111,8 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse //When upload fails the new application will be deleted and ven app will be renamed ReversePrevious: func() error { ui.Failed("error while uploading / deploying the application... roll everything back") - _ = appRepo.DeleteApplication(parsedArguments.AppName) - return appRepo.RenameApplication(venName, parsedArguments.AppName) + _ = appRepo.v2Resources.DeleteApplication(parsedArguments.AppName) + return appRepo.v2Resources.RenameApplication(venName, parsedArguments.AppName) }, }, // start @@ -124,7 +124,7 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse _ = appRepo.ShowLogs(parsedArguments.AppName) } if parsedArguments.NoStart == false { - return appRepo.StartApplication(parsedArguments.AppName) + return appRepo.v2Resources.StartApplication(parsedArguments.AppName) } return nil }, @@ -132,14 +132,14 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse if parsedArguments.ShowCrashLogs { //print logs before application delete ui.Say("show crash logs") - _ = appRepo.ShowCrashLogs(parsedArguments.AppName) + _ = appRepo.v2Resources.ShowCrashLogs(parsedArguments.AppName) } // If the app cannot start we'll have a lingering application // We delete this application so that the rename can succeed - _ = appRepo.DeleteApplication(parsedArguments.AppName) + _ = appRepo.v2Resources.DeleteApplication(parsedArguments.AppName) - return appRepo.RenameApplication(venName, parsedArguments.AppName) + return appRepo.v2Resources.RenameApplication(venName, parsedArguments.AppName) }, }, //check vor venerable application again -> because venerable action was set correct and ven app could exist now. @@ -161,9 +161,9 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse Forward: func() error { //if venerableAction was set to stop if strings.ToLower(parsedArguments.VenerableAction) == "stop" && venApp != nil { - return appRepo.StopApplication(venName) + return appRepo.v2Resources.StopApplication(venName) } else if strings.ToLower(parsedArguments.VenerableAction) == "delete" && venApp != nil { - return appRepo.DeleteApplication(venName) + return appRepo.v2Resources.DeleteApplication(venName) } //do nothing with the ven app return nil @@ -195,7 +195,7 @@ func (plugin CfPuppeteerPlugin) Run(cliConnection plugin.CliConnection, args []s ui.Say("A new version of your application has successfully been pushed!") ui.Say("") - _ = appRepo.ListApplications() + _ = appRepo.v2Resources.ListApplications() } // GetMetadata get plugin metadata @@ -204,8 +204,8 @@ func (CfPuppeteerPlugin) GetMetadata() plugin.PluginMetadata { Name: "cf-puppeteer", Version: plugin.VersionType{ Major: 1, - Minor: 1, - Build: 3, + Minor: 2, + Build: 0, }, Commands: []plugin.Command{ { @@ -253,36 +253,6 @@ func NewApplicationRepo(conn plugin.CliConnection, traceLogging bool) *Applicati } } -func (repo *ApplicationRepo) RenameApplication(oldName, newName string) error { - _, err := repo.conn.CliCommand("rename", oldName, newName) - return err -} - -func (repo *ApplicationRepo) StopApplication(appName string) error { - _, err := repo.conn.CliCommand("stop", appName) - return err -} - -func (repo *ApplicationRepo) StartApplication(appName string) error { - _, err := repo.conn.CliCommand("start", appName) - return err -} - -func (repo *ApplicationRepo) DeleteApplication(appName string) error { - _, err := repo.conn.CliCommand("delete", appName, "-f") - return err -} - -func (repo *ApplicationRepo) ShowCrashLogs(appName string) error { - _, err := repo.conn.CliCommand("logs", "--recent", appName) - return err -} - -func (repo *ApplicationRepo) ListApplications() error { - _, err := repo.conn.CliCommand("apps") - return err -} - func (repo *ApplicationRepo) ShowLogs(appName string) error { app, err := repo.conn.GetApp(appName) if err != nil { From 1f408f0730eccc63929a790da7e80b435c3e3f89 Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Sun, 15 Dec 2019 23:00:01 +0100 Subject: [PATCH 02/17] [NEW][#13] add vars parser and manifest replacer --- arguments/arguments.go | 7 ++- cf/v3/push_application.go | 2 +- fixtures/manifest_vars.yml | 21 +++++++++ fixtures/valid_vars_file.yml | 3 ++ manifest/manifest.go | 88 ++++++++++++++++++++++++++++++++---- manifest/manifest_test.go | 27 +++++++---- puppeteer_test.go | 46 ------------------- 7 files changed, 128 insertions(+), 66 deletions(-) create mode 100644 fixtures/manifest_vars.yml create mode 100644 fixtures/valid_vars_file.yml delete mode 100644 puppeteer_test.go diff --git a/arguments/arguments.go b/arguments/arguments.go index 7cb6aa5..03f6a9b 100644 --- a/arguments/arguments.go +++ b/arguments/arguments.go @@ -8,6 +8,7 @@ import ( "github.com/happytobi/cf-puppeteer/manifest" "os" "regexp" + "strconv" "strings" ) @@ -33,6 +34,7 @@ type ParserArguments struct { NoRoute bool AddRoutes bool NoStart bool + VarsFile string } type stringSlice []string @@ -84,6 +86,7 @@ func ParseArgs(args []string) (*ParserArguments, error) { flags.BoolVar(&pta.NoRoute, "no-route", false, "deploy new application without adding routes") flags.BoolVar(&pta.AddRoutes, "route-only", false, "only add routes from manifest to the application") flags.BoolVar(&pta.NoStart, "no-start", false, "don't start application after deployment; venerable action is none") + flags.StringVar(&pta.VarsFile, "vars-file", "", "path to a variable substitution file for manifest") //flags.BoolVar(&pta.ShowLogs, "show-app-log", false, "tail and show application log during application start") flags.StringVar(&pta.DockerImage, "docker-image", "", "docker image url") flags.StringVar(&pta.DockerUserName, "docker-username", "", "docker repository username; used with password from env CF_DOCKER_PASSWORD") @@ -113,7 +116,7 @@ func ParseArgs(args []string) (*ParserArguments, error) { } //parse manifest - parsedManifest, err := manifest.Parse(pta.ManifestPath) + parsedManifest, err := manifest.ParseApplicationManifest(pta.ManifestPath, pta.VarsFile) if err != nil { return pta, err //ErrManifest } @@ -130,7 +133,7 @@ func ParseArgs(args []string) (*ParserArguments, error) { } //set timeout - manifestTimeout := parsedManifest.ApplicationManifests[0].Timeout + manifestTimeout, _ := strconv.Atoi(parsedManifest.ApplicationManifests[0].Timeout) if manifestTimeout > 0 && pta.Timeout <= 0 { pta.Timeout = manifestTimeout } else if manifestTimeout <= 0 && pta.Timeout <= 0 { diff --git a/cf/v3/push_application.go b/cf/v3/push_application.go index 5f401f7..f53cbae 100644 --- a/cf/v3/push_application.go +++ b/cf/v3/push_application.go @@ -97,7 +97,7 @@ func (resource *ResourcesData) GenerateNoRouteYml(appName string, originalManife return "", err } //clean up manifest - newTempManifest.ApplicationManifests[0].NoRoute = true + newTempManifest.ApplicationManifests[0].NoRoute = "true" newTempManifest.ApplicationManifests[0].Routes = []map[string]string{} _, err = manifest.WriteYmlFile(manifestPathTemp, originalManifest) diff --git a/fixtures/manifest_vars.yml b/fixtures/manifest_vars.yml new file mode 100644 index 0000000..993a3e3 --- /dev/null +++ b/fixtures/manifest_vars.yml @@ -0,0 +1,21 @@ +--- +applications: + - name: myApp + memory: ((memory)) + buildpacks: + - java_buildpack + - go_buildpack + instances: ((instances)) + health-check-type: http + health-check-http-endpoint: /health + timeout: 2 + routes: + - route: ((host)).external.test.com + - route: ((host)).internal.test.com + - route: super.external.test.com + services: + - service1 + - service2 + env: + VAR1: 1 + VAR2: https://github.com/happytobi diff --git a/fixtures/valid_vars_file.yml b/fixtures/valid_vars_file.yml new file mode 100644 index 0000000..f5bcbbe --- /dev/null +++ b/fixtures/valid_vars_file.yml @@ -0,0 +1,3 @@ +instances: 2 +memory: 1G +host: myHost \ No newline at end of file diff --git a/manifest/manifest.go b/manifest/manifest.go index 4159848..28c8598 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -4,25 +4,29 @@ import ( "fmt" "gopkg.in/yaml.v2" "io/ioutil" + "reflect" + "regexp" + "strings" ) /* Application yaml represents the complete yaml structure +type is always string because you can use a vars placeholder in all attributes. */ type Application struct { Name string `yaml:"name"` - Instances int `yaml:"instances,omitempty"` + Instances string `yaml:"instances,omitempty"` Memory string `yaml:"memory,omitempty"` DiskQuota string `yaml:"disk_quota,omitempty"` Routes []map[string]string `yaml:"routes,omitempty"` - NoRoute bool `yaml:"no-route,omitempty"` + NoRoute string `yaml:"no-route,omitempty"` Buildpacks []string `yaml:"buildpacks,omitempty"` Command string `yaml:"command,omitempty"` Env map[string]string `yaml:"env,omitempty"` Services []string `yaml:"services,omitempty"` Stack string `yaml:"stack,omitempty"` Path string `yaml:"path,omitempty"` - Timeout int `yaml:"timeout,omitempty"` + Timeout string `yaml:"timeout,omitempty"` HealthCheckType string `yaml:"health-check-type,omitempty"` HealthCheckHTTPEndpoint string `yaml:"health-check-http-endpoint,omitempty"` } @@ -32,18 +36,85 @@ type Manifest struct { ApplicationManifests []Application `yaml:"applications"` } -// Parse parse application manifest files from provided path and return -// (right now) the app name of the first found application. -func Parse(manifestFilePath string) (manifest Manifest, err error) { +//VarsFile +type Variables map[string]interface{} + +//regex pattern - @see cf cli code +var ( + variablePlaceholderPattern = `\(\((!?[-/\.\w\pL]+)\)\)` + interpolationRegex = regexp.MustCompile(`\(\((!?[-/\.\w\pL]+)\)\)`) +) + +//ParseAndReplaceWithVars parse a manifest and vars file. +// get all values from vars file and put them into the manifest file so there will be a returned new manifest without +// placeholders +func ParseApplicationManifest(manifestFilePath string, varsFilePath string) (manifest Manifest, err error) { document, err := loadYmlFile(manifestFilePath) if err != nil || document.ApplicationManifests == nil { - return document, fmt.Errorf("could not parse file - file not valid") + return Manifest{}, fmt.Errorf("could not parse file, file not valid") + } + + //if there's no vars file, we can return the parsed manifest direct + if len(varsFilePath) <= 0 { + return document, nil + } + + varsFile, err := loadVarsFile(varsFilePath) + if err != nil { + return Manifest{}, fmt.Errorf("could not parse vars file, file not valid") + } + + //iterate through all the applications an check if vars are existing + for aI, app := range document.ApplicationManifests { + appManifestElement := reflect.ValueOf(&app).Elem() + for i := 0; i < appManifestElement.NumField(); i++ { + appElementField := appManifestElement.Field(i) + fieldValue := fmt.Sprintf("%v", appElementField.Interface()) + for mIndex, match := range interpolationRegex.FindAllSubmatch([]byte(fieldValue), -1) { + //get variable name + matchedVar := strings.TrimPrefix(string(match[1]), "!") + varsValue := varsFile[matchedVar] + //change string fields + if appElementField.Kind() == reflect.String { + replacedVar := strings.ReplaceAll(fieldValue, fmt.Sprintf("((%v))", matchedVar), fmt.Sprintf("%v", varsValue)) + appElementField.SetString(replacedVar) + } + //change slices + if appElementField.Kind() == reflect.Slice { + sliceElementField := appElementField.Index(mIndex) + if sliceElementField.Kind() == reflect.Map { + for _, mv := range sliceElementField.MapKeys() { + x := sliceElementField.MapIndex(mv) + nx := strings.ReplaceAll(x.String(), fmt.Sprintf("((%v))", matchedVar), fmt.Sprintf("%v", varsValue)) + sliceElementField.SetMapIndex(mv, reflect.ValueOf(nx)) + } + } + } + } + } + document.ApplicationManifests[aI] = app } return document, nil } +//load the vars file an throw errors then there is a issue +func loadVarsFile(varsFilePath string) (variables Variables, err error) { + fileBytes, err := ioutil.ReadFile(varsFilePath) + if err != nil { + return variables, fmt.Errorf("error while reading varsfile: %s", varsFilePath) + } + + err = yaml.Unmarshal(fileBytes, &variables) + if err != nil { + return variables, fmt.Errorf("error while parsing the varsfile %s error: %v", varsFilePath, err) + } + + return variables, nil +} + +//load the application yml file an throw errors then there is a issue func loadYmlFile(manifestFilePath string) (manifest Manifest, err error) { fileBytes, err := ioutil.ReadFile(manifestFilePath) if err != nil { @@ -70,6 +141,5 @@ func WriteYmlFile(manifestFilePath string, manifest Manifest) (newManifest Manif if err != nil { return Manifest{}, err } - return Parse(manifestFilePath) - + return ParseApplicationManifest(manifestFilePath, "") } diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 2ca6d1d..5231a52 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -18,25 +18,25 @@ func TestManifestParser(t *testing.T) { var _ = Describe("Parse Manifest", func() { It("parses complete manifest", func() { - manifest, err := Parse("../fixtures/manifest.yml") + manifest, err := ParseApplicationManifest("../fixtures/manifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("myApp")) Expect(manifest.ApplicationManifests[0].Buildpacks[0]).Should(Equal("java_buildpack")) Expect(manifest.ApplicationManifests[0].Buildpacks[1]).Should(Equal("go_buildpack")) - Expect(manifest.ApplicationManifests[0].Timeout).Should(Equal(2)) + Expect(manifest.ApplicationManifests[0].Timeout).Should(Equal("2")) Expect(manifest.ApplicationManifests[0].Routes[0]["route"]).Should(Equal("route1.external.test.com")) Expect(manifest.ApplicationManifests[0].Routes[1]["route"]).Should(Equal("route2.internal.test.com")) }) It("parses complete manifest with services", func() { - manifest, err := Parse("../fixtures/manifest.yml") + manifest, err := ParseApplicationManifest("../fixtures/manifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("myApp")) Expect(manifest.ApplicationManifests[0].Services[0]).Should(Equal("service1")) Expect(manifest.ApplicationManifests[0].Services[1]).Should(Equal("service2")) }) It("parses complete manifest with buildpack url", func() { - manifest, err := Parse("../fixtures/phpManifest.yml") + manifest, err := ParseApplicationManifest("../fixtures/phpManifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("appname")) Expect(manifest.ApplicationManifests[0].Services[0]).Should(Equal("ma-db")) @@ -50,7 +50,7 @@ var _ = Describe("Parse Manifest", func() { var _ = Describe("Parse multi Application Manifest", func() { It("parses complete manifest", func() { - manifest, err := Parse("../fixtures/multiManifest.yml") + manifest, err := ParseApplicationManifest("../fixtures/multiManifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("myApp")) Expect(manifest.ApplicationManifests[1].Name).Should(Equal("myApp2")) @@ -59,7 +59,7 @@ var _ = Describe("Parse multi Application Manifest", func() { var _ = Describe("Parse invalid Application Manifest", func() { It("parses invalid manifest", func() { - manifest, err := Parse("../fixtures/invalidManifest.yml") + manifest, err := ParseApplicationManifest("../fixtures/invalidManifest.yml", "") Expect(err).ShouldNot(BeNil()) Expect(manifest.ApplicationManifests).Should(BeNil()) }) @@ -67,7 +67,7 @@ var _ = Describe("Parse invalid Application Manifest", func() { var _ = Describe("Parse comp Manifest", func() { It("parses complicated manifest", func() { - manifest, err := Parse("../fixtures/defaultMultiManifest.yml") + manifest, err := ParseApplicationManifest("../fixtures/defaultMultiManifest.yml", "") Expect(err).Should(BeNil()) Expect(manifest.ApplicationManifests).ShouldNot(BeNil()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("app")) @@ -77,7 +77,7 @@ var _ = Describe("Parse comp Manifest", func() { var _ = Describe("Write new manifest", func() { It("write manifest file to specified path", func() { - manifest, err := Parse("../fixtures/manifest.yml") + manifest, err := ParseApplicationManifest("../fixtures/manifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) tempFile := fmt.Sprintf("%s%s", os.TempDir(), "testManifest.yml") parsedTempManifest, err := WriteYmlFile(tempFile, manifest) @@ -87,3 +87,14 @@ var _ = Describe("Write new manifest", func() { Expect(manifest.ApplicationManifests[0].Name).Should(Equal(parsedTempManifest.ApplicationManifests[0].Name)) }) }) + +var _ = Describe("Parse Manifest with vars-file", func() { + It("parse manifest with valid vars-file", func() { + manifest, err := ParseApplicationManifest("../fixtures/manifest_vars.yml", "../fixtures/valid_vars_file.yml") + Expect(err).ShouldNot(HaveOccurred()) + Expect(manifest.ApplicationManifests[0].Memory).Should(Equal("1G")) + Expect(manifest.ApplicationManifests[0].Instances).Should(Equal("2")) + Expect(manifest.ApplicationManifests[0].Routes[0]["route"]).Should(Equal("myHost.external.test.com")) + Expect(manifest.ApplicationManifests[0].Routes[1]["route"]).Should(Equal("myHost.internal.test.com")) + }) +}) diff --git a/puppeteer_test.go b/puppeteer_test.go deleted file mode 100644 index 96cbcf1..0000000 --- a/puppeteer_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package main_test - -import ( - "errors" - "testing" - - "code.cloudfoundry.org/cli/plugin/pluginfakes" - . "github.com/happytobi/cf-puppeteer" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestPuppeteer(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Puppeteer Suite") -} - -var _ = Describe("ApplicationRepo", func() { - var ( - cliConn *pluginfakes.FakeCliConnection - repo *ApplicationRepo - ) - - BeforeEach(func() { - cliConn = &pluginfakes.FakeCliConnection{} - repo = NewApplicationRepo(cliConn, false) - }) - - Describe("RenameApplication", func() { - It("renames the application", func() { - err := repo.RenameApplication("old-name", "new-name") - Expect(err).ToNot(HaveOccurred()) - - Expect(cliConn.CliCommandCallCount()).To(Equal(1)) - args := cliConn.CliCommandArgsForCall(0) - Expect(args).To(Equal([]string{"rename", "old-name", "new-name"})) - }) - - It("returns an error if one occurs", func() { - cliConn.CliCommandReturns([]string{}, errors.New("no app")) - - err := repo.RenameApplication("old-name", "new-name") - Expect(err).To(MatchError("no app")) - }) - }) -}) From fabbe7d117d085013b374a0d57e4fe6d89903282 Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Sun, 15 Dec 2019 23:09:19 +0100 Subject: [PATCH 03/17] [UPDATE] clean up cloud controller check --- cf/push.go | 112 +++--------------------------- cf/push_test.go | 180 ------------------------------------------------ 2 files changed, 10 insertions(+), 282 deletions(-) delete mode 100644 cf/push_test.go diff --git a/cf/push.go b/cf/push.go index 21ad01e..e9d3728 100644 --- a/cf/push.go +++ b/cf/push.go @@ -1,18 +1,12 @@ package cf import ( - "encoding/json" - "fmt" - "os" - "github.com/happytobi/cf-puppeteer/arguments" v2 "github.com/happytobi/cf-puppeteer/cf/v2" v3 "github.com/happytobi/cf-puppeteer/cf/v3" "code.cloudfoundry.org/cli/plugin" - "github.com/blang/semver" - "github.com/happytobi/cf-puppeteer/cf/cli" ) //ApplicationPushData struct @@ -26,12 +20,8 @@ type PuppeteerPush interface { PushApplication(venAppName string, venAppExists bool, spaceGUID string, parsedArguments *arguments.ParserArguments) error } -var cliCalls cli.Calls - //NewApplicationPush generate new cf puppeteer push func NewApplicationPush(conn plugin.CliConnection, traceLogging bool) *ApplicationPushData { - cliCalls = cli.NewCli(conn, traceLogging) - return &ApplicationPushData{ Connection: conn, TraceLogging: traceLogging, @@ -40,101 +30,19 @@ func NewApplicationPush(conn plugin.CliConnection, traceLogging bool) *Applicati //PushApplication push application to cf func (adp *ApplicationPushData) PushApplication(venAppName string, venAppExists bool, spaceGUID string, parsedArguments *arguments.ParserArguments) error { - v3Push, err := useV3Push() - if err != nil { - //fatal exit - os.Exit(1) - } - - if v3Push && parsedArguments.LegacyPush != true { - var v2Resources v2.Resources = v2.NewV2Resources(adp.Connection, adp.TraceLogging) - var push v3.Push = v3.NewV3Push(adp.Connection, adp.TraceLogging) + if parsedArguments.LegacyPush == true { + var legacyPush v2.Push = v2.NewV2LegacyPush(adp.Connection, adp.TraceLogging) if parsedArguments.AddRoutes { - //TODO loop over applications - return push.SwitchRoutesOnly(venAppName, parsedArguments.AppName, parsedArguments.Manifest.ApplicationManifests[0].Routes) + return legacyPush.SwitchRoutesOnly(venAppName, venAppExists, parsedArguments.AppName, parsedArguments.Manifest.ApplicationManifests[0].Routes) } - return push.PushApplication(venAppName, spaceGUID, parsedArguments, v2Resources) + return legacyPush.PushApplication(venAppName, spaceGUID, parsedArguments) } - - var legacyPush v2.Push = v2.NewV2LegacyPush(adp.Connection, adp.TraceLogging) + //v3 push + var v2Resources v2.Resources = v2.NewV2Resources(adp.Connection, adp.TraceLogging) + var push v3.Push = v3.NewV3Push(adp.Connection, adp.TraceLogging) if parsedArguments.AddRoutes { - return legacyPush.SwitchRoutesOnly(venAppName, venAppExists, parsedArguments.AppName, parsedArguments.Manifest.ApplicationManifests[0].Routes) - } - return legacyPush.PushApplication(venAppName, spaceGUID, parsedArguments) -} - -func useV3Push() (bool, error) { - _, v3ServerVersion, err := getCloudControllerAPIVersion() - if err != nil { - return false, err - } - v3SerSemVer, err := semver.Make(v3ServerVersion) - if err != nil { - return false, nil - } - - expectedRange, err := semver.ParseRange(fmt.Sprintf(">=%s", v3.MinControllerVersion)) - //check if we can use the v3 push - if expectedRange(v3SerSemVer) { - return true, nil - } - return false, nil -} - -// cloudControllerResponse -type cloudControllerResponse struct { - Links struct { - Self struct { - Href string `json:"href"` - } `json:"self"` - BitsService interface{} `json:"bits_service"` - CloudControllerV2 struct { - Href string `json:"href"` - Meta struct { - Version string `json:"version"` - } `json:"meta"` - } `json:"cloud_controller_v2"` - CloudControllerV3 struct { - Href string `json:"href"` - Meta struct { - Version string `json:"version"` - } `json:"meta"` - } `json:"cloud_controller_v3"` - NetworkPolicyV0 struct { - Href string `json:"href"` - } `json:"network_policy_v0"` - NetworkPolicyV1 struct { - Href string `json:"href"` - } `json:"network_policy_v1"` - Uaa struct { - Href string `json:"href"` - } `json:"uaa"` - Credhub interface{} `json:"credhub"` - Routing struct { - Href string `json:"href"` - } `json:"routing"` - Logging struct { - Href string `json:"href"` - } `json:"logging"` - LogStream struct { - Href string `json:"href"` - } `json:"log_stream"` - AppSSH struct { - Href string `json:"href"` - Meta struct { - HostKeyFingerprint string `json:"host_key_fingerprint"` - OauthClient string `json:"oauth_client"` - } `json:"meta"` - } `json:"app_ssh"` - } `json:"links"` -} - -func getCloudControllerAPIVersion() (string, string, error) { - callResp, err := cliCalls.GetJSON("/") - var response cloudControllerResponse - err = json.Unmarshal([]byte(callResp), &response) - if err != nil { - return "", "", err + //TODO loop over applications + return push.SwitchRoutesOnly(venAppName, parsedArguments.AppName, parsedArguments.Manifest.ApplicationManifests[0].Routes) } - return response.Links.CloudControllerV2.Meta.Version, response.Links.CloudControllerV3.Meta.Version, nil + return push.PushApplication(venAppName, spaceGUID, parsedArguments, v2Resources) } diff --git a/cf/push_test.go b/cf/push_test.go deleted file mode 100644 index a1d1a29..0000000 --- a/cf/push_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package cf - -import ( - "testing" - - "code.cloudfoundry.org/cli/plugin/pluginfakes" - "github.com/happytobi/cf-puppeteer/cf/cli" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestCfResources(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Puppeteer Version Check Test") -} - -var _ = Describe("cf push version check", func() { - var ( - cliConn *pluginfakes.FakeCliConnection - ) - - BeforeEach(func() { - cliConn = &pluginfakes.FakeCliConnection{} - //set cliCalls internal used var - cliCalls = cli.NewCli(cliConn, false) - }) - - Describe("check controller call", func() { - It("should return version", func() { - response := []string{ - `{ - "links": { - "self": { - "href": "https://puppeteer-fake-url.com" - }, - "bits_service": null, - "cloud_controller_v2": { - "href": "https://puppeteer-fake-url.com/v2", - "meta": { - "version": "2.134.0" - } - }, - "cloud_controller_v3": { - "href": "https://puppeteer-fake-url.com/v3", - "meta": { - "version": "3.69.0" - } - }, - "network_policy_v0": { - "href": "https://puppeteer-fake-url.com/networking/v0/external" - }, - "network_policy_v1": { - "href": "https://puppeteer-fake-url.com/networking/v1/external" - }, - "uaa": { - "href": "https://uaa.puppeteer-fake-url.com" - }, - "credhub": null, - "routing": { - "href": "https://puppeteer-fake-url.com/routing" - }, - "logging": { - "href": "wss://doppler.puppeteer-fake-url.com:443" - }, - "log_stream": { - "href": "https://log-stream.puppeteer-fake-url.com" - } - } - }`, - } - - cliConn.CliCommandWithoutTerminalOutputReturns(response, nil) - v2Ver, v3Ver, err := getCloudControllerAPIVersion() - Expect(cliConn.CliCommandWithoutTerminalOutputCallCount()).To(Equal(1)) - Expect(v3Ver).To(Equal("3.69.0")) - Expect(v2Ver).To(Equal("2.134.0")) - Expect(err).ToNot(HaveOccurred()) - }) - - It("check useV3Version", func() { - response := []string{ - `{ - "links": { - "self": { - "href": "https://puppeteer-fake-url.com" - }, - "bits_service": null, - "cloud_controller_v2": { - "href": "https://puppeteer-fake-url.com/v2", - "meta": { - "version": "2.134.0" - } - }, - "cloud_controller_v3": { - "href": "https://puppeteer-fake-url.com/v3", - "meta": { - "version": "3.69.0" - } - }, - "network_policy_v0": { - "href": "https://puppeteer-fake-url.com/networking/v0/external" - }, - "network_policy_v1": { - "href": "https://puppeteer-fake-url.com/networking/v1/external" - }, - "uaa": { - "href": "https://uaa.puppeteer-fake-url.com" - }, - "credhub": null, - "routing": { - "href": "https://puppeteer-fake-url.com/routing" - }, - "logging": { - "href": "wss://doppler.puppeteer-fake-url.com:443" - }, - "log_stream": { - "href": "https://log-stream.puppeteer-fake-url.com" - } - } - }`, - } - - cliConn.CliCommandWithoutTerminalOutputReturns(response, nil) - useV3, err := useV3Push() - Expect(cliConn.CliCommandWithoutTerminalOutputCallCount()).To(Equal(1)) - Expect(useV3).To(Equal(true)) - Expect(err).ToNot(HaveOccurred()) - }) - - It("check useV3Version - false", func() { - response := []string{ - `{ - "links": { - "self": { - "href": "https://puppeteer-fake-url.com" - }, - "bits_service": null, - "cloud_controller_v2": { - "href": "https://puppeteer-fake-url.com/v2", - "meta": { - "version": "2.134.0" - } - }, - "cloud_controller_v3": { - "href": "https://puppeteer-fake-url.com/v3", - "meta": { - "version": "3.10.0" - } - }, - "network_policy_v0": { - "href": "https://puppeteer-fake-url.com/networking/v0/external" - }, - "network_policy_v1": { - "href": "https://puppeteer-fake-url.com/networking/v1/external" - }, - "uaa": { - "href": "https://uaa.puppeteer-fake-url.com" - }, - "credhub": null, - "routing": { - "href": "https://puppeteer-fake-url.com/routing" - }, - "logging": { - "href": "wss://doppler.puppeteer-fake-url.com:443" - }, - "log_stream": { - "href": "https://log-stream.puppeteer-fake-url.com" - } - } - }`, - } - - cliConn.CliCommandWithoutTerminalOutputReturns(response, nil) - useV3, err := useV3Push() - Expect(cliConn.CliCommandWithoutTerminalOutputCallCount()).To(Equal(1)) - Expect(useV3).To(Equal(false)) - Expect(err).ToNot(HaveOccurred()) - }) - }) -}) From 9c9d52120aa7936de95f97d111ac0ac1d2f8670c Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Sun, 15 Dec 2019 23:31:45 +0100 Subject: [PATCH 04/17] [UPDATE][#13] add argument test for vars file --- arguments/arguments_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arguments/arguments_test.go b/arguments/arguments_test.go index b01a5ec..bb41a62 100644 --- a/arguments/arguments_test.go +++ b/arguments/arguments_test.go @@ -160,6 +160,7 @@ var _ = Describe("Flag Parsing", func() { "--show-app-log", "--route-only", "--no-start", + "--vars-file", "../fixtures/valid_vars_file.yml", }, ) Expect(err).ToNot(HaveOccurred()) @@ -179,6 +180,7 @@ var _ = Describe("Flag Parsing", func() { Expect(parsedArguments.HealthCheckHTTPEndpoint).To(Equal("/foo/bar")) Expect(parsedArguments.AddRoutes).To(Equal(true)) Expect(parsedArguments.NoStart).To(Equal(true)) + Expect(parsedArguments.VarsFile).To(Equal("../fixtures/valid_vars_file.yml")) }) It("parses args without appName and wrong envs format", func() { From 8210c9ebd6e644af951c8f0e4887b19d3883e3cd Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Sun, 15 Dec 2019 23:32:12 +0100 Subject: [PATCH 05/17] [UPDATE] clean up unused code --- cf/push.go | 18 +++++++++++------- cf/v3/version.go | 11 ----------- cf/v3/version_test.go | 32 -------------------------------- 3 files changed, 11 insertions(+), 50 deletions(-) delete mode 100644 cf/v3/version.go delete mode 100644 cf/v3/version_test.go diff --git a/cf/push.go b/cf/push.go index e9d3728..81cfe90 100644 --- a/cf/push.go +++ b/cf/push.go @@ -18,6 +18,7 @@ type ApplicationPushData struct { //PuppeteerPush push application interface type PuppeteerPush interface { PushApplication(venAppName string, venAppExists bool, spaceGUID string, parsedArguments *arguments.ParserArguments) error + SwitchRoutes(venAppName string, venAppExists bool, appName string, routes []map[string]string, legacyPush bool) error } //NewApplicationPush generate new cf puppeteer push @@ -32,17 +33,20 @@ func NewApplicationPush(conn plugin.CliConnection, traceLogging bool) *Applicati func (adp *ApplicationPushData) PushApplication(venAppName string, venAppExists bool, spaceGUID string, parsedArguments *arguments.ParserArguments) error { if parsedArguments.LegacyPush == true { var legacyPush v2.Push = v2.NewV2LegacyPush(adp.Connection, adp.TraceLogging) - if parsedArguments.AddRoutes { - return legacyPush.SwitchRoutesOnly(venAppName, venAppExists, parsedArguments.AppName, parsedArguments.Manifest.ApplicationManifests[0].Routes) - } return legacyPush.PushApplication(venAppName, spaceGUID, parsedArguments) } //v3 push var v2Resources v2.Resources = v2.NewV2Resources(adp.Connection, adp.TraceLogging) var push v3.Push = v3.NewV3Push(adp.Connection, adp.TraceLogging) - if parsedArguments.AddRoutes { - //TODO loop over applications - return push.SwitchRoutesOnly(venAppName, parsedArguments.AppName, parsedArguments.Manifest.ApplicationManifests[0].Routes) - } return push.PushApplication(venAppName, spaceGUID, parsedArguments, v2Resources) } + +//handle route switch +func (adp *ApplicationPushData) SwitchRoutes(venAppName string, venAppExists bool, appName string, routes []map[string]string, legacyPush bool) error { + if legacyPush { + var legacyPush v2.Push = v2.NewV2LegacyPush(adp.Connection, adp.TraceLogging) + return legacyPush.SwitchRoutesOnly(venAppName, venAppExists, appName, routes) + } + var push v3.Push = v3.NewV3Push(adp.Connection, adp.TraceLogging) + return push.SwitchRoutesOnly(venAppName, appName, routes) +} diff --git a/cf/v3/version.go b/cf/v3/version.go deleted file mode 100644 index a611f8b..0000000 --- a/cf/v3/version.go +++ /dev/null @@ -1,11 +0,0 @@ -package v3 - -import "github.com/blang/semver" - -//MinControllerVersion represents the v3 controller version -var MinControllerVersion = "3.27.0" - -//GetMinSemVersion retun varsion as semver.Version -func GetMinSemVersion() (semver.Version, error) { - return semver.Make(MinControllerVersion) -} diff --git a/cf/v3/version_test.go b/cf/v3/version_test.go deleted file mode 100644 index e54e190..0000000 --- a/cf/v3/version_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package v3 - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestCfResources(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Puppeteer Version Check Test") -} - -var _ = Describe("check version", func() { - - BeforeEach(func() { - - }) - Describe("check min v3 version", func() { - It("check version string", func() { - Expect(MinControllerVersion).To(Equal("3.27.0")) - }) - - It("check sem version", func() { - v3SemVer, err := GetMinSemVersion() - - Expect(v3SemVer.String()).To(Equal("3.27.0")) - Expect(err).ToNot(HaveOccurred()) - }) - }) -}) From 5cee3f4ced73b4f8c832ade026948b62548819ac Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Sun, 29 Dec 2019 22:40:32 +0100 Subject: [PATCH 06/17] [UPDATE] changelog --- CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72958da..66b2d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,15 +10,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - add argument to cleanup unused environment variables - log deployment time - refactor push - add rewind to all push things -- add parallel push to multi manifest - multi manifest pushed in parallel - - multi manifest works correct. -## [1.2.0] - XXXX-XX-XX +## [1.2.0] - 2019-12-XX + +### Added +- --vars-file argument was available again ### Changed - move all v2 operations into the v2 package +- delete cloud controller version check +- all routes will be switched lazy (after starting up the new application) +- clean up code ## [1.1.3] - 2019-11-26 From 93ca0d88d690261fc05056128947364b49fbd65e Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Sun, 29 Dec 2019 22:41:11 +0100 Subject: [PATCH 07/17] [NEW] add vars-file argument in posix style, add lazy route switch --- puppeteer.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/puppeteer.go b/puppeteer.go index e3aad92..fa1f07d 100644 --- a/puppeteer.go +++ b/puppeteer.go @@ -37,8 +37,10 @@ func venerableAppName(appName string) string { func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.ParserArguments) []rewind.Action { venName := venerableAppName(parsedArguments.AppName) + puppeteerPush := cf.NewApplicationPush(appRepo.conn, appRepo.traceLogging) var err error - var curApp, venApp *v2.AppResourcesEntity + var curApp *v2.AppResourcesEntity + var venApp *v2.AppResourcesEntity return []rewind.Action{ // get info about current app @@ -105,8 +107,10 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse if err != nil { return err } - var puppeteerPush cf.PuppeteerPush = cf.NewApplicationPush(appRepo.conn, appRepo.traceLogging) - return puppeteerPush.PushApplication(venName, venAppExists, space.Guid, parsedArguments) + if parsedArguments.AddRoutes == false { + return puppeteerPush.PushApplication(venName, venAppExists, space.Guid, parsedArguments) + } + return nil }, //When upload fails the new application will be deleted and ven app will be renamed ReversePrevious: func() error { @@ -142,6 +146,25 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse return appRepo.v2Resources.RenameApplication(venName, parsedArguments.AppName) }, }, + //switch routes because new application was started correct + { + Forward: func() error { + //switch route only is application was started and route switch option was set + ui.Say("check if routes should be added or switched from existing one") + if parsedArguments.NoStart == false && parsedArguments.NoRoute == false { + venAppExists := venApp != nil + return puppeteerPush.SwitchRoutes(venName, venAppExists, parsedArguments.AppName, parsedArguments.Manifest.ApplicationManifests[0].Routes, parsedArguments.LegacyPush) + } + ui.Say("nothing to do") + return nil + }, + ReversePrevious: func() error { + // If the app cannot start we'll have a lingering application + // We delete this application so that the rename can succeed + _ = appRepo.v2Resources.DeleteApplication(parsedArguments.AppName) + return appRepo.v2Resources.RenameApplication(venName, parsedArguments.AppName) + }, + }, //check vor venerable application again -> because venerable action was set correct and ven app could exist now. { Forward: func() error { @@ -232,6 +255,7 @@ func (CfPuppeteerPlugin) GetMetadata() plugin.PluginMetadata { "-no-start": "don't start application after deployment; venerable action will none", "-docker-image": "docker image url", "-docker-username": "docker repository username; used with password from env CF_DOCKER_PASSWORD", + "-vars-file": "path to a variable substitution file for manifest", }, }, }, From 4c693274081392a852aad6618d2c27b90112ed91 Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 11:22:08 +0100 Subject: [PATCH 08/17] [FIX] environment variable passing for legacy-push --- cf/v2/push_application.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cf/v2/push_application.go b/cf/v2/push_application.go index 66013d6..0fdac74 100644 --- a/cf/v2/push_application.go +++ b/cf/v2/push_application.go @@ -12,7 +12,7 @@ import ( //Push interface with all v3 actions type Push interface { - PushApplication(venAppName string, spaceGUID string, parsedArguments *arguments.ParserArguments) error + PushApplication(parsedArguments *arguments.ParserArguments) error SwitchRoutesOnly(venAppName string, venAppExists bool, appName string, routes []map[string]string) error } @@ -30,8 +30,8 @@ func NewV2LegacyPush(conn plugin.CliConnection, traceLogging bool) *LegacyResour } } -func (resource *LegacyResourcesData) PushApplication(venAppName, spaceGUID string, parsedArguments *arguments.ParserArguments) error { - ui.Say("use legacy push") +func (resource *LegacyResourcesData) PushApplication(parsedArguments *arguments.ParserArguments) error { + ui.InfoMessage("Use legacy push") args := []string{"push", parsedArguments.AppName, "-f", parsedArguments.ManifestPath, "--no-start"} if parsedArguments.AppPath != "" { @@ -71,7 +71,7 @@ func (resource *LegacyResourcesData) PushApplication(venAppName, spaceGUID strin } //SwitchRoutes switch route interface method to provide switch routes only option -func (resource *LegacyResourcesData) SwitchRoutesOnly(venAppName string,venAppExists bool, appName string, routes []map[string]string) (err error) { +func (resource *LegacyResourcesData) SwitchRoutesOnly(venAppName string, venAppExists bool, appName string, routes []map[string]string) (err error) { domains, err := resource.GetDomain(routes) if err != nil { return err @@ -102,13 +102,11 @@ func (resource *LegacyResourcesData) SwitchRoutesOnly(venAppName string,venAppEx func (resource *LegacyResourcesData) setEnvironmentVariables(parsedArguments *arguments.ParserArguments) (err error) { ui.Say("set passed environment variables") - varArgs := []string{"set-env", parsedArguments.AppName} //set all variables passed by --var for envKey, envVal := range parsedArguments.Envs { - tmpArgs := make([]string, len(parsedArguments.Envs)) - copy(tmpArgs, varArgs) - tmpArgs = append(tmpArgs, fmt.Sprintf("%s %s", envKey, envVal)) - err := resource.Executor.Execute(tmpArgs) + executeArgument := []string{"set-env", parsedArguments.AppName, envKey, envVal} + ui.DebugMessage("set-env: %s", executeArgument) + err := resource.Executor.Execute(executeArgument) if err != nil { return errors.Wrap(err, fmt.Sprintf("could not set env-variable with key %s to application %s", envKey, parsedArguments.AppName)) } From 588b2b0ee144bc8c376f04355883e553634e5afa Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 11:22:54 +0100 Subject: [PATCH 09/17] [UPDATE] clean up arguments, drop "show-app-log" --- arguments/arguments.go | 7 ++----- arguments/arguments_test.go | 6 ------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/arguments/arguments.go b/arguments/arguments.go index 03f6a9b..cdc5a42 100644 --- a/arguments/arguments.go +++ b/arguments/arguments.go @@ -25,7 +25,6 @@ type ParserArguments struct { StackName string VenerableAction string Envs map[string]string - ShowLogs bool ShowCrashLogs bool DockerImage string DockerUserName string @@ -75,10 +74,9 @@ func ParseArgs(args []string) (*ParserArguments, error) { flags.StringVar(&pta.StackName, "s", "", "name of the stack to use") flags.StringVar(&pta.HealthCheckType, "health-check-type", "port", "type of health check to perform") flags.StringVar(&pta.HealthCheckHTTPEndpoint, "health-check-http-endpoint", "", "endpoint for the 'http' health check type") - flags.IntVar(&pta.Timeout, "t", 0, "push timeout in seconds (defaults to 60 seconds)") + flags.IntVar(&pta.Timeout, "t", 0, "push timeout in seconds (default 60 seconds)") flags.IntVar(&pta.InvocationTimeout, "invocation-timeout", -1, "health check invocation timeout in seconds") flags.StringVar(&pta.Process, "process", "", "use health check type process") - flags.BoolVar(&pta.ShowLogs, "show-app-log", false, "tail and show application log during application start") flags.BoolVar(&pta.ShowCrashLogs, "show-crash-log", false, "Show recent logs when applications crashes while the deployment") flags.StringVar(&pta.VenerableAction, "venerable-action", "delete", "option to delete,stop,none application action on venerable app default is delete") flags.Var(&envs, "env", "Variable key value pair for adding dynamic environment variables; can specify multiple times") @@ -87,7 +85,6 @@ func ParseArgs(args []string) (*ParserArguments, error) { flags.BoolVar(&pta.AddRoutes, "route-only", false, "only add routes from manifest to the application") flags.BoolVar(&pta.NoStart, "no-start", false, "don't start application after deployment; venerable action is none") flags.StringVar(&pta.VarsFile, "vars-file", "", "path to a variable substitution file for manifest") - //flags.BoolVar(&pta.ShowLogs, "show-app-log", false, "tail and show application log during application start") flags.StringVar(&pta.DockerImage, "docker-image", "", "docker image url") flags.StringVar(&pta.DockerUserName, "docker-username", "", "docker repository username; used with password from env CF_DOCKER_PASSWORD") dockerPass := os.Getenv("CF_DOCKER_PASSWORD") @@ -101,7 +98,7 @@ func ParseArgs(args []string) (*ParserArguments, error) { argumentStartIndex := 2 //if first argument is not the appName we have to read the appName out of the manifest noAppNameProvided, _ := regexp.MatchString("^-[a-z]{0,3}", args[1]) - //noAppNameProvided := strings.Contains(args[1], "-") + if noAppNameProvided { argumentStartIndex = 1 } diff --git a/arguments/arguments_test.go b/arguments/arguments_test.go index bb41a62..83b8185 100644 --- a/arguments/arguments_test.go +++ b/arguments/arguments_test.go @@ -42,7 +42,6 @@ var _ = Describe("Flag Parsing", func() { Expect(parsedArguments.StackName).To(Equal("stack-name")) Expect(parsedArguments.Envs).To(Equal(map[string]string{"foo": "bar", "baz": "bob=true"})) Expect(parsedArguments.VenerableAction).Should(Equal("stop")) - Expect(parsedArguments.ShowLogs).To(Equal(false)) Expect(parsedArguments.ShowCrashLogs).To(Equal(false)) Expect(parsedArguments.Timeout).To(Equal(120)) Expect(parsedArguments.InvocationTimeout).To(Equal(2211)) @@ -69,7 +68,6 @@ var _ = Describe("Flag Parsing", func() { Expect(parsedArguments.AppPath).To(Equal("app-path")) Expect(parsedArguments.StackName).To(Equal("stack-name")) Expect(parsedArguments.VenerableAction).Should(Equal("delete")) - Expect(parsedArguments.ShowLogs).To(Equal(false)) Expect(parsedArguments.ShowCrashLogs).To(Equal(false)) Expect(parsedArguments.NoRoute).To(Equal(false)) Expect(parsedArguments.Timeout).To(Equal(2)) @@ -101,7 +99,6 @@ var _ = Describe("Flag Parsing", func() { Expect(parsedArguments.StackName).To(Equal("stack-name")) Expect(parsedArguments.Envs).To(Equal(map[string]string{"foo": "bar", "baz": "bob"})) Expect(parsedArguments.VenerableAction).Should(Equal("stop")) - Expect(parsedArguments.ShowLogs).To(Equal(false)) Expect(parsedArguments.ShowCrashLogs).To(Equal(false)) Expect(parsedArguments.NoRoute).To(Equal(true)) Expect(parsedArguments.Timeout).To(Equal(2)) @@ -133,7 +130,6 @@ var _ = Describe("Flag Parsing", func() { Expect(parsedArguments.StackName).To(Equal("stack-name")) Expect(parsedArguments.Envs).To(Equal(map[string]string{"foo": "bar", "baz": "bob"})) Expect(parsedArguments.VenerableAction).Should(Equal("stop")) - Expect(parsedArguments.ShowLogs).To(Equal(false)) Expect(parsedArguments.ShowCrashLogs).To(Equal(true)) Expect(parsedArguments.Timeout).To(Equal(60)) Expect(parsedArguments.InvocationTimeout).To(Equal(-1)) @@ -157,7 +153,6 @@ var _ = Describe("Flag Parsing", func() { "--process", "process-name", "--health-check-type", "process", "--health-check-http-endpoint", "/foo/bar", - "--show-app-log", "--route-only", "--no-start", "--vars-file", "../fixtures/valid_vars_file.yml", @@ -171,7 +166,6 @@ var _ = Describe("Flag Parsing", func() { Expect(parsedArguments.StackName).To(Equal("stack-name")) Expect(parsedArguments.Envs).To(Equal(map[string]string{"foo": "bar", "baz": "bob"})) Expect(parsedArguments.VenerableAction).Should(Equal("none")) - Expect(parsedArguments.ShowLogs).To(Equal(true)) Expect(parsedArguments.ShowCrashLogs).To(Equal(false)) Expect(parsedArguments.Timeout).To(Equal(120)) Expect(parsedArguments.InvocationTimeout).To(Equal(2211)) From b5f651d1988e4fbcf2bf2717e1accdfcca9109df Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 11:23:53 +0100 Subject: [PATCH 10/17] [UPDATE] ui messages --- cf/v2/app.go | 2 +- ui/ui.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/cf/v2/app.go b/cf/v2/app.go index ea43f0a..6558582 100644 --- a/cf/v2/app.go +++ b/cf/v2/app.go @@ -99,7 +99,7 @@ func (resource *ResourcesData) GetAppMetadata(appName string) (*AppResourcesEnti } if len(metaDataResponseEntity.AppResourcesEntity) == 0 { - ui.Warn("No venerable application found") + ui.Warn(fmt.Sprintf("No application with name: %s found", appName)) return nil, ErrAppNotFound } diff --git a/ui/ui.go b/ui/ui.go index 7a63b89..721f9fa 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -4,6 +4,7 @@ import ( "code.cloudfoundry.org/cli/cf/i18n" "code.cloudfoundry.org/cli/cf/terminal" "code.cloudfoundry.org/cli/cf/trace" + "fmt" "os" ) @@ -31,16 +32,35 @@ func Ok() { ui.Ok() } +//InfoMessage print out message in green / ok colored mode +func InfoMessage(message string) { + ui.Say(terminal.SuccessColor(message)) +} + //Failed message see cf/terminal func Failed(message string, args ...interface{}) { ui.Failed(message, args...) } +//FailedMessage print out message in error color without the "FAILED" message +func FailedMessage(message string) { + ui.Say(terminal.FailureColor(message)) +} + //Warn message see cf/terminal func Warn(message string, args ...interface{}) { ui.Warn(message, args...) } +func DebugMessage(message string, args ...interface{}) { + traceEnv := os.Getenv("CF_TRACE") + if traceEnv == "true" || (traceEnv != "false" && len(traceEnv) > 0) { + //check env for CF_TRACE + message = fmt.Sprintf(message, args...) + ui.Say(terminal.AdvisoryColor(message)) + } +} + //LoadingIndication message see cf/terminal func LoadingIndication() { ui.LoadingIndication() From 59899d5b5d625e3037f279833dc7495704e0c681 Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 11:24:29 +0100 Subject: [PATCH 11/17] [UPDATE] drop unused variables --- cf/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/push.go b/cf/push.go index 81cfe90..fcc68dc 100644 --- a/cf/push.go +++ b/cf/push.go @@ -33,7 +33,7 @@ func NewApplicationPush(conn plugin.CliConnection, traceLogging bool) *Applicati func (adp *ApplicationPushData) PushApplication(venAppName string, venAppExists bool, spaceGUID string, parsedArguments *arguments.ParserArguments) error { if parsedArguments.LegacyPush == true { var legacyPush v2.Push = v2.NewV2LegacyPush(adp.Connection, adp.TraceLogging) - return legacyPush.PushApplication(venAppName, spaceGUID, parsedArguments) + return legacyPush.PushApplication(parsedArguments) } //v3 push var v2Resources v2.Resources = v2.NewV2Resources(adp.Connection, adp.TraceLogging) From 6ae7c1b47ea13b7e62c4a233ff4a776d66933a8e Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 11:25:22 +0100 Subject: [PATCH 12/17] [UPDATE] clean up unused variables --- manifest/manifest.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/manifest/manifest.go b/manifest/manifest.go index 28c8598..a2ff1be 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -19,7 +19,7 @@ type Application struct { Memory string `yaml:"memory,omitempty"` DiskQuota string `yaml:"disk_quota,omitempty"` Routes []map[string]string `yaml:"routes,omitempty"` - NoRoute string `yaml:"no-route,omitempty"` + NoRoute bool `yaml:"no-route,omitempty"` Buildpacks []string `yaml:"buildpacks,omitempty"` Command string `yaml:"command,omitempty"` Env map[string]string `yaml:"env,omitempty"` @@ -41,8 +41,7 @@ type Variables map[string]interface{} //regex pattern - @see cf cli code var ( - variablePlaceholderPattern = `\(\((!?[-/\.\w\pL]+)\)\)` - interpolationRegex = regexp.MustCompile(`\(\((!?[-/\.\w\pL]+)\)\)`) + interpolationRegex = regexp.MustCompile(`\(\((!?[-/\.\w\pL]+)\)\)`) ) //ParseAndReplaceWithVars parse a manifest and vars file. From 2088e1be289e274be617a97f95ec4a2e5784a563 Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 11:26:31 +0100 Subject: [PATCH 13/17] [UPDATE] generate no route yml without dependencies --- cf/v3/push_application.go | 34 +++++++++++-------------- cf/v3/push_application_test.go | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 cf/v3/push_application_test.go diff --git a/cf/v3/push_application.go b/cf/v3/push_application.go index f53cbae..6ee9b91 100644 --- a/cf/v3/push_application.go +++ b/cf/v3/push_application.go @@ -9,7 +9,6 @@ import ( v2 "github.com/happytobi/cf-puppeteer/cf/v2" "github.com/happytobi/cf-puppeteer/manifest" "github.com/happytobi/cf-puppeteer/ui" - "github.com/jinzhu/copier" "github.com/pkg/errors" "strconv" ) @@ -50,17 +49,14 @@ func (resource *ResourcesData) PushApplication(venAppName, spaceGUID string, par return err } - ui.Say("apply manifest file") - manifestPath := parsedArguments.ManifestPath - if parsedArguments.NoRoute { - newManifestPath, err := resource.GenerateNoRouteYml(parsedArguments.AppName, parsedArguments.Manifest) - if err != nil { - return errors.Wrap(err, "could not generate a new temp manifest without routes") - } - ui.Say("use no route manifest") - manifestPath = newManifestPath + ui.Say("generate manifest without routes...") + + manifestPath, err := resource.GenerateNoRouteYml(parsedArguments.AppName, parsedArguments.Manifest) + if err != nil { + return errors.Wrap(err, "could not generate a new temp manifest without routes") } + ui.Say("apply manifest file") err = resource.AssignAppManifest(manifestPath) if err != nil { return err @@ -89,18 +85,18 @@ func (resource *ResourcesData) SwitchRoutesOnly(venAppName string, appName strin //GenerateNoRouteYml generate temp manifest without routes to skip route creation func (resource *ResourcesData) GenerateNoRouteYml(appName string, originalManifest manifest.Manifest) (newManifestPath string, err error) { - manifestPathTemp := resource.GenerateTempFile(appName, "yml") //Clone manifest to change them without side effects - newTempManifest := manifest.Manifest{} - err = copier.Copy(&newTempManifest, &originalManifest) - if err != nil { - return "", err + newTempManifest := manifest.Manifest{ApplicationManifests: make([]manifest.Application, len(originalManifest.ApplicationManifests))} + + //copy important information into no route yml (only resources are important) + for index, app := range originalManifest.ApplicationManifests { + newApp := manifest.Application{Name: app.Name, Instances: app.Instances, Memory: app.Memory, DiskQuota: app.DiskQuota, NoRoute: true, Routes: []map[string]string{}} + newTempManifest.ApplicationManifests[index] = newApp } - //clean up manifest - newTempManifest.ApplicationManifests[0].NoRoute = "true" - newTempManifest.ApplicationManifests[0].Routes = []map[string]string{} - _, err = manifest.WriteYmlFile(manifestPathTemp, originalManifest) + manifestPathTemp := resource.GenerateTempFile(appName, "yml") + _, err = manifest.WriteYmlFile(manifestPathTemp, newTempManifest) + if err != nil { return "", err } diff --git a/cf/v3/push_application_test.go b/cf/v3/push_application_test.go new file mode 100644 index 0000000..684d4a3 --- /dev/null +++ b/cf/v3/push_application_test.go @@ -0,0 +1,46 @@ +package v3_test + +import ( + "code.cloudfoundry.org/cli/plugin/pluginfakes" + "github.com/happytobi/cf-puppeteer/cf/cli" + "github.com/happytobi/cf-puppeteer/cf/v3" + manifest "github.com/happytobi/cf-puppeteer/manifest" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestCfPushApplicationActions(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Puppeteer CF Push Application") +} + +var _ = Describe("cf-push application test", func() { + var ( + cliConn *pluginfakes.FakeCliConnection + resourcesData *v3.ResourcesData + manifestFile manifest.Manifest + ) + + BeforeEach(func() { + cliConn = &pluginfakes.FakeCliConnection{} + resourcesData = &v3.ResourcesData{Connection: cliConn, Cli: cli.NewCli(cliConn, true)} + manifestFile, _ = manifest.ParseApplicationManifest("../../fixtures/manifest.yml", "") + }) + Describe("Test temp file generation without routes", func() { + It("app-name without prefix", func() { + noRouteYmlPath, err := resourcesData.GenerateNoRouteYml("my-test-application", manifestFile) + noRouteYml, errNoRouteYml := manifest.ParseApplicationManifest(noRouteYmlPath, "") + + Expect(err).ToNot(HaveOccurred()) + Expect(errNoRouteYml).ToNot(HaveOccurred()) + + Expect(len(manifestFile.ApplicationManifests[0].Routes)).To(Equal(2)) + Expect(len(noRouteYml.ApplicationManifests[0].Routes)).To(Equal(0)) + Expect(manifestFile.ApplicationManifests[0].DiskQuota).To(Equal(noRouteYml.ApplicationManifests[0].DiskQuota)) + Expect(manifestFile.ApplicationManifests[0].Instances).To(Equal(noRouteYml.ApplicationManifests[0].Instances)) + Expect(manifestFile.ApplicationManifests[0].Memory).To(Equal(noRouteYml.ApplicationManifests[0].Memory)) + }) + }) +}) From 0ccee5c807e3e614f2006ace3a0dc420d80463f8 Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 11:27:28 +0100 Subject: [PATCH 14/17] [UPDATE] error logging while uploading application --- puppeteer.go | 81 ++++++++++------------------------------------------ 1 file changed, 15 insertions(+), 66 deletions(-) diff --git a/puppeteer.go b/puppeteer.go index fa1f07d..9ec4daf 100644 --- a/puppeteer.go +++ b/puppeteer.go @@ -1,21 +1,15 @@ package main import ( - "context" - "fmt" - "log" - "os" - "strings" - "time" - - "code.cloudfoundry.org/cli/cf/api/logs" "code.cloudfoundry.org/cli/plugin" - "github.com/cloudfoundry/noaa/consumer" + "fmt" "github.com/happytobi/cf-puppeteer/arguments" "github.com/happytobi/cf-puppeteer/cf" v2 "github.com/happytobi/cf-puppeteer/cf/v2" "github.com/happytobi/cf-puppeteer/rewind" "github.com/happytobi/cf-puppeteer/ui" + "os" + "strings" ) func fatalIf(err error) { @@ -114,19 +108,15 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse }, //When upload fails the new application will be deleted and ven app will be renamed ReversePrevious: func() error { - ui.Failed("error while uploading / deploying the application... roll everything back") + ui.FailedMessage("error while uploading / deploying the application... roll everything back") _ = appRepo.v2Resources.DeleteApplication(parsedArguments.AppName) - return appRepo.v2Resources.RenameApplication(venName, parsedArguments.AppName) + _ = appRepo.v2Resources.RenameApplication(venName, parsedArguments.AppName) + return nil }, }, // start { Forward: func() error { - if parsedArguments.ShowLogs { - ui.Say("show logs...") - // TODO not working anymore - _ = appRepo.ShowLogs(parsedArguments.AppName) - } if parsedArguments.NoStart == false { return appRepo.v2Resources.StartApplication(parsedArguments.AppName) } @@ -142,7 +132,6 @@ func getActionsForApp(appRepo *ApplicationRepo, parsedArguments *arguments.Parse // If the app cannot start we'll have a lingering application // We delete this application so that the rename can succeed _ = appRepo.v2Resources.DeleteApplication(parsedArguments.AppName) - return appRepo.v2Resources.RenameApplication(venName, parsedArguments.AppName) }, }, @@ -202,7 +191,7 @@ func (plugin CfPuppeteerPlugin) Run(cliConnection plugin.CliConnection, args []s } var traceLogging bool - if os.Getenv("CF_PUPPETEER_TRACE") == "true" { + if os.Getenv("CF_TRACE") == "true" { traceLogging = true } appRepo := NewApplicationRepo(cliConnection, traceLogging) @@ -247,15 +236,14 @@ func (CfPuppeteerPlugin) GetMetadata() plugin.PluginMetadata { "-health-check-http-endpoint": "endpoint for the 'http' health check type", "-invocation-timeout": "timeout (in seconds) that controls individual health check invocations", "-show-crash-log": "Show recent logs when applications crashes while the deployment", - //"-show-app-log": "tail and show application log during application start", - "-process": "use health check type process", - "-legacy-push": "use legacy push instead of new v3 api", - "-no-route": "deploy new application without adding routes", - "-route-only": "only add routes from manifest to application", - "-no-start": "don't start application after deployment; venerable action will none", - "-docker-image": "docker image url", - "-docker-username": "docker repository username; used with password from env CF_DOCKER_PASSWORD", - "-vars-file": "path to a variable substitution file for manifest", + "-process": "use health check type process", + "-legacy-push": "use legacy push instead of new v3 api", + "-no-route": "deploy new application without adding routes", + "-route-only": "only add routes from manifest to application", + "-no-start": "don't start application after deployment; venerable action will none", + "-docker-image": "docker image url", + "-docker-username": "docker repository username; used with password from env CF_DOCKER_PASSWORD", + "-vars-file": "path to a variable substitution file for manifest", }, }, }, @@ -276,42 +264,3 @@ func NewApplicationRepo(conn plugin.CliConnection, traceLogging bool) *Applicati v2Resources: v2.NewV2Resources(conn, traceLogging), } } - -func (repo *ApplicationRepo) ShowLogs(appName string) error { - app, err := repo.conn.GetApp(appName) - if err != nil { - return err - } - - dopplerEndpoint, err := repo.conn.DopplerEndpoint() - if err != nil { - return err - } - token, err := repo.conn.AccessToken() - if err != nil { - return err - } - - cons := consumer.New(dopplerEndpoint, nil, nil) - defer cons.Close() - - messages, chanError := cons.TailingLogs(app.Guid, token) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - go func() { - for { - select { - case m := <-messages: - if m.GetSourceType() != "STG" { // skip STG messages as the cf tool already prints them - os.Stderr.WriteString(logs.NewNoaaLogMessage(m).ToLog(time.Local) + "\n") - } - case e := <-chanError: - log.Println("error reading logs:", e) - case <-ctx.Done(): - return - } - } - }() - return nil -} From be33262dc0d3d4a76ed27b3f544368fae2b1ffba Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 14:20:21 +0100 Subject: [PATCH 15/17] [UPDATE] move no route gen to manifest --- arguments/arguments.go | 4 ++- cf/v3/package.go | 17 ------------ cf/v3/package_test.go | 49 ---------------------------------- manifest/manifest.go | 55 +++++++++++++++++++++++++++++++------- manifest/manifest_test.go | 56 ++++++++++++++++++++++++++++++++------- 5 files changed, 95 insertions(+), 86 deletions(-) delete mode 100644 cf/v3/package.go delete mode 100644 cf/v3/package_test.go diff --git a/arguments/arguments.go b/arguments/arguments.go index cdc5a42..b65c5bb 100644 --- a/arguments/arguments.go +++ b/arguments/arguments.go @@ -16,6 +16,7 @@ import ( type ParserArguments struct { AppName string ManifestPath string + NoRouteManifestPath string AppPath string HealthCheckType string HealthCheckHTTPEndpoint string @@ -113,7 +114,7 @@ func ParseArgs(args []string) (*ParserArguments, error) { } //parse manifest - parsedManifest, err := manifest.ParseApplicationManifest(pta.ManifestPath, pta.VarsFile) + parsedManifest, noRouteManifestPath, err := manifest.ParseApplicationManifest(pta.ManifestPath, pta.VarsFile) if err != nil { return pta, err //ErrManifest } @@ -123,6 +124,7 @@ func ParseArgs(args []string) (*ParserArguments, error) { } pta.Manifest = parsedManifest + pta.NoRouteManifestPath = noRouteManifestPath //check if a docker image shouldbe pushed and verify passed args combination if len(pta.DockerUserName) > 0 && (len(dockerPass) == 0 || len(pta.DockerImage) == 0) { diff --git a/cf/v3/package.go b/cf/v3/package.go deleted file mode 100644 index a75aa7c..0000000 --- a/cf/v3/package.go +++ /dev/null @@ -1,17 +0,0 @@ -package v3 - -import ( - "fmt" - "os" - "strings" -) - -//GenerateTempFile generate path in temp directory bases on appName -func (resource *ResourcesData) GenerateTempFile(appName string, fileExtension string) (zipFile string) { - tempDir := strings.TrimSuffix(os.TempDir(), "/") - pathFormat := "%s/%s.%s" - if strings.HasPrefix(appName, "/") { - pathFormat = "%s%s.%s" - } - return fmt.Sprintf(pathFormat, tempDir, appName, fileExtension) -} diff --git a/cf/v3/package_test.go b/cf/v3/package_test.go deleted file mode 100644 index ca74e7b..0000000 --- a/cf/v3/package_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package v3_test - -import ( - "code.cloudfoundry.org/cli/plugin/pluginfakes" - "fmt" - "github.com/happytobi/cf-puppeteer/cf/cli" - "github.com/happytobi/cf-puppeteer/cf/v3" - "strings" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestCfPackageActions(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Puppeteer CF Package") -} - -var _ = Describe("cf-package test", func() { - var ( - cliConn *pluginfakes.FakeCliConnection - resourcesData *v3.ResourcesData - ) - - BeforeEach(func() { - cliConn = &pluginfakes.FakeCliConnection{} - resourcesData = &v3.ResourcesData{Connection: cliConn, Cli: cli.NewCli(cliConn, true)} - - }) - Describe("Test temp file generation", func() { - It("app-name without prefix", func() { - appName := "myApplication" - zipFile := resourcesData.GenerateTempFile(appName, "zip") - fmt.Printf("zipFilePath %s", zipFile) - Expect(strings.HasSuffix(zipFile, "/myApplication.zip")).To(Equal(true)) - Expect(strings.HasSuffix(zipFile, "//myApplication.zip")).To(Equal(false)) - }) - - It("app-name with prefix", func() { - appName := "/myApplication" - zipFile := resourcesData.GenerateTempFile(appName, "zip") - fmt.Printf("zipFilePath %s", zipFile) - Expect(strings.HasSuffix(zipFile, "/myApplication.zip")).To(Equal(true)) - Expect(strings.HasSuffix(zipFile, "//myApplication.zip")).To(Equal(false)) - - }) - }) -}) diff --git a/manifest/manifest.go b/manifest/manifest.go index a2ff1be..feb789c 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -2,8 +2,10 @@ package manifest import ( "fmt" + "github.com/pkg/errors" "gopkg.in/yaml.v2" "io/ioutil" + "os" "reflect" "regexp" "strings" @@ -47,21 +49,23 @@ var ( //ParseAndReplaceWithVars parse a manifest and vars file. // get all values from vars file and put them into the manifest file so there will be a returned new manifest without // placeholders -func ParseApplicationManifest(manifestFilePath string, varsFilePath string) (manifest Manifest, err error) { +func ParseApplicationManifest(manifestFilePath string, varsFilePath string) (manifest Manifest, noRouteManifestPath string, err error) { document, err := loadYmlFile(manifestFilePath) if err != nil || document.ApplicationManifests == nil { - return Manifest{}, fmt.Errorf("could not parse file, file not valid") + return Manifest{}, "", fmt.Errorf("could not parse file, file not valid") } //if there's no vars file, we can return the parsed manifest direct if len(varsFilePath) <= 0 { - return document, nil + //generate file when no vars file was passed + noRouteManifestPath, err = GenerateNoRouteYml(document) + return document, noRouteManifestPath, err } varsFile, err := loadVarsFile(varsFilePath) if err != nil { - return Manifest{}, fmt.Errorf("could not parse vars file, file not valid") + return Manifest{}, "", fmt.Errorf("could not parse vars file, file not valid") } //iterate through all the applications an check if vars are existing @@ -95,7 +99,13 @@ func ParseApplicationManifest(manifestFilePath string, varsFilePath string) (man document.ApplicationManifests[aI] = app } - return document, nil + //generate new temp file when vars parsed because placeholders are resolved + noRouteManifestPath, err = GenerateNoRouteYml(document) + if err != nil { + return Manifest{}, "", errors.Wrap(err, "could not generate no route manifest") + } + + return document, noRouteManifestPath, nil } //load the vars file an throw errors then there is a issue @@ -130,15 +140,40 @@ func loadYmlFile(manifestFilePath string) (manifest Manifest, err error) { } //WriteYmlFile write yml file to specified path and return them parsed -func WriteYmlFile(manifestFilePath string, manifest Manifest) (newManifest Manifest, err error) { +func WriteYmlFile(manifestFilePath string, manifest Manifest) (err error) { mManifest, err := yaml.Marshal(&manifest) if err != nil { - return Manifest{}, err + return err } bManifest := []byte(string(mManifest)) - err = ioutil.WriteFile(manifestFilePath, bManifest, 0644) + return ioutil.WriteFile(manifestFilePath, bManifest, 0644) +} + +//GenerateNoRouteYml generate temp manifest without routes to skip route creation +func GenerateNoRouteYml(originalManifest Manifest) (tempManifestPath string, err error) { + //Clone manifest to change them without side effects + newTempManifest := Manifest{ApplicationManifests: make([]Application, len(originalManifest.ApplicationManifests))} + + //copy important information into no route yml (only resources are important) + for index, app := range originalManifest.ApplicationManifests { + newApp := Application{Name: app.Name, Instances: app.Instances, Memory: app.Memory, DiskQuota: app.DiskQuota, NoRoute: true, Routes: []map[string]string{}} + newTempManifest.ApplicationManifests[index] = newApp + } + + manifestPathTemp := GenerateTempFile(originalManifest.ApplicationManifests[0].Name, "yml") + err = WriteYmlFile(manifestPathTemp, newTempManifest) + if err != nil { - return Manifest{}, err + return "", err + } + return manifestPathTemp, nil +} + +func GenerateTempFile(fileName string, fileExtension string) (zipFile string) { + tempDir := strings.TrimSuffix(os.TempDir(), "/") + pathFormat := "%s/%s.%s" + if strings.HasPrefix(fileName, "/") { + pathFormat = "%s%s.%s" } - return ParseApplicationManifest(manifestFilePath, "") + return fmt.Sprintf(pathFormat, tempDir, fileName, fileExtension) } diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 5231a52..ec35aa4 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -3,6 +3,7 @@ package manifest_test import ( "fmt" "os" + "strings" "testing" . "github.com/onsi/ginkgo" @@ -18,7 +19,7 @@ func TestManifestParser(t *testing.T) { var _ = Describe("Parse Manifest", func() { It("parses complete manifest", func() { - manifest, err := ParseApplicationManifest("../fixtures/manifest.yml", "") + manifest, _, err := ParseApplicationManifest("../fixtures/manifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("myApp")) Expect(manifest.ApplicationManifests[0].Buildpacks[0]).Should(Equal("java_buildpack")) @@ -29,14 +30,14 @@ var _ = Describe("Parse Manifest", func() { }) It("parses complete manifest with services", func() { - manifest, err := ParseApplicationManifest("../fixtures/manifest.yml", "") + manifest, _, err := ParseApplicationManifest("../fixtures/manifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("myApp")) Expect(manifest.ApplicationManifests[0].Services[0]).Should(Equal("service1")) Expect(manifest.ApplicationManifests[0].Services[1]).Should(Equal("service2")) }) It("parses complete manifest with buildpack url", func() { - manifest, err := ParseApplicationManifest("../fixtures/phpManifest.yml", "") + manifest, _, err := ParseApplicationManifest("../fixtures/phpManifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("appname")) Expect(manifest.ApplicationManifests[0].Services[0]).Should(Equal("ma-db")) @@ -50,7 +51,7 @@ var _ = Describe("Parse Manifest", func() { var _ = Describe("Parse multi Application Manifest", func() { It("parses complete manifest", func() { - manifest, err := ParseApplicationManifest("../fixtures/multiManifest.yml", "") + manifest, _, err := ParseApplicationManifest("../fixtures/multiManifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("myApp")) Expect(manifest.ApplicationManifests[1].Name).Should(Equal("myApp2")) @@ -59,7 +60,7 @@ var _ = Describe("Parse multi Application Manifest", func() { var _ = Describe("Parse invalid Application Manifest", func() { It("parses invalid manifest", func() { - manifest, err := ParseApplicationManifest("../fixtures/invalidManifest.yml", "") + manifest, _, err := ParseApplicationManifest("../fixtures/invalidManifest.yml", "") Expect(err).ShouldNot(BeNil()) Expect(manifest.ApplicationManifests).Should(BeNil()) }) @@ -67,7 +68,7 @@ var _ = Describe("Parse invalid Application Manifest", func() { var _ = Describe("Parse comp Manifest", func() { It("parses complicated manifest", func() { - manifest, err := ParseApplicationManifest("../fixtures/defaultMultiManifest.yml", "") + manifest, _, err := ParseApplicationManifest("../fixtures/defaultMultiManifest.yml", "") Expect(err).Should(BeNil()) Expect(manifest.ApplicationManifests).ShouldNot(BeNil()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal("app")) @@ -77,20 +78,22 @@ var _ = Describe("Parse comp Manifest", func() { var _ = Describe("Write new manifest", func() { It("write manifest file to specified path", func() { - manifest, err := ParseApplicationManifest("../fixtures/manifest.yml", "") + manifest, _, err := ParseApplicationManifest("../fixtures/manifest.yml", "") Expect(err).ShouldNot(HaveOccurred()) tempFile := fmt.Sprintf("%s%s", os.TempDir(), "testManifest.yml") - parsedTempManifest, err := WriteYmlFile(tempFile, manifest) + err = WriteYmlFile(tempFile, manifest) Expect(err).ShouldNot(HaveOccurred()) fmt.Printf("%s", tempFile) Expect(err).ShouldNot(HaveOccurred()) + parsedTempManifest, _, err := ParseApplicationManifest(tempFile, "") + Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Name).Should(Equal(parsedTempManifest.ApplicationManifests[0].Name)) }) }) var _ = Describe("Parse Manifest with vars-file", func() { It("parse manifest with valid vars-file", func() { - manifest, err := ParseApplicationManifest("../fixtures/manifest_vars.yml", "../fixtures/valid_vars_file.yml") + manifest, _, err := ParseApplicationManifest("../fixtures/manifest_vars.yml", "../fixtures/valid_vars_file.yml") Expect(err).ShouldNot(HaveOccurred()) Expect(manifest.ApplicationManifests[0].Memory).Should(Equal("1G")) Expect(manifest.ApplicationManifests[0].Instances).Should(Equal("2")) @@ -98,3 +101,38 @@ var _ = Describe("Parse Manifest with vars-file", func() { Expect(manifest.ApplicationManifests[0].Routes[1]["route"]).Should(Equal("myHost.internal.test.com")) }) }) + +var _ = Describe("Test temp file generation", func() { + It("app-name without prefix", func() { + appName := "myApplication" + zipFile := GenerateTempFile(appName, "zip") + fmt.Printf("zipFilePath %s", zipFile) + Expect(strings.HasSuffix(zipFile, "/myApplication.zip")).To(Equal(true)) + Expect(strings.HasSuffix(zipFile, "//myApplication.zip")).To(Equal(false)) + }) + + It("app-name with prefix", func() { + appName := "/myApplication" + zipFile := GenerateTempFile(appName, "zip") + fmt.Printf("zipFilePath %s", zipFile) + Expect(strings.HasSuffix(zipFile, "/myApplication.zip")).To(Equal(true)) + Expect(strings.HasSuffix(zipFile, "//myApplication.zip")).To(Equal(false)) + }) +}) + +var _ = Describe("Test temp file generation without routes", func() { + It("app-name without prefix", func() { + manifest, noRouteYmlPath, err := ParseApplicationManifest("../fixtures/manifest.yml", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(len(manifest.ApplicationManifests[0].Routes)).To(Equal(2)) + Expect(len(noRouteYmlPath)).ToNot(Equal(0)) + + noRouteYml, _, err := ParseApplicationManifest(noRouteYmlPath, "") + Expect(err).ToNot(HaveOccurred()) + Expect(len(noRouteYml.ApplicationManifests[0].Routes)).To(Equal(0)) + Expect(manifest.ApplicationManifests[0].DiskQuota).To(Equal(noRouteYml.ApplicationManifests[0].DiskQuota)) + Expect(manifest.ApplicationManifests[0].Instances).To(Equal(noRouteYml.ApplicationManifests[0].Instances)) + Expect(manifest.ApplicationManifests[0].Memory).To(Equal(noRouteYml.ApplicationManifests[0].Memory)) + }) +}) From c5d3e2c82825f1e647d315d86cd01b9fc8895b0c Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 14:21:32 +0100 Subject: [PATCH 16/17] [UPDATE] add routes lazy --- cf/v2/push_application.go | 6 +--- cf/v3/push_application.go | 28 +--------------- cf/v3/push_application_test.go | 58 +++++++++++++++++----------------- 3 files changed, 31 insertions(+), 61 deletions(-) diff --git a/cf/v2/push_application.go b/cf/v2/push_application.go index 0fdac74..4b715ec 100644 --- a/cf/v2/push_application.go +++ b/cf/v2/push_application.go @@ -33,7 +33,7 @@ func NewV2LegacyPush(conn plugin.CliConnection, traceLogging bool) *LegacyResour func (resource *LegacyResourcesData) PushApplication(parsedArguments *arguments.ParserArguments) error { ui.InfoMessage("Use legacy push") - args := []string{"push", parsedArguments.AppName, "-f", parsedArguments.ManifestPath, "--no-start"} + args := []string{"push", parsedArguments.AppName, "-f", parsedArguments.ManifestPath, "--no-start", "--no-route"} if parsedArguments.AppPath != "" { args = append(args, "-p", parsedArguments.AppPath) } @@ -51,10 +51,6 @@ func (resource *LegacyResourcesData) PushApplication(parsedArguments *arguments. args = append(args, "--no-start") } - if parsedArguments.NoRoute { - args = append(args, "--no-route") - } - ui.Say("start pushing application with arguments %s", args) err := resource.Executor.Execute(args) if err != nil { diff --git a/cf/v3/push_application.go b/cf/v3/push_application.go index 6ee9b91..77933e1 100644 --- a/cf/v3/push_application.go +++ b/cf/v3/push_application.go @@ -7,7 +7,6 @@ import ( "github.com/happytobi/cf-puppeteer/arguments" "github.com/happytobi/cf-puppeteer/cf/cli" v2 "github.com/happytobi/cf-puppeteer/cf/v2" - "github.com/happytobi/cf-puppeteer/manifest" "github.com/happytobi/cf-puppeteer/ui" "github.com/pkg/errors" "strconv" @@ -51,13 +50,8 @@ func (resource *ResourcesData) PushApplication(venAppName, spaceGUID string, par ui.Say("generate manifest without routes...") - manifestPath, err := resource.GenerateNoRouteYml(parsedArguments.AppName, parsedArguments.Manifest) - if err != nil { - return errors.Wrap(err, "could not generate a new temp manifest without routes") - } - ui.Say("apply manifest file") - err = resource.AssignAppManifest(manifestPath) + err = resource.AssignAppManifest(parsedArguments.NoRouteManifestPath) if err != nil { return err } @@ -83,26 +77,6 @@ func (resource *ResourcesData) SwitchRoutesOnly(venAppName string, appName strin return resource.SwitchRoutes(venAppName, appName, routes) } -//GenerateNoRouteYml generate temp manifest without routes to skip route creation -func (resource *ResourcesData) GenerateNoRouteYml(appName string, originalManifest manifest.Manifest) (newManifestPath string, err error) { - //Clone manifest to change them without side effects - newTempManifest := manifest.Manifest{ApplicationManifests: make([]manifest.Application, len(originalManifest.ApplicationManifests))} - - //copy important information into no route yml (only resources are important) - for index, app := range originalManifest.ApplicationManifests { - newApp := manifest.Application{Name: app.Name, Instances: app.Instances, Memory: app.Memory, DiskQuota: app.DiskQuota, NoRoute: true, Routes: []map[string]string{}} - newTempManifest.ApplicationManifests[index] = newApp - } - - manifestPathTemp := resource.GenerateTempFile(appName, "yml") - _, err = manifest.WriteYmlFile(manifestPathTemp, newTempManifest) - - if err != nil { - return "", err - } - return manifestPathTemp, nil -} - //SwitchRoutes add new routes and switch "old" one from venerable app to the one func (resource *ResourcesData) SwitchRoutes(venAppName string, appName string, routes []map[string]string) (err error) { domains, err := resource.GetDomain(routes) diff --git a/cf/v3/push_application_test.go b/cf/v3/push_application_test.go index 684d4a3..850f7e2 100644 --- a/cf/v3/push_application_test.go +++ b/cf/v3/push_application_test.go @@ -1,10 +1,6 @@ package v3_test import ( - "code.cloudfoundry.org/cli/plugin/pluginfakes" - "github.com/happytobi/cf-puppeteer/cf/cli" - "github.com/happytobi/cf-puppeteer/cf/v3" - manifest "github.com/happytobi/cf-puppeteer/manifest" "testing" . "github.com/onsi/ginkgo" @@ -17,30 +13,34 @@ func TestCfPushApplicationActions(t *testing.T) { } var _ = Describe("cf-push application test", func() { - var ( - cliConn *pluginfakes.FakeCliConnection - resourcesData *v3.ResourcesData - manifestFile manifest.Manifest - ) - - BeforeEach(func() { - cliConn = &pluginfakes.FakeCliConnection{} - resourcesData = &v3.ResourcesData{Connection: cliConn, Cli: cli.NewCli(cliConn, true)} - manifestFile, _ = manifest.ParseApplicationManifest("../../fixtures/manifest.yml", "") - }) - Describe("Test temp file generation without routes", func() { - It("app-name without prefix", func() { - noRouteYmlPath, err := resourcesData.GenerateNoRouteYml("my-test-application", manifestFile) - noRouteYml, errNoRouteYml := manifest.ParseApplicationManifest(noRouteYmlPath, "") - - Expect(err).ToNot(HaveOccurred()) - Expect(errNoRouteYml).ToNot(HaveOccurred()) - - Expect(len(manifestFile.ApplicationManifests[0].Routes)).To(Equal(2)) - Expect(len(noRouteYml.ApplicationManifests[0].Routes)).To(Equal(0)) - Expect(manifestFile.ApplicationManifests[0].DiskQuota).To(Equal(noRouteYml.ApplicationManifests[0].DiskQuota)) - Expect(manifestFile.ApplicationManifests[0].Instances).To(Equal(noRouteYml.ApplicationManifests[0].Instances)) - Expect(manifestFile.ApplicationManifests[0].Memory).To(Equal(noRouteYml.ApplicationManifests[0].Memory)) + /* + var ( + cliConn *pluginfakes.FakeCliConnection + resourcesData *v3.ResourcesData + manifestFile manifest.Manifest + ) + + BeforeEach(func() { + cliConn = &pluginfakes.FakeCliConnection{} + resourcesData = &v3.ResourcesData{Connection: cliConn, Cli: cli.NewCli(cliConn, true)} + manifestFile, _ = manifest.ParseApplicationManifest("../../fixtures/manifest.yml", "") }) - }) + Describe("Test temp file generation without routes", func() { + It("app-name without prefix", func() { + noRouteYmlPath, err := resourcesData.GenerateNoRouteYml("my-test-application", manifestFile) + noRouteYml, errNoRouteYml := manifest.ParseApplicationManifest(noRouteYmlPath, "") + + Expect(err).ToNot(HaveOccurred()) + Expect(errNoRouteYml).ToNot(HaveOccurred()) + + Expect(len(manifestFile.ApplicationManifests[0].Routes)).To(Equal(2)) + Expect(len(noRouteYml.ApplicationManifests[0].Routes)).To(Equal(0)) + Expect(manifestFile.ApplicationManifests[0].DiskQuota).To(Equal(noRouteYml.ApplicationManifests[0].DiskQuota)) + Expect(manifestFile.ApplicationManifests[0].Instances).To(Equal(noRouteYml.ApplicationManifests[0].Instances)) + Expect(manifestFile.ApplicationManifests[0].Memory).To(Equal(noRouteYml.ApplicationManifests[0].Memory)) + }) + }) + */ + + //TODO add more v2 push application tests }) From ccb86d2fd04fa6bd7b86481833b10243d0741bd5 Mon Sep 17 00:00:00 2001 From: HappyTobi Date: Mon, 24 Feb 2020 14:27:05 +0100 Subject: [PATCH 17/17] [UPDATE] README.md and CHANGELOG.md --- CHANGELOG.md | 15 ++++++++++----- README.md | 9 ++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66b2d50..6cae859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,23 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Upcoming features [1.x.x] -- add argument to cleanup unused environment variables - log deployment time - refactor push - add rewind to all push things - multi manifest pushed in parallel - multi manifest works correct. +- check space quota before deployment (if user pass the option) +- set timeout how long the deployment will wait for more / free space -## [1.2.0] - 2019-12-XX +## [1.2.0] - 2020-02-24 ### Added -- --vars-file argument was available again +- --vars-file argument was available again ### Changed - move all v2 operations into the v2 package - delete cloud controller version check -- all routes will be switched lazy (after starting up the new application) +- all routes will be switched lazy now (after starting up the new application) +- dropped CF_PUPPETEER_TRACE and replace them with CF_TRACE (read the README "Known issues section" for using it) - clean up code +- move temp manifest file generator to the manifest file +### Fixed +- error while passing environment variables on legacy push (--env) ## [1.1.3] - 2019-11-26 @@ -44,7 +49,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - documentation update - back port "--no-route" and "--route-only" option to "--legacy-push" (v2). - docker support was available again -- vendor-option droped +- vendor-option dropped - options was sorted right now (go 1.12) ### Fixed diff --git a/README.md b/README.md index 11aea47..9714c62 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ *cf plugin for hands-off, zero downtime application deploys* + + ## Documentation / Website [CF-Puppeteer Website](https://cf-puppeteer.happytobi.com). @@ -102,4 +104,9 @@ You can't push an Application-Manifest with a wildcard in tha `path` deceleratio That only works with the option `--legacy-push` To fix that issue, please remove the `path` from your manifest and pass the Artifact / Folder with the `-p` option. - \ No newline at end of file + +If there is a problem while pushing you application you can see the complete trace by setting + +Windows: `CF_TRACE=true` + +Unix: `CF_TRACE=/dev/stdout` \ No newline at end of file