Generate YAML from Go struct #60
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,7 +17,7 @@ import ( | |
| ) | ||
|
|
||
| type Config struct { | ||
| Version int `yaml:"version"` | ||
|
|
||
| Cache CacheConfig `yaml:"cache"` | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,201 @@ | ||
| // Copyright (c) Bartłomiej Płotka @bwplotka | ||
| // Licensed under the Apache License 2.0. | ||
|
|
||
| package yamlgen | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "go/ast" | ||
| "go/parser" | ||
| "go/token" | ||
| "go/types" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
| "regexp" | ||
| "strings" | ||
|
|
||
| "github.com/dave/jennifer/jen" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| // TODO(saswatamcode): Add tests. | ||
| // TODO(saswatamcode): Check jennifer code for some safety. | ||
| // TODO(saswatamcode): Add mechanism for caching output from generated code. | ||
| // TODO(saswatamcode): Currently takes file names, need to make it module based(something such as https://golang.org/pkg/cmd/go/internal/list/). | ||
|
|
||
| // GenGoCode generates Go code for yaml gen from structs in src file. | ||
| func GenGoCode(src []byte) (string, error) { | ||
| // Create new main file. | ||
| fset := token.NewFileSet() | ||
| generatedCode := jen.NewFile("main") | ||
| // Don't really need to format here, saves time. | ||
| // generatedCode.NoFormat = true | ||
|
|
||
| // Parse source file. | ||
| f, err := parser.ParseFile(fset, "", src, parser.AllErrors) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| // Init statements for structs. | ||
| var init []jen.Code | ||
| // Declare config map, i.e, `configs := map[string]interface{}{}`. | ||
| init = append(init, jen.Id("configs").Op(":=").Map(jen.String()).Interface().Values()) | ||
|
|
||
| // Loop through declarations in file. | ||
| for _, decl := range f.Decls { | ||
|
There was a problem hiding this comment. I think I would expect some recursion here (: |
||
| // Cast to generic declaration node. | ||
| if genericDecl, ok := decl.(*ast.GenDecl); ok { | ||
| // Check if declaration spec is `type`. | ||
| if typeDecl, ok := genericDecl.Specs[0].(*ast.TypeSpec); ok { | ||
| // Cast to `type struct`. | ||
| structDecl, ok := typeDecl.Type.(*ast.StructType) | ||
| if !ok { | ||
| generatedCode.Type().Id(typeDecl.Name.Name).Id(string(src[typeDecl.Type.Pos()-1 : typeDecl.Type.End()-1])) | ||
| continue | ||
| } | ||
|
|
||
| var structFields []jen.Code | ||
| arrayInit := make(jen.Dict) | ||
|
|
||
| // Loop and generate fields for each field. | ||
| fields := structDecl.Fields.List | ||
| for _, field := range fields { | ||
| // Each field might have multiple names. | ||
| names := field.Names | ||
| for _, n := range names { | ||
| if n.IsExported() { | ||
| pos := n.Obj.Decl.(*ast.Field) | ||
|
|
||
| // Copy struct field to generated code, with imports in case of other package imported field. | ||
| if pos.Tag != nil { | ||
| typeStr := string(src[pos.Type.Pos()-1 : pos.Type.End()-1]) | ||
| // Check if field is imported from other package. | ||
| if strings.Contains(typeStr, ".") { | ||
| typeArr := strings.SplitN(typeStr, ".", 2) | ||
| // Match the import name with the import statement. | ||
| for _, s := range f.Imports { | ||
| moduleName := "" | ||
| // Choose to copy same alias as in source file or use package name. | ||
| if s.Name == nil { | ||
| _, moduleName = filepath.Split(s.Path.Value[1 : len(s.Path.Value)-1]) | ||
| generatedCode.ImportName(s.Path.Value[1:len(s.Path.Value)-1], moduleName) | ||
| } else { | ||
| moduleName = s.Name.String() | ||
| generatedCode.ImportAlias(s.Path.Value[1:len(s.Path.Value)-1], moduleName) | ||
| } | ||
| // Add field to struct only if import name matches. | ||
| if moduleName == typeArr[0] { | ||
| structFields = append(structFields, jen.Id(n.Name).Qual(s.Path.Value[1:len(s.Path.Value)-1], typeArr[1]).Id(pos.Tag.Value)) | ||
| } | ||
| } | ||
| } else { | ||
| structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(pos.Tag.Value)) | ||
| } | ||
| } | ||
|
|
||
| // Check if field is a slice type. | ||
| sliceRe := regexp.MustCompile(`^\[.*\].*$`) | ||
| typeStr := types.ExprString(field.Type) | ||
| if sliceRe.MatchString(typeStr) { | ||
| iArr := "[]int" | ||
| fArr := "[]float" | ||
| cArr := "[]complex" | ||
| uArr := "[]uint" | ||
| switch typeStr { | ||
| case "[]bool", "[]string", "[]byte", "[]rune", | ||
| iArr, iArr + "8", iArr + "16", iArr + "32", iArr + "64", | ||
| fArr + "32", fArr + "64", | ||
| cArr + "64", cArr + "128", | ||
| uArr, uArr + "8", uArr + "16", uArr + "32", uArr + "64", uArr + "ptr": | ||
| arrayInit[jen.Id(n.Name)] = jen.Id(typeStr + "{}") | ||
| default: | ||
| arrayInit[jen.Id(n.Name)] = jen.Id(string(src[pos.Type.Pos()-1 : pos.Type.End()-1])).Values(jen.Id(string(src[pos.Type.Pos()+1 : pos.Type.End()-1])).Values()) | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Add initialize statements for struct via code like `configs["Type"] = Type{}`. | ||
| // If struct has array members, use array initializer via code like `configs["Config"] = Config{ArrayMember: []Type{Type{}}}`. | ||
| init = append(init, jen.Id("configs").Index(jen.Lit(typeDecl.Name.Name)).Op("=").Id(typeDecl.Name.Name).Values(arrayInit)) | ||
|
|
||
| // Finally put struct inside generated code. | ||
| generatedCode.Type().Id(typeDecl.Name.Name).Struct(structFields...) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Add for loop to iterate through map and return config YAML. | ||
| init = append(init, jen.For( | ||
| jen.List(jen.Id("k"), jen.Id("config")).Op(":=").Range().Id("configs"), | ||
| ).Block( | ||
| // We import the cfggen Generate method directly to generate output. | ||
| jen.Qual("fmt", "Println").Call(jen.Lit("---")), | ||
| jen.Qual("fmt", "Println").Call(jen.Id("k")), | ||
| // TODO(saswatamcode): Replace with import from mdox itself once merged. | ||
| jen.Qual("github.com/bwplotka/mdox/pkg/yamlgen", "Generate").Call(jen.Id("config"), jen.Qual("os", "Stderr")), | ||
| )) | ||
|
|
||
| // Generate main function in new module. | ||
| generatedCode.Func().Id("main").Params().Block(init...) | ||
| return generatedCode.GoString(), nil | ||
| } | ||
|
|
||
| // execGoCode executes and returns output from generated Go code. | ||
| func ExecGoCode(ctx context.Context, mainGo string) ([]byte, error) { | ||
| tmpDir, err := os.MkdirTemp("", "structgen") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer os.RemoveAll(tmpDir) | ||
|
|
||
| // Copy generated code to main.go. | ||
| main, err := os.Create(filepath.Join(tmpDir, "main.go")) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer main.Close() | ||
|
|
||
| _, err = main.Write([]byte(mainGo)) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // Create go.mod in temp dir. | ||
| cmd := exec.CommandContext(ctx, "go", "mod", "init", "structgen") | ||
| cmd.Dir = tmpDir | ||
| if err := cmd.Run(); err != nil { | ||
| return nil, errors.Wrapf(err, "mod init %v", cmd) | ||
| } | ||
|
|
||
| // Replace for unreleased mdox yamlgen so don't need to copy cfggen code to new dir and compile. | ||
| // Currently in github.com/saswatamcode/[email protected]. Replace once #79 is merged. | ||
| cmd = exec.CommandContext(ctx, "go", "mod", "edit", "-replace", "github.com/bwplotka/mdox=github.com/saswatamcode/[email protected]") | ||
| cmd.Dir = tmpDir | ||
| if err := cmd.Run(); err != nil { | ||
| return nil, errors.Wrapf(err, "mod edit %v", cmd) | ||
| } | ||
|
|
||
| // Import required packages(generate go.sum). | ||
| cmd = exec.CommandContext(ctx, "go", "mod", "tidy") | ||
| cmd.Dir = tmpDir | ||
| if err := cmd.Run(); err != nil { | ||
| return nil, errors.Wrapf(err, "mod tidy %v", cmd) | ||
| } | ||
|
|
||
| // Execute generate code and return output. | ||
| b := bytes.Buffer{} | ||
| cmd = exec.CommandContext(ctx, "go", "run", "main.go") | ||
| cmd.Dir = tmpDir | ||
| cmd.Stderr = &b | ||
| cmd.Stdout = &b | ||
| if err := cmd.Run(); err != nil { | ||
| return nil, errors.Wrapf(err, "run %v out %v", cmd, b.String()) | ||
| } | ||
|
|
||
| return b.Bytes(), nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| // Copyright (c) Bartłomiej Płotka @bwplotka | ||
| // Licensed under the Apache License 2.0. | ||
|
|
||
| package yamlgen | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/efficientgo/core/testutil" | ||
| "golang.org/x/net/context" | ||
| ) | ||
|
|
||
| func TestYAMLGen_GenGoCode(t *testing.T) { | ||
| t.Run("normal struct", func(t *testing.T) { | ||
| source := []byte("package main\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n") | ||
| generatedCode, err := GenGoCode(source) | ||
| testutil.Ok(t, err) | ||
|
|
||
| expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"TestConfig\"] = TestConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" | ||
| testutil.Equals(t, expected, generatedCode) | ||
| }) | ||
|
|
||
| t.Run("struct with unexported field", func(t *testing.T) { | ||
| source := []byte("package main\n\nimport \"regexp\"\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n\n\tr *regexp.Regexp\n}\n") | ||
|
There was a problem hiding this comment. I would use ` and add + backtick + , something like that. Still easier than long string manually crafted. Another idea - put all in files - that would do as well. |
||
| generatedCode, err := GenGoCode(source) | ||
| testutil.Ok(t, err) | ||
|
|
||
| expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" | ||
| testutil.Equals(t, expected, generatedCode) | ||
| }) | ||
|
|
||
| t.Run("struct with array fields", func(t *testing.T) { | ||
| source := []byte("package main\n\nimport \"regexp\"\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n\n\tr *regexp.Regexp\n}\n\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n") | ||
| generatedCode, err := GenGoCode(source) | ||
| testutil.Ok(t, err) | ||
|
|
||
| expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"Config\"] = Config{\n\t\tIgnore: []IgnoreConfig{IgnoreConfig{}},\n\t\tValidator: []ValidatorConfig{ValidatorConfig{}},\n\t}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tconfigs[\"IgnoreConfig\"] = IgnoreConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" | ||
| testutil.Equals(t, expected, generatedCode) | ||
| }) | ||
| } | ||
|
|
||
| func TestYAMLGen_ExecGoCode(t *testing.T) { | ||
| t.Run("normal struct", func(t *testing.T) { | ||
| generatedCode := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"TestConfig\"] = TestConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" | ||
| output, err := ExecGoCode(context.TODO(), generatedCode) | ||
| testutil.Ok(t, err) | ||
|
|
||
| expected := "---\nTestConfig\nurl: \"\"\nid: 0\ntoken: \"\"\n" | ||
| testutil.Equals(t, expected, string(output)) | ||
| }) | ||
|
|
||
| t.Run("struct with array fields", func(t *testing.T) { | ||
| generatedCode := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"Config\"] = Config{\n\t\tIgnore: []IgnoreConfig{IgnoreConfig{}},\n\t\tValidator: []ValidatorConfig{ValidatorConfig{}},\n\t}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tconfigs[\"IgnoreConfig\"] = IgnoreConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" | ||
| output, err := ExecGoCode(context.TODO(), generatedCode) | ||
| testutil.Ok(t, err) | ||
|
|
||
| expected := "---\nConfig\nversion: 0\nvalidators:\n - type: \"\"\n regex: \"\"\n token: \"\"\nignore:\n - url: \"\"\n id: 0\n token: \"\"\n---\nValidatorConfig\ntype: \"\"\nregex: \"\"\ntoken: \"\"\n---\nIgnoreConfig\nurl: \"\"\nid: 0\ntoken: \"\"\n" | ||
| testutil.Equals(t, expected, string(output)) | ||
| }) | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good for start, but I wonder if we can scope to just one desired structure, to limit imports. Maybe it's later optimization - so fine for now
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we add todo for that?