From aa2c913308501901c1ecb4938a81de1db1bb3b1c Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Mon, 11 Nov 2024 20:34:52 -0500 Subject: [PATCH] impl(generator): configurable packages for Rust codec --- generator/cmd/main_test.go | 8 ++ generator/internal/genclient/genclient.go | 5 + .../language/internal/golang/golang.go | 4 + .../genclient/language/internal/rust/rust.go | 101 ++++++++++++++++-- .../language/internal/rust/rust_test.go | 95 +++++++++++++++- .../internal/genclient/language/language.go | 8 +- generator/internal/genclient/templatedata.go | 5 + generator/templates/rust/Cargo.toml.mustache | 6 +- .../testdata/rust/gclient/golden/Cargo.toml | 4 +- .../testdata/rust/openapi/golden/Cargo.toml | 4 +- 10 files changed, 214 insertions(+), 26 deletions(-) diff --git a/generator/cmd/main_test.go b/generator/cmd/main_test.go index bbcf191..ed41cc7 100644 --- a/generator/cmd/main_test.go +++ b/generator/cmd/main_test.go @@ -47,6 +47,10 @@ func TestRustFromOpenAPI(t *testing.T) { ProjectRoot: projectRoot, OutDir: "testdata/rust/openapi/golden", TemplateDir: "../templates", + Options: map[string]string{ + "package:gax_placeholder": "package=types,path=../../../../../types,source=google.protobuf", + "package:gax": "package=gax,path=../../../../../gax", + }, } err := Generate("openapi", popts, copts) if err != nil { @@ -81,6 +85,10 @@ func TestRustFromProtobuf(t *testing.T) { ProjectRoot: projectRoot, OutDir: "testdata/rust/gclient/golden", TemplateDir: "../templates", + Options: map[string]string{ + "package:gax_placeholder": "package=types,path=../../../../../types,source=google.protobuf", + "package:gax": "package=gax,path=../../../../../gax", + }, } err := Generate("protobuf", popts, copts) if err != nil { diff --git a/generator/internal/genclient/genclient.go b/generator/internal/genclient/genclient.go index 8819c30..5df29a9 100644 --- a/generator/internal/genclient/genclient.go +++ b/generator/internal/genclient/genclient.go @@ -38,6 +38,8 @@ type LanguageCodec interface { LoadWellKnownTypes(s *APIState) // FieldType returns a string representation of a message field type. FieldType(f *Field, state *APIState) string + // The name of a message type ID when used as an input or output argument + // in the client methods. MethodInOutTypeName(id string, state *APIState) string // The (unqualified) message name, as used when defining the type to // represent it. @@ -93,6 +95,9 @@ type LanguageCodec interface { // any ```-sections. Without this annotation Rustdoc assumes the // blockquote is an Rust code snippet and attempts to compile it. FormatDocComments(string) []string + // Returns a extra set of lines to insert in the module file. + // The format of these lines is specific to each language. + RequiredPackages() []string } // Parser converts an textual specification to a `genclient.API` object. diff --git a/generator/internal/genclient/language/internal/golang/golang.go b/generator/internal/genclient/language/internal/golang/golang.go index f58ad9a..06766cf 100644 --- a/generator/internal/genclient/language/internal/golang/golang.go +++ b/generator/internal/genclient/language/internal/golang/golang.go @@ -216,6 +216,10 @@ func (*Codec) FormatDocComments(documentation string) []string { return ss } +func (*Codec) RequiredPackages() []string { + return []string{} +} + // The list of Golang keywords and reserved words can be found at: // // https://go.dev/ref/spec#Keywords diff --git a/generator/internal/genclient/language/internal/rust/rust.go b/generator/internal/genclient/language/internal/rust/rust.go index 0037a46..0716f89 100644 --- a/generator/internal/genclient/language/internal/rust/rust.go +++ b/generator/internal/genclient/language/internal/rust/rust.go @@ -17,6 +17,7 @@ package rust import ( "fmt" "log/slog" + "sort" "strings" "unicode" @@ -24,11 +25,68 @@ import ( "github.com/iancoleman/strcase" ) -func NewCodec() *Codec { - return &Codec{} +func NewCodec(copts *genclient.CodecOptions) (*Codec, error) { + codec := &Codec{ + extraPackages: []*RustPackage{}, + packageMapping: map[string]*RustPackage{}, + } + for key, definition := range copts.Options { + if !strings.HasPrefix(key, "package:") { + continue + } + var specificationPackages []string + pkg := &RustPackage{ + Name: strings.TrimPrefix(key, "package:"), + } + for _, element := range strings.Split(definition, ",") { + s := strings.SplitN(element, "=", 2) + if len(s) != 2 { + return nil, fmt.Errorf("the definition for package %s should be a comma-separated list of key=value pairs, got=%q", key, definition) + } + switch s[0] { + case "package": + pkg.Package = s[1] + case "path": + pkg.Path = s[1] + case "version": + pkg.Version = s[1] + case "source": + specificationPackages = append(specificationPackages, s[1]) + default: + return nil, fmt.Errorf("unknown field (%s) in definition of rust package %s, got=%s", s[0], key, definition) + } + } + if pkg.Package == "" { + return nil, fmt.Errorf("missing rust package name for package %s, got=%s", key, definition) + } + codec.extraPackages = append(codec.extraPackages, pkg) + for _, source := range specificationPackages { + codec.packageMapping[source] = pkg + } + } + return codec, nil } -type Codec struct{} +type Codec struct { + // Additional Rust packages imported by this module. The Mustache template + // hardcodes a number of packages, but some are configured via the + // command-line. + extraPackages []*RustPackage + // A mapping between the specification package names (typically Protobuf), + // and the Rust package name that contains these types. + packageMapping map[string]*RustPackage +} + +type RustPackage struct { + // The name we import this package under. + Name string + // What the Rust package calls itself. + Package string + // The path to file the package locally, unused if empty. + Path string + // The version of the package, unused if empty. + Version string +} func (c *Codec) LoadWellKnownTypes(s *genclient.APIState) { // TODO(#77) - replace these placeholders with real types @@ -36,27 +94,27 @@ func (c *Codec) LoadWellKnownTypes(s *genclient.APIState) { { ID: ".google.protobuf.Any", Name: "Any", - Package: "gax_placeholder", + Package: "google.protobuf", }, { ID: ".google.protobuf.Empty", Name: "Empty", - Package: "gax_placeholder", + Package: "google.protobuf", }, { ID: ".google.protobuf.FieldMask", Name: "FieldMask", - Package: "gax_placeholder", + Package: "google.protobuf", }, { ID: ".google.protobuf.Duration", Name: "Duration", - Package: "gax_placeholder", + Package: "google.protobuf", }, { ID: ".google.protobuf.Timestamp", Name: "Timestamp", - Package: "gax_placeholder", + Package: "google.protobuf", }, } for _, message := range wellKnown { @@ -174,8 +232,12 @@ func (c *Codec) rustPackage(packageName string) string { if packageName == "" { return "crate::model" } - // TODO(#158) - this should be mapped via some configuration. - return packageName + mapped, ok := c.packageMapping[packageName] + if !ok { + slog.Error("unknown source package name", "name", packageName) + return packageName + } + return mapped.Name } func (c *Codec) MessageName(m *genclient.Message, state *genclient.APIState) string { @@ -369,6 +431,25 @@ func (*Codec) FormatDocComments(documentation string) []string { return ss } +func (c *Codec) RequiredPackages() []string { + lines := []string{} + for _, pkg := range c.extraPackages { + components := []string{} + if pkg.Version != "" { + components = append(components, fmt.Sprintf("version = %q", pkg.Version)) + } + if pkg.Path != "" { + components = append(components, fmt.Sprintf("path = %q", pkg.Path)) + } + if pkg.Package != "" { + components = append(components, fmt.Sprintf("package = %q", pkg.Package)) + } + lines = append(lines, fmt.Sprintf("%s = { %s }", pkg.Name, strings.Join(components, ", "))) + } + sort.Strings(lines) + return lines +} + // The list of Rust keywords and reserved words can be found at: // // https://doc.rust-lang.org/reference/keywords.html diff --git a/generator/internal/genclient/language/internal/rust/rust_test.go b/generator/internal/genclient/language/internal/rust/rust_test.go index 4daf5ed..890ca57 100644 --- a/generator/internal/genclient/language/internal/rust/rust_test.go +++ b/generator/internal/genclient/language/internal/rust/rust_test.go @@ -19,9 +19,94 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/googleapis/google-cloud-rust/generator/internal/genclient" ) +func testCodec() *Codec { + wkt := &RustPackage{ + Name: "gax_wkt", + Package: "types", + Path: "../../types", + } + + return &Codec{ + extraPackages: []*RustPackage{wkt}, + packageMapping: map[string]*RustPackage{ + "google.protobuf": wkt, + }, + } +} + +func TestParseOptions(t *testing.T) { + copts := &genclient.CodecOptions{ + Language: "rust", + Options: map[string]string{ + "package:gax_placeholder": "package=types,path=../../types,source=google.protobuf,source=test-only", + "package:gax": "package=gax,path=../../gax", + }, + } + codec, err := NewCodec(copts) + if err != nil { + t.Fatal(err) + } + want := &Codec{ + extraPackages: []*RustPackage{ + { + Name: "gax_placeholder", + Package: "types", + Path: "../../types", + }, + { + Name: "gax", + Package: "gax", + Path: "../../gax", + }, + }, + } + checkPackages(t, codec, want) + for _, source := range []string{"google.protobuf", "test-only"} { + pkg, ok := codec.packageMapping[source] + if !ok { + t.Errorf("missing package mapping for source=%s", source) + } + if pkg.Name != "gax_placeholder" { + t.Errorf("mismatched package for %s, want=%s, got=%s", source, "gax_placeholder", pkg.Name) + } + } +} + +func TestRequiredPackages(t *testing.T) { + copts := &genclient.CodecOptions{ + Language: "rust", + Options: map[string]string{ + "package:gax_placeholder": "package=types,path=../../types,source=google.protobuf,source=test-only", + "package:gax": "package=gax,path=../../gax,version=1.2.3", + }, + } + codec, err := NewCodec(copts) + if err != nil { + t.Fatal(err) + } + got := codec.RequiredPackages() + want := []string{ + "gax_placeholder = { path = \"../../types\", package = \"types\" }", + "gax = { version = \"1.2.3\", path = \"../../gax\", package = \"gax\" }", + } + less := func(a, b string) bool { return a < b } + if diff := cmp.Diff(want, got, cmpopts.SortSlices(less)); len(diff) > 0 { + t.Errorf("mismatched required packages (-want, +got):\n%s", diff) + } +} + +func checkPackages(t *testing.T, got *Codec, want *Codec) { + t.Helper() + less := func(a, b *RustPackage) bool { return a.Name < b.Name } + if diff := cmp.Diff(want.extraPackages, got.extraPackages, cmpopts.SortSlices(less)); len(diff) > 0 { + t.Errorf("package mismatch (-want, +got):\n%s", diff) + } +} + func TestWellKnownTypesExist(t *testing.T) { api := genclient.NewTestAPI([]*genclient.Message{}, []*genclient.Enum{}, []*genclient.Service{}) c := &Codec{} @@ -35,10 +120,10 @@ func TestWellKnownTypesExist(t *testing.T) { func TestWellKnownTypesAsMethod(t *testing.T) { api := genclient.NewTestAPI([]*genclient.Message{}, []*genclient.Enum{}, []*genclient.Service{}) - c := &Codec{} + c := testCodec() c.LoadWellKnownTypes(api.State) - want := "gax_placeholder::Empty" + want := "gax_wkt::Empty" got := c.MethodInOutTypeName(".google.protobuf.Empty", api.State) if want != got { t.Errorf("mismatched well-known type name as method argument or response, want=%s, got=%s", want, got) @@ -137,10 +222,10 @@ func TestFieldType(t *testing.T) { "f_int32_repeated": "Vec", "f_msg": "Option", "f_msg_repeated": "Vec", - "f_timestamp": "Option", - "f_timestamp_repeated": "Vec", + "f_timestamp": "Option", + "f_timestamp_repeated": "Vec", } - c := &Codec{} + c := testCodec() c.LoadWellKnownTypes(api.State) for _, field := range message.Fields { want, ok := expectedTypes[field.Name] diff --git a/generator/internal/genclient/language/language.go b/generator/internal/genclient/language/language.go index 4656af0..5ced8bb 100644 --- a/generator/internal/genclient/language/language.go +++ b/generator/internal/genclient/language/language.go @@ -22,12 +22,12 @@ import ( "github.com/googleapis/google-cloud-rust/generator/internal/genclient/language/internal/rust" ) -type createCodec func(*genclient.CodecOptions) genclient.LanguageCodec +type createCodec func(*genclient.CodecOptions) (genclient.LanguageCodec, error) func knownCodecs() map[string]createCodec { return map[string]createCodec{ - "rust": func(*genclient.CodecOptions) genclient.LanguageCodec { return rust.NewCodec() }, - "go": func(*genclient.CodecOptions) genclient.LanguageCodec { return golang.NewCodec() }, + "rust": func(copts *genclient.CodecOptions) (genclient.LanguageCodec, error) { return rust.NewCodec(copts) }, + "go": func(*genclient.CodecOptions) (genclient.LanguageCodec, error) { return golang.NewCodec(), nil }, } } @@ -37,5 +37,5 @@ func NewCodec(copts *genclient.CodecOptions) (genclient.LanguageCodec, error) { if !ok { return nil, fmt.Errorf("unknown language: %s", copts.Language) } - return create(copts), nil + return create(copts) } diff --git a/generator/internal/genclient/templatedata.go b/generator/internal/genclient/templatedata.go index 5a2e9af..d6100ec 100644 --- a/generator/internal/genclient/templatedata.go +++ b/generator/internal/genclient/templatedata.go @@ -46,10 +46,15 @@ func (t *templateData) Name() string { func (t *templateData) Title() string { return t.s.Title } + func (t *templateData) Description() string { return t.s.Description } +func (t *templateData) RequiredPackages() []string { + return t.c.RequiredPackages() +} + func (t *templateData) Services() []*service { return mapSlice(t.s.Services, func(s *Service) *service { return &service{ diff --git a/generator/templates/rust/Cargo.toml.mustache b/generator/templates/rust/Cargo.toml.mustache index f283160..1776e50 100644 --- a/generator/templates/rust/Cargo.toml.mustache +++ b/generator/templates/rust/Cargo.toml.mustache @@ -26,6 +26,6 @@ serde_json = "1.0.132" time = { version = "0.3.36", features = ["formatting", "parsing"] } reqwest = { version = "0.12.9", features = ["json"] } bytes = { version = "1.8.0", features = ["serde"] } -{{! TODO(#93) - this path should be configurable for local development }} -gax_placeholder = { path = "../../../../../types", package="types" } -gax = { path = "../../../../../gax", package="gax" } +{{#RequiredPackages}} +{{{.}}} +{{/RequiredPackages}} diff --git a/generator/testdata/rust/gclient/golden/Cargo.toml b/generator/testdata/rust/gclient/golden/Cargo.toml index 53fbbee..d7f4dbd 100755 --- a/generator/testdata/rust/gclient/golden/Cargo.toml +++ b/generator/testdata/rust/gclient/golden/Cargo.toml @@ -10,5 +10,5 @@ serde_json = "1.0.132" time = { version = "0.3.36", features = ["formatting", "parsing"] } reqwest = { version = "0.12.9", features = ["json"] } bytes = { version = "1.8.0", features = ["serde"] } -gax_placeholder = { path = "../../../../../types", package="types" } -gax = { path = "../../../../../gax", package="gax" } +gax = { path = "../../../../../gax", package = "gax" } +gax_placeholder = { path = "../../../../../types", package = "types" } diff --git a/generator/testdata/rust/openapi/golden/Cargo.toml b/generator/testdata/rust/openapi/golden/Cargo.toml index a58bad6..334f7d1 100755 --- a/generator/testdata/rust/openapi/golden/Cargo.toml +++ b/generator/testdata/rust/openapi/golden/Cargo.toml @@ -10,5 +10,5 @@ serde_json = "1.0.132" time = { version = "0.3.36", features = ["formatting", "parsing"] } reqwest = { version = "0.12.9", features = ["json"] } bytes = { version = "1.8.0", features = ["serde"] } -gax_placeholder = { path = "../../../../../types", package="types" } -gax = { path = "../../../../../gax", package="gax" } +gax = { path = "../../../../../gax", package = "gax" } +gax_placeholder = { path = "../../../../../types", package = "types" }