From 11165f199582364d6132b9ec736f2e81b87cf032 Mon Sep 17 00:00:00 2001 From: switchupcb Date: Sun, 7 Aug 2022 04:51:17 +0200 Subject: [PATCH 1/5] feat: support for Go modules to imports --- README.md | 5 ++ go.mod | 7 +++ go.sum | 6 ++ interp/doc.go | 3 +- interp/interp.go | 4 +- interp/src.go | 154 ++++++++------------------------------------- interp/src_test.go | 69 +------------------- 7 files changed, 52 insertions(+), 196 deletions(-) create mode 100644 go.sum diff --git a/README.md b/README.md index 3a87490af..ddcad7fd4 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ It powers executable Go scripts and plugins, in embedded interpreters or interac * Complete support of [Go specification][specs] * Written in pure Go, using only the standard library * Simple interpreter API: `New()`, `Eval()`, `Use()` +* Supports Go Modules * Works everywhere Go works * All Go & runtime resources accessible from script (with control) * Security: `unsafe` and `syscall` packages neither used nor exported by default @@ -114,6 +115,10 @@ func main() { [Go Playground](https://play.golang.org/p/WvwH4JqrU-p) +### Use Go Modules + +TODO + ### As a command-line interpreter The Yaegi command can run an interactive Read-Eval-Print-Loop: diff --git a/go.mod b/go.mod index d4a2c9c0f..f141d0c98 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,10 @@ module github.com/traefik/yaegi go 1.18 + +require golang.org/x/tools v0.1.12 + +require ( + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..4965383ee --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/interp/doc.go b/interp/doc.go index 2253eca5d..1c7c6aec9 100644 --- a/interp/doc.go +++ b/interp/doc.go @@ -9,8 +9,7 @@ Importing packages Packages can be imported in source or binary form, using the standard Go import statement. In source form, packages are searched first in the vendor directory, the preferred way to store source dependencies. If not -found in vendor, sources modules will be searched in GOPATH. Go modules -are not supported yet by yaegi. +found in vendor, sources modules will be searched in GOPATH. Binary form packages are compiled and linked with the interpreter executable, and exposed to scripts with the Use method. The extract diff --git a/interp/interp.go b/interp/interp.go index d323f5b64..73b23cfcc 100644 --- a/interp/interp.go +++ b/interp/interp.go @@ -328,7 +328,9 @@ type Options struct { // New returns a new interpreter. func New(options Options) *Interpreter { i := Interpreter{ - opt: opt{context: build.Default, filesystem: &realFS{}, env: map[string]string{}}, + opt: opt{ + context: build.Default, filesystem: &realFS{}, env: make(map[string]string), + }, frame: newFrame(nil, 0, 0), fset: token.NewFileSet(), universe: initUniverse(), diff --git a/interp/src.go b/interp/src.go index f7ccfad87..7186c1b4e 100644 --- a/interp/src.go +++ b/interp/src.go @@ -3,12 +3,13 @@ package interp import ( "fmt" "io/fs" - "os" "path/filepath" "strings" + + "golang.org/x/tools/go/packages" ) -// importSrc calls gta on the source code for the package identified by +// importSrc calls global tag analysis on the source code for the package identified by // importPath. rPath is the relative path to the directory containing the source // code for the package. It can also be "main" as a special value. func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (string, error) { @@ -23,24 +24,9 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s return name, nil } - // For relative import paths in the form "./xxx" or "../xxx", the initial - // base path is the directory of the interpreter input file, or "." if no file - // was provided. - // In all other cases, absolute import paths are resolved from the GOPATH - // and the nested "vendor" directories. - if isPathRelative(importPath) { - if rPath == mainID { - rPath = "." - } - dir = filepath.Join(filepath.Dir(interp.name), rPath, importPath) - } else if dir, rPath, err = interp.pkgDir(interp.context.GOPATH, rPath, importPath); err != nil { - // Try again, assuming a root dir at the source location. - if rPath, err = interp.rootFromSourceLocation(); err != nil { - return "", err - } - if dir, rPath, err = interp.pkgDir(interp.context.GOPATH, rPath, importPath); err != nil { - return "", err - } + // resolve relative and absolute import paths. + if dir, err = interp.getPackageDir(importPath); err != nil { + return "", err } if interp.rdir[importPath] { @@ -171,119 +157,39 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s return pkgName, nil } -// rootFromSourceLocation returns the path to the directory containing the input -// Go file given to the interpreter, relative to $GOPATH/src. -// It is meant to be called in the case when the initial input is a main package. -func (interp *Interpreter) rootFromSourceLocation() (string, error) { - sourceFile := interp.name - if sourceFile == DefaultSourceName { - return "", nil - } - wd, err := os.Getwd() +// getPackageDir uses the provided Go module environment variables to find the absolute path of an import path. +func (interp *Interpreter) getPackageDir(importPath string) (string, error) { + absImportPath, err := filepath.Abs(importPath) if err != nil { - return "", err - } - pkgDir := filepath.Join(wd, filepath.Dir(sourceFile)) - root := strings.TrimPrefix(pkgDir, filepath.Join(interp.context.GOPATH, "src")+"/") - if root == wd { - return "", fmt.Errorf("package location %s not in GOPATH", pkgDir) - } - return root, nil -} - -// pkgDir returns the absolute path in filesystem for a package given its import path -// and the root of the subtree dependencies. -func (interp *Interpreter) pkgDir(goPath string, root, importPath string) (string, string, error) { - rPath := filepath.Join(root, "vendor") - dir := filepath.Join(goPath, "src", rPath, importPath) - - if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil { - return dir, rPath, nil // found! + return "", fmt.Errorf("an error occurred determining the absolute path of import path %v: %w", importPath, err) } - dir = filepath.Join(goPath, "src", effectivePkg(root, importPath)) - - if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil { - return dir, root, nil // found! - } - - if len(root) == 0 { - if interp.context.GOPATH == "" { - return "", "", fmt.Errorf("unable to find source related to: %q. Either the GOPATH environment variable, or the Interpreter.Options.GoPath needs to be set", importPath) - } - return "", "", fmt.Errorf("unable to find source related to: %q", importPath) + config := packages.Config{ + Env: []string{ + "GOPATH=" + interp.context.GOPATH, + "GOCACHE=" + interp.opt.env["GOCACHE"], + "GOROOT=" + interp.opt.env["GOROOT"], + "GOPRIVATE=" + interp.opt.env["GOPRIVATE"], + "GOMODCACHE=" + interp.opt.env["GOMODCACHE"], + "GO111MODULE=" + interp.opt.env["GO111MODULE"], + }, } - rootPath := filepath.Join(goPath, "src", root) - prevRoot, err := previousRoot(interp.opt.filesystem, rootPath, root) + pkgs, err := packages.Load(&config, absImportPath) if err != nil { - return "", "", err + return "", fmt.Errorf("an error occurred retrieving a package: %v\n%v\nIf Access is denied, run in administrator", absImportPath, err) } - return interp.pkgDir(goPath, prevRoot, importPath) -} - -const vendor = "vendor" - -// Find the previous source root (vendor > vendor > ... > GOPATH). -func previousRoot(filesystem fs.FS, rootPath, root string) (string, error) { - rootPath = filepath.Clean(rootPath) - parent, final := filepath.Split(rootPath) - parent = filepath.Clean(parent) - - // TODO(mpl): maybe it works for the special case main, but can't be bothered for now. - if root != mainID && final != vendor { - root = strings.TrimSuffix(root, string(filepath.Separator)) - prefix := strings.TrimSuffix(strings.TrimSuffix(rootPath, root), string(filepath.Separator)) - - // look for the closest vendor in one of our direct ancestors, as it takes priority. - var vendored string - for { - fi, err := fs.Stat(filesystem, filepath.Join(parent, vendor)) - if err == nil && fi.IsDir() { - vendored = strings.TrimPrefix(strings.TrimPrefix(parent, prefix), string(filepath.Separator)) - break - } - if !os.IsNotExist(err) { - return "", err - } - - // stop when we reach GOPATH/src/blah - parent = filepath.Dir(parent) - if parent == prefix { - break + // confirm the import path is found. + for _, pkg := range pkgs { + for _, goFile := range pkg.GoFiles { + if strings.Contains(filepath.Dir(goFile), pkg.Name) { + return filepath.Dir(goFile), nil } - - // just an additional failsafe, stop if we reach the filesystem root, or dot (if - // we are dealing with relative paths). - // TODO(mpl): It should probably be a critical error actually, - // as we shouldn't have gone that high up in the tree. - if parent == string(filepath.Separator) || parent == "." { - break - } - } - - if vendored != "" { - return vendored, nil } } - // TODO(mpl): the algorithm below might be redundant with the one above, - // but keeping it for now. Investigate/simplify/remove later. - splitRoot := strings.Split(root, string(filepath.Separator)) - var index int - for i := len(splitRoot) - 1; i >= 0; i-- { - if splitRoot[i] == "vendor" { - index = i - break - } - } - - if index == 0 { - return "", nil - } - - return filepath.Join(splitRoot[:index]...), nil + return "", fmt.Errorf("an import source could not be found: %q. Did you set the required environment variable in the Interpreter.Options?", absImportPath) } func effectivePkg(root, path string) string { @@ -313,9 +219,3 @@ func effectivePkg(root, path string) string { return filepath.Join(root, frag) } - -// isPathRelative returns true if path starts with "./" or "../". -// It is intended for use on import paths, where "/" is always the directory separator. -func isPathRelative(s string) bool { - return strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") -} diff --git a/interp/src_test.go b/interp/src_test.go index 9944c23af..b4848b34f 100644 --- a/interp/src_test.go +++ b/interp/src_test.go @@ -183,7 +183,7 @@ func Test_pkgDir(t *testing.T) { } } - dir, rPath, err := interp.pkgDir(goPath, test.root, test.path) + dir, err := interp.getPackageDir(test.path) if err != nil { t.Fatal(err) } @@ -192,71 +192,8 @@ func Test_pkgDir(t *testing.T) { t.Errorf("[dir] got: %s, want: %s", dir, test.expected.dir) } - if rPath != test.expected.rpath { - t.Errorf(" [rpath] got: %s, want: %s", rPath, test.expected.rpath) - } - }) - } -} - -func Test_previousRoot(t *testing.T) { - testCases := []struct { - desc string - root string - rootPathSuffix string - expected string - }{ - { - desc: "GOPATH", - root: "github.com/foo/pkg/", - expected: "", - }, - { - desc: "vendor level 1", - root: "github.com/foo/pkg/vendor/guthib.com/traefik/fromage", - expected: "github.com/foo/pkg", - }, - { - desc: "vendor level 2", - root: "github.com/foo/pkg/vendor/guthib.com/traefik/fromage/vendor/guthib.com/traefik/fuu", - expected: "github.com/foo/pkg/vendor/guthib.com/traefik/fromage", - }, - { - desc: "vendor is sibling", - root: "github.com/foo/bar", - rootPathSuffix: "testdata/src/github.com/foo/bar", - expected: "github.com/foo", - }, - { - desc: "vendor is uncle", - root: "github.com/foo/bar/baz", - rootPathSuffix: "testdata/src/github.com/foo/bar/baz", - expected: "github.com/foo", - }, - } - - for _, test := range testCases { - test := test - t.Run(test.desc, func(t *testing.T) { - t.Parallel() - - var rootPath string - if test.rootPathSuffix != "" { - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - rootPath = filepath.Join(wd, test.rootPathSuffix) - } else { - rootPath = vendor - } - p, err := previousRoot(&realFS{}, rootPath, test.root) - if err != nil { - t.Error(err) - } - - if p != test.expected { - t.Errorf("got: %s, want: %s", p, test.expected) + if test.root != test.expected.rpath { + t.Errorf(" [rpath] got: %s, want: %s", test.root, test.expected.rpath) } }) } From 6f296d9b1f7e149087f5ad926de52012b598729b Mon Sep 17 00:00:00 2001 From: switchupcb Date: Tue, 9 Aug 2022 21:58:32 -0500 Subject: [PATCH 2/5] resolve relative GOPATH --- interp/src.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/interp/src.go b/interp/src.go index 7186c1b4e..ef1330b24 100644 --- a/interp/src.go +++ b/interp/src.go @@ -159,16 +159,24 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s // getPackageDir uses the provided Go module environment variables to find the absolute path of an import path. func (interp *Interpreter) getPackageDir(importPath string) (string, error) { + // ensure that an absolute import path is used. absImportPath, err := filepath.Abs(importPath) if err != nil { - return "", fmt.Errorf("an error occurred determining the absolute path of import path %v: %w", importPath, err) + return "", fmt.Errorf("an error occurred determining the absolute path of import path %q: %w", importPath, err) } + // ensure that an absolute GOPATH is used. + absGoPath, err := filepath.Abs(interp.context.GOPATH) + if err != nil { + return "", fmt.Errorf("an error occurred determining the absolute path of a GOPATH %q: %w", interp.context.GOPATH, err) + } + + // load the package. config := packages.Config{ Env: []string{ - "GOPATH=" + interp.context.GOPATH, + "GOPATH=" + absGoPath, "GOCACHE=" + interp.opt.env["GOCACHE"], - "GOROOT=" + interp.opt.env["GOROOT"], + "GOROOT=" + interp.context.GOROOT, "GOPRIVATE=" + interp.opt.env["GOPRIVATE"], "GOMODCACHE=" + interp.opt.env["GOMODCACHE"], "GO111MODULE=" + interp.opt.env["GO111MODULE"], From 2dae3c261a736f717cd2600d937d07053e09318b Mon Sep 17 00:00:00 2001 From: switchupcb Date: Tue, 9 Aug 2022 21:59:21 -0500 Subject: [PATCH 3/5] revert absolute import path resolution --- interp/src.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/interp/src.go b/interp/src.go index ef1330b24..f3652cc44 100644 --- a/interp/src.go +++ b/interp/src.go @@ -159,12 +159,6 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s // getPackageDir uses the provided Go module environment variables to find the absolute path of an import path. func (interp *Interpreter) getPackageDir(importPath string) (string, error) { - // ensure that an absolute import path is used. - absImportPath, err := filepath.Abs(importPath) - if err != nil { - return "", fmt.Errorf("an error occurred determining the absolute path of import path %q: %w", importPath, err) - } - // ensure that an absolute GOPATH is used. absGoPath, err := filepath.Abs(interp.context.GOPATH) if err != nil { @@ -183,9 +177,9 @@ func (interp *Interpreter) getPackageDir(importPath string) (string, error) { }, } - pkgs, err := packages.Load(&config, absImportPath) + pkgs, err := packages.Load(&config, importPath) if err != nil { - return "", fmt.Errorf("an error occurred retrieving a package: %v\n%v\nIf Access is denied, run in administrator", absImportPath, err) + return "", fmt.Errorf("an error occurred retrieving a package: %v\n%v\nIf Access is denied, run in administrator", importPath, err) } // confirm the import path is found. @@ -197,7 +191,7 @@ func (interp *Interpreter) getPackageDir(importPath string) (string, error) { } } - return "", fmt.Errorf("an import source could not be found: %q. Did you set the required environment variable in the Interpreter.Options?", absImportPath) + return "", fmt.Errorf("an import source could not be found: %q. Did you set the required environment variable in the Interpreter.Options?", importPath) } func effectivePkg(root, path string) string { From 657705dd493f834f10405a863d6e9b0827c370d2 Mon Sep 17 00:00:00 2001 From: switchupcb Date: Tue, 9 Aug 2022 22:03:17 -0500 Subject: [PATCH 4/5] add GOCACHE to go env --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8196bb805..829426d8f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -103,6 +103,9 @@ jobs: - name: Setup GOPATH run: go env -w GOPATH=${{ github.workspace }}/go + - name: Setup GOCACHE + run: go env -w GOCACHE=${{ github.workspace }}/.cache/go-build + - name: Build run: go build -v ./... From 715142863099d4d5be9231553302ed08515a441a Mon Sep 17 00:00:00 2001 From: switchupcb Date: Tue, 9 Aug 2022 22:37:23 -0500 Subject: [PATCH 5/5] update error message for failed import --- interp/src.go | 2 +- interp/src_test.go | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/interp/src.go b/interp/src.go index f3652cc44..f33887731 100644 --- a/interp/src.go +++ b/interp/src.go @@ -191,7 +191,7 @@ func (interp *Interpreter) getPackageDir(importPath string) (string, error) { } } - return "", fmt.Errorf("an import source could not be found: %q. Did you set the required environment variable in the Interpreter.Options?", importPath) + return "", fmt.Errorf("an import source could not be found for %q", importPath) } func effectivePkg(root, path string) string { diff --git a/interp/src_test.go b/interp/src_test.go index b4848b34f..7e2770e8d 100644 --- a/interp/src_test.go +++ b/interp/src_test.go @@ -160,11 +160,9 @@ func Test_pkgDir(t *testing.T) { }, } - interp := &Interpreter{ - opt: opt{ - filesystem: &realFS{}, - }, - } + interp := New(Options{ + Env: []string{"GOCACHE=" + os.Getenv("GOCACHE")}, + }) for _, test := range testCases { test := test