feat(gnovm/pkg/packages): categorize imports (#3323)

This makes the imports utils split imports by file kinds, allowing to
make explicit decisions about what imports to use at the various
callsites

- Create `FileKind` enum to categorize gno files, with variants
`PackageSource`, `Test`, `XTest` and `Filetest`
- Create `GetFileKind` util to derive the `FileKind` from a file name
and body
- Create `ImportsMap` type that maps `FileKind`s to lists of imports. It
has a single method `Merge` to select and merge various imports from
multiple `FileKind`s
- Modify the`packages.Imports` helper to return an `ImportsMap` instead
of a `[]string` and adapt callsites by using`ImportMap.Merge` to
preserve existing behavior

This is something I need for #3304 and #2932 but to help reviews I made
an atomic PR here instead

---------

Signed-off-by: Norman Meier <[email protected]>
Co-authored-by: Morgan <[email protected]>

Author: 2 people

gno.land/pkg/integration/pkgloader.go

@@ -131,10 +131,11 @@ func (pl *PkgsLoader) LoadPackage(modroot string, path, name string) error {
 			if err != nil {
 				return fmt.Errorf("unable to read package at %q: %w", currentPkg.Dir, err)
 			}
-			imports, err := packages.Imports(pkg, nil)
+			importsMap, err := packages.Imports(pkg, nil)
 			if err != nil {
 				return fmt.Errorf("unable to load package imports in %q: %w", currentPkg.Dir, err)
 			}
+			imports := importsMap.Merge(packages.FileKindPackageSource, packages.FileKindTest, packages.FileKindFiletest)
 			for _, imp := range imports {
 				if imp.PkgPath == currentPkg.Name || gnolang.IsStdlib(imp.PkgPath) {
 					continue

gnovm/cmd/gno/download_deps.go

@@ -25,10 +25,11 @@ func downloadDeps(io commands.IO, pkgDir string, gnoMod *gnomod.File, fetcher pk
 	if err != nil {
 		return fmt.Errorf("read package at %q: %w", pkgDir, err)
 	}
-	imports, err := packages.Imports(pkg, nil)
+	importsMap, err := packages.Imports(pkg, nil)
 	if err != nil {
 		return fmt.Errorf("read imports at %q: %w", pkgDir, err)
 	}
+	imports := importsMap.Merge(packages.FileKindPackageSource, packages.FileKindTest, packages.FileKindXTest)
 
 	for _, pkgPath := range imports {
 		resolved := gnoMod.Resolve(module.Version{Path: pkgPath.PkgPath})

gnovm/pkg/doc/dirs.go

@@ -105,11 +105,12 @@ func packageImportsRecursive(root string, pkgPath string) []string {
 		pkg = &gnovm.MemPackage{}
 	}
 
-	resRaw, err := packages.Imports(pkg, nil)
+	importsMap, err := packages.Imports(pkg, nil)
 	if err != nil {
 		// ignore packages with invalid imports
-		resRaw = nil
+		importsMap = nil
 	}
+	resRaw := importsMap.Merge(packages.FileKindPackageSource, packages.FileKindTest, packages.FileKindXTest)
 	res := make([]string, len(resRaw))
 	for idx, imp := range resRaw {
 		res[idx] = imp.PkgPath

gnovm/pkg/gnomod/pkg.go

@@ -121,11 +121,13 @@ func ListPkgs(root string) (PkgList, error) {
 			pkg = &gnovm.MemPackage{}
 		}
 
-		importsRaw, err := packages.Imports(pkg, nil)
+		importsMap, err := packages.Imports(pkg, nil)
 		if err != nil {
 			// ignore imports on error
-			importsRaw = nil
+			importsMap = nil
 		}
+		importsRaw := importsMap.Merge(packages.FileKindPackageSource, packages.FileKindTest, packages.FileKindXTest)
+
 		imports := make([]string, 0, len(importsRaw))
 		for _, imp := range importsRaw {
 			// remove self and standard libraries from imports

gnovm/pkg/packages/filekind.go

@@ -0,0 +1,56 @@
+package packages
+
+import (
+	"fmt"
+	"go/parser"
+	"go/token"
+	"strings"
+)
+
+// FileKind represent the category a gno source file falls in, can be one of:
+//
+// - [FileKindPackageSource] -> A *.gno file that will be included in the gnovm package
+//
+// - [FileKindTest] -> A *_test.gno file that will be used for testing
+//
+// - [FileKindXTest] -> A *_test.gno file with a package name ending in _test that will be used for blackbox testing
+//
+// - [FileKindFiletest] -> A *_filetest.gno file that will be used for snapshot testing
+type FileKind uint
+
+const (
+	FileKindUnknown FileKind = iota
+	FileKindPackageSource
+	FileKindTest
+	FileKindXTest
+	FileKindFiletest
+)
+
+// GetFileKind analyzes a file's name and body to get it's [FileKind], fset is optional
+func GetFileKind(filename string, body string, fset *token.FileSet) (FileKind, error) {
+	if !strings.HasSuffix(filename, ".gno") {
+		return FileKindUnknown, fmt.Errorf("%s:1:1: not a gno file", filename)
+	}
+
+	if strings.HasSuffix(filename, "_filetest.gno") {
+		return FileKindFiletest, nil
+	}
+
+	if !strings.HasSuffix(filename, "_test.gno") {
+		return FileKindPackageSource, nil
+	}
+
+	if fset == nil {
+		fset = token.NewFileSet()
+	}
+	ast, err := parser.ParseFile(fset, filename, body, parser.PackageClauseOnly)
+	if err != nil {
+		return FileKindUnknown, err
+	}
+	packageName := ast.Name.Name
+
+	if strings.HasSuffix(packageName, "_test") {
+		return FileKindXTest, nil
+	}
+	return FileKindTest, nil
+}

gnovm/pkg/packages/filekind_test.go

@@ -0,0 +1,63 @@
+package packages
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetFileKind(t *testing.T) {
+	tcs := []struct {
+		name          string
+		filename      string
+		body          string
+		fileKind      FileKind
+		errorContains string
+	}{
+		{
+			name:     "compiled",
+			filename: "foo.gno",
+			fileKind: FileKindPackageSource,
+		},
+		{
+			name:     "test",
+			filename: "foo_test.gno",
+			body:     "package foo",
+			fileKind: FileKindTest,
+		},
+		{
+			name:     "xtest",
+			filename: "foo_test.gno",
+			body:     "package foo_test",
+			fileKind: FileKindXTest,
+		},
+		{
+			name:     "filetest",
+			filename: "foo_filetest.gno",
+			fileKind: FileKindFiletest,
+		},
+		{
+			name:          "err_badpkgclause",
+			filename:      "foo_test.gno",
+			body:          "pakage foo",
+			errorContains: "foo_test.gno:1:1: expected 'package', found pakage",
+		},
+		{
+			name:          "err_notgnofile",
+			filename:      "foo.gno.bck",
+			errorContains: `foo.gno.bck:1:1: not a gno file`,
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			out, err := GetFileKind(tc.filename, tc.body, nil)
+			if len(tc.errorContains) != 0 {
+				require.ErrorContains(t, err, tc.errorContains)
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, tc.fileKind, out)
+		})
+	}
+}

gnovm/pkg/packages/imports.go

@@ -14,33 +14,40 @@ import (
 
 // Imports returns the list of gno imports from a [gnovm.MemPackage].
 // fset is optional.
-func Imports(pkg *gnovm.MemPackage, fset *token.FileSet) ([]FileImport, error) {
-	allImports := make([]FileImport, 0, 16)
-	seen := make(map[string]struct{}, 16)
+func Imports(pkg *gnovm.MemPackage, fset *token.FileSet) (ImportsMap, error) {
+	res := make(ImportsMap, 16)
+	seen := make(map[FileKind]map[string]struct{}, 16)
+
 	for _, file := range pkg.Files {
 		if !strings.HasSuffix(file.Name, ".gno") {
 			continue
 		}
-		if strings.HasSuffix(file.Name, "_filetest.gno") {
-			continue
+
+		fileKind, err := GetFileKind(file.Name, file.Body, fset)
+		if err != nil {
+			return nil, err
 		}
 		imports, err := FileImports(file.Name, file.Body, fset)
 		if err != nil {
 			return nil, err
 		}
 		for _, im := range imports {
-			if _, ok := seen[im.PkgPath]; ok {
+			if _, ok := seen[fileKind][im.PkgPath]; ok {
 				continue
 			}
-			allImports = append(allImports, im)
-			seen[im.PkgPath] = struct{}{}
+			res[fileKind] = append(res[fileKind], im)
+			if _, ok := seen[fileKind]; !ok {
+				seen[fileKind] = make(map[string]struct{}, 16)
+			}
+			seen[fileKind][im.PkgPath] = struct{}{}
 		}
 	}
-	sort.Slice(allImports, func(i, j int) bool {
-		return allImports[i].PkgPath < allImports[j].PkgPath
-	})
 
-	return allImports, nil
+	for _, imports := range res {
+		sortImports(imports)
+	}
+
+	return res, nil
 }
 
 // FileImport represents an import
@@ -75,3 +82,31 @@ func FileImports(filename string, src string, fset *token.FileSet) ([]FileImport
 	}
 	return res, nil
 }
+
+type ImportsMap map[FileKind][]FileImport
+
+// Merge merges imports, it removes duplicates and sorts the result
+func (imap ImportsMap) Merge(kinds ...FileKind) []FileImport {
+	res := make([]FileImport, 0, 16)
+	seen := make(map[string]struct{}, 16)
+
+	for _, kind := range kinds {
+		for _, im := range imap[kind] {
+			if _, ok := seen[im.PkgPath]; ok {
+				continue
+			}
+			seen[im.PkgPath] = struct{}{}
+
+			res = append(res, im)
+		}
+	}
+
+	sortImports(res)
+	return res
+}
+
+func sortImports(imports []FileImport) {
+	sort.Slice(imports, func(i, j int) bool {
+		return imports[i].PkgPath < imports[j].PkgPath
+	})
+}

gnovm/pkg/packages/imports_test.go

@@ -58,13 +58,26 @@ func TestImports(t *testing.T) {
 			)
 			`,
 		},
+		{
+			name: "file2_test.gno",
+			data: `
+			package tmp_test
+
+			import (
+				"testing"
+
+				"gno.land/p/demo/testpkg"
+				"gno.land/p/demo/xtestdep"
+			)
+			`,
+		},
 		{
 			name: "z_0_filetest.gno",
 			data: `
 			package main
 
 			import (
-				"gno.land/p/demo/filetestpkg"
+				"gno.land/p/demo/filetestdep"
 			)
 			`,
 		},
@@ -95,17 +108,28 @@ func TestImports(t *testing.T) {
 		},
 	}
 
-	// Expected list of imports
+	// Expected lists of imports
 	// - ignore subdirs
 	// - ignore duplicate
-	// - ignore *_filetest.gno
 	// - should be sorted
-	expected := []string{
-		"gno.land/p/demo/pkg1",
-		"gno.land/p/demo/pkg2",
-		"gno.land/p/demo/testpkg",
-		"std",
-		"testing",
+	expected := map[FileKind][]string{
+		FileKindPackageSource: {
+			"gno.land/p/demo/pkg1",
+			"gno.land/p/demo/pkg2",
+			"std",
+		},
+		FileKindTest: {
+			"gno.land/p/demo/testpkg",
+			"testing",
+		},
+		FileKindXTest: {
+			"gno.land/p/demo/testpkg",
+			"gno.land/p/demo/xtestdep",
+			"testing",
+		},
+		FileKindFiletest: {
+			"gno.land/p/demo/filetestdep",
+		},
 	}
 
 	// Create subpkg dir
@@ -120,12 +144,19 @@ func TestImports(t *testing.T) {
 
 	pkg, err := gnolang.ReadMemPackage(tmpDir, "test")
 	require.NoError(t, err)
-	imports, err := Imports(pkg, nil)
+
+	importsMap, err := Imports(pkg, nil)
 	require.NoError(t, err)
-	importsStrings := make([]string, len(imports))
-	for idx, imp := range imports {
-		importsStrings[idx] = imp.PkgPath
+
+	// ignore specs
+	got := map[FileKind][]string{}
+	for key, vals := range importsMap {
+		stringVals := make([]string, len(vals))
+		for i, val := range vals {
+			stringVals[i] = val.PkgPath
+		}
+		got[key] = stringVals
 	}
 
-	require.Equal(t, expected, importsStrings)
+	require.Equal(t, expected, got)
 }

gnovm/pkg/test/imports.go

@@ -242,10 +242,11 @@ func LoadImports(store gno.Store, memPkg *gnovm.MemPackage) (err error) {
 	}()
 
 	fset := token.NewFileSet()
-	imports, err := packages.Imports(memPkg, fset)
+	importsMap, err := packages.Imports(memPkg, fset)
 	if err != nil {
 		return err
 	}
+	imports := importsMap.Merge(packages.FileKindPackageSource, packages.FileKindTest, packages.FileKindXTest)
 	for _, imp := range imports {
 		if gno.IsRealmPath(imp.PkgPath) {
 			// Don't eagerly load realms.