Add "accepted diffs" to test runner, accept diffs for Programs with unsupported extensions (#655)

Author: jakebailey

Diff for: .gitignore

@@ -192,3 +192,5 @@ custom-gcl.hash
 .DS_Store
 
 .idea
+
+!testdata/submoduleAccepted.txt

Diff for: cmd/tsgo/main.go

@@ -237,15 +237,8 @@ func main() {
 		printDiagnostics(ts.SortAndDeduplicateDiagnostics(diagnostics), host, compilerOptions)
 	}
 
-	var unsupportedExtensions []string
-	for _, file := range program.SourceFiles() {
-		extension := tspath.TryGetExtensionFromPath(file.FileName())
-		if extension == tspath.ExtensionTsx || slices.Contains(tspath.SupportedJSExtensionsFlat, extension) {
-			unsupportedExtensions = core.AppendIfUnique(unsupportedExtensions, extension)
-		}
-	}
-	if len(unsupportedExtensions) != 0 {
-		fmt.Fprintf(os.Stderr, "Warning: The project contains unsupported file types (%s), which are currently not fully type-checked.\n", strings.Join(unsupportedExtensions, ", "))
+	if exts := program.UnsupportedExtensions(); len(exts) != 0 {
+		fmt.Fprintf(os.Stderr, "Warning: The project contains unsupported file types (%s), which are currently not fully type-checked.\n", strings.Join(exts, ", "))
 	}
 
 	if compilerOptions.ListFiles.IsTrue() {

Diff for: internal/compiler/program.go

@@ -63,6 +63,9 @@ type Program struct {
 
 	commonSourceDirectory     string
 	commonSourceDirectoryOnce sync.Once
+
+	// List of present unsupported extensions
+	unsupportedExtensions []string
 }
 
 var extensions = []string{".ts", ".tsx"}
@@ -155,6 +158,13 @@ func NewProgram(options ProgramOptions) *Program {
 		p.filesByPath[file.Path()] = file
 	}
 
+	for _, file := range p.files {
+		extension := tspath.TryGetExtensionFromPath(file.FileName())
+		if extension == tspath.ExtensionTsx || slices.Contains(tspath.SupportedJSExtensionsFlat, extension) {
+			p.unsupportedExtensions = append(p.unsupportedExtensions, extension)
+		}
+	}
+
 	return p
 }
 
@@ -609,3 +619,9 @@ type FileIncludeReason struct {
 	Kind  FileIncludeKind
 	Index int
 }
+
+// UnsupportedExtensions returns a list of all present "unsupported" extensions,
+// e.g. extensions that are not yet supported by the port.
+func (p *Program) UnsupportedExtensions() []string {
+	return p.unsupportedExtensions
+}

Diff for: internal/testrunner/compiler_runner.go

@@ -337,7 +337,11 @@ func (c *compilerTest) verifyDiagnostics(t *testing.T, suiteName string, isSubmo
 
 		defer testutil.RecoverAndFail(t, "Panic on creating error baseline for test "+c.filename)
 		files := core.Concatenate(c.tsConfigFiles, core.Concatenate(c.toBeCompiled, c.otherFiles))
-		tsbaseline.DoErrorBaseline(t, c.configuredName, files, c.result.Diagnostics, c.result.Options.Pretty.IsTrue(), baseline.Options{Subfolder: suiteName, IsSubmodule: isSubmodule})
+		tsbaseline.DoErrorBaseline(t, c.configuredName, files, c.result.Diagnostics, c.result.Options.Pretty.IsTrue(), baseline.Options{
+			Subfolder:           suiteName,
+			IsSubmodule:         isSubmodule,
+			IsSubmoduleAccepted: len(c.result.Program.UnsupportedExtensions()) != 0, // TODO(jakebailey): read submoduleAccepted.txt
+		})
 	})
 }
 

Diff for: internal/testutil/baseline/baseline.go

@@ -6,39 +6,103 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
 
+	"github.com/microsoft/typescript-go/internal/core"
 	"github.com/microsoft/typescript-go/internal/repo"
+	"github.com/microsoft/typescript-go/internal/tspath"
 	"github.com/pkg/diff"
+	"gotest.tools/v3/assert"
 )
 
 type Options struct {
-	Subfolder    string
-	IsSubmodule  bool
-	DiffFixupOld func(string) string
+	Subfolder           string
+	IsSubmodule         bool
+	IsSubmoduleAccepted bool
+	DiffFixupOld        func(string) string
 }
 
-const (
-	NoContent       = "<no content>"
-	submoduleFolder = "submodule"
-)
+const NoContent = "<no content>"
 
 func Run(t *testing.T, fileName string, actual string, opts Options) {
-	if opts.IsSubmodule {
-		opts.Subfolder = filepath.Join(submoduleFolder, opts.Subfolder)
-		diff := getBaselineDiff(t, actual, fileName, opts.DiffFixupOld)
-		diffFileName := fileName + ".diff"
-		writeComparison(t, diff, diffFileName, false, opts)
+	origSubfolder := opts.Subfolder
+
+	{
+		subfolder := opts.Subfolder
+		if opts.IsSubmodule {
+			subfolder = filepath.Join("submodule", subfolder)
+		}
+
+		localPath := filepath.Join(localRoot, subfolder, fileName)
+		referencePath := filepath.Join(referenceRoot, subfolder, fileName)
+
+		writeComparison(t, actual, localPath, referencePath, false)
+	}
+
+	if !opts.IsSubmodule {
+		// Not a submodule, no diffs.
+		return
+	}
+
+	submoduleReference := filepath.Join(submoduleReferenceRoot, fileName)
+	submoduleExpected := readFileOrNoContent(submoduleReference)
+
+	const (
+		submoduleFolder         = "submodule"
+		submoduleAcceptedFolder = "submoduleAccepted"
+	)
+
+	diffFileName := fileName + ".diff"
+	isSubmoduleAccepted := opts.IsSubmoduleAccepted || submoduleAcceptedFileNames().Has(origSubfolder+"/"+diffFileName)
+
+	outRoot := core.IfElse(isSubmoduleAccepted, submoduleAcceptedFolder, submoduleFolder)
+	unusedOutRoot := core.IfElse(isSubmoduleAccepted, submoduleFolder, submoduleAcceptedFolder)
+
+	{
+		localPath := filepath.Join(localRoot, outRoot, origSubfolder, diffFileName)
+		referencePath := filepath.Join(referenceRoot, outRoot, origSubfolder, diffFileName)
+
+		diff := getBaselineDiff(t, actual, submoduleExpected, fileName, opts.DiffFixupOld)
+		writeComparison(t, diff, localPath, referencePath, false)
+	}
+
+	// Delete the other diff file if it exists
+	{
+		localPath := filepath.Join(localRoot, unusedOutRoot, origSubfolder, diffFileName)
+		referencePath := filepath.Join(referenceRoot, unusedOutRoot, origSubfolder, diffFileName)
+		writeComparison(t, NoContent, localPath, referencePath, false)
 	}
-	writeComparison(t, actual, fileName, false, opts)
 }
 
-func getBaselineDiff(t *testing.T, actual string, fileName string, fixupOld func(string) string) string {
-	expected := NoContent
-	refFileName := submoduleReferencePath(fileName, "" /*subfolder*/)
-	if content, err := os.ReadFile(refFileName); err == nil {
-		expected = string(content)
+var submoduleAcceptedFileNames = sync.OnceValue(func() *core.Set[string] {
+	var set core.Set[string]
+
+	submoduleAccepted := filepath.Join(repo.TestDataPath, "submoduleAccepted.txt")
+	if content, err := os.ReadFile(submoduleAccepted); err == nil {
+		for line := range strings.SplitSeq(string(content), "\n") {
+			line = strings.TrimSpace(line)
+			if line == "" || line[0] == '#' {
+				continue
+			}
+			set.Add(line)
+		}
+	} else {
+		panic(fmt.Sprintf("failed to read submodule accepted file: %v", err))
+	}
+
+	return &set
+})
+
+func readFileOrNoContent(fileName string) string {
+	content, err := os.ReadFile(fileName)
+	if err != nil {
+		return NoContent
 	}
+	return string(content)
+}
+
+func getBaselineDiff(t *testing.T, actual string, expected string, fileName string, fixupOld func(string) string) string {
 	if fixupOld != nil {
 		expected = fixupOld(expected)
 	}
@@ -77,84 +141,72 @@ func getBaselineDiff(t *testing.T, actual string, fileName string, fixupOld func
 var fixUnifiedDiff = regexp.MustCompile(`@@ -\d+,\d+ \+\d+,\d+ @@`)
 
 func RunAgainstSubmodule(t *testing.T, fileName string, actual string, opts Options) {
-	writeComparison(t, actual, fileName, true /*useSubmodule*/, opts)
+	local := filepath.Join(localRoot, opts.Subfolder, fileName)
+	reference := filepath.Join(submoduleReferenceRoot, opts.Subfolder, fileName)
+	writeComparison(t, actual, local, reference, true)
 }
 
-func writeComparison(t *testing.T, actual string, relativeFileName string, useSubmodule bool, opts Options) {
-	if actual == "" {
+func writeComparison(t *testing.T, actualContent string, local, reference string, comparingAgainstSubmodule bool) {
+	if actualContent == "" {
 		panic("the generated content was \"\". Return 'baseline.NoContent' if no baselining is required.")
 	}
-	var (
-		localFileName     string
-		referenceFileName string
-	)
-
-	if useSubmodule {
-		localFileName = submoduleLocalPath(relativeFileName, opts.Subfolder)
-		referenceFileName = submoduleReferencePath(relativeFileName, opts.Subfolder)
-	} else {
-		localFileName = localPath(relativeFileName, opts.Subfolder)
-		referenceFileName = referencePath(relativeFileName, opts.Subfolder)
-	}
 
-	if err := os.MkdirAll(filepath.Dir(localFileName), 0o755); err != nil {
-		t.Error(fmt.Errorf("failed to create directories for the local baseline file %s: %w", localFileName, err))
+	if err := os.MkdirAll(filepath.Dir(local), 0o755); err != nil {
+		t.Error(fmt.Errorf("failed to create directories for the local baseline file %s: %w", local, err))
 		return
 	}
 
-	if _, err := os.Stat(localFileName); err == nil {
-		if err := os.Remove(localFileName); err != nil {
-			t.Error(fmt.Errorf("failed to remove the local baseline file %s: %w", localFileName, err))
+	if _, err := os.Stat(local); err == nil {
+		if err := os.Remove(local); err != nil {
+			t.Error(fmt.Errorf("failed to remove the local baseline file %s: %w", local, err))
 			return
 		}
 	}
 
 	expected := NoContent
 	foundExpected := false
-	if content, err := os.ReadFile(referenceFileName); err == nil {
+	if content, err := os.ReadFile(reference); err == nil {
 		expected = string(content)
 		foundExpected = true
 	}
 
-	if expected != actual || actual == NoContent && foundExpected {
-		if actual == NoContent {
-			if err := os.WriteFile(localFileName+".delete", []byte{}, 0o644); err != nil {
-				t.Error(fmt.Errorf("failed to write the local baseline file %s: %w", localFileName+".delete", err))
+	if expected != actualContent || actualContent == NoContent && foundExpected {
+		if actualContent == NoContent {
+			if err := os.WriteFile(local+".delete", []byte{}, 0o644); err != nil {
+				t.Error(fmt.Errorf("failed to write the local baseline file %s: %w", local+".delete", err))
 				return
 			}
 		} else {
-			if err := os.WriteFile(localFileName, []byte(actual), 0o644); err != nil {
-				t.Error(fmt.Errorf("failed to write the local baseline file %s: %w", localFileName, err))
+			if err := os.WriteFile(local, []byte(actualContent), 0o644); err != nil {
+				t.Error(fmt.Errorf("failed to write the local baseline file %s: %w", local, err))
 				return
 			}
 		}
 
-		if _, err := os.Stat(referenceFileName); err != nil {
-			if useSubmodule {
-				t.Errorf("the baseline file %s does not exist in the TypeScript submodule", referenceFileName)
+		relReference, err := filepath.Rel(repo.RootPath, reference)
+		assert.NilError(t, err)
+		relReference = tspath.NormalizeSlashes(relReference)
+
+		relLocal, err := filepath.Rel(repo.RootPath, local)
+		assert.NilError(t, err)
+		relLocal = tspath.NormalizeSlashes(relLocal)
+
+		if _, err := os.Stat(reference); err != nil {
+			if comparingAgainstSubmodule {
+				t.Errorf("the baseline file %s does not exist in the TypeScript submodule", relReference)
 			} else {
-				t.Errorf("new baseline created at %s.", localFileName)
+				t.Errorf("new baseline created at %s.", relLocal)
 			}
-		} else if useSubmodule {
-			t.Errorf("the baseline file %s does not match the reference in the TypeScript submodule", relativeFileName)
+		} else if comparingAgainstSubmodule {
+			t.Errorf("the baseline file %s does not match the reference in the TypeScript submodule", relReference)
 		} else {
-			t.Errorf("the baseline file %s has changed. (Run `hereby baseline-accept` if the new baseline is correct.)", relativeFileName)
+			t.Errorf("the baseline file %s has changed. (Run `hereby baseline-accept` if the new baseline is correct.)", relReference)
 		}
 	}
 }
 
-func localPath(fileName string, subfolder string) string {
-	return filepath.Join(repo.TestDataPath, "baselines", "local", subfolder, fileName)
-}
-
-func submoduleLocalPath(fileName string, subfolder string) string {
-	return filepath.Join(repo.TestDataPath, "baselines", "tmp", subfolder, fileName)
-}
-
-func referencePath(fileName string, subfolder string) string {
-	return filepath.Join(repo.TestDataPath, "baselines", "reference", subfolder, fileName)
-}
-
-func submoduleReferencePath(fileName string, subfolder string) string {
-	return filepath.Join(repo.TypeScriptSubmodulePath, "tests", "baselines", "reference", subfolder, fileName)
-}
+var (
+	localRoot              = filepath.Join(repo.TestDataPath, "baselines", "local")
+	referenceRoot          = filepath.Join(repo.TestDataPath, "baselines", "reference")
+	submoduleReferenceRoot = filepath.Join(repo.TypeScriptSubmodulePath, "tests", "baselines", "reference")
+)

Diff for: internal/testutil/tsbaseline/type_symbol_baseline.go

@@ -71,6 +71,7 @@ func DoTypeAndSymbolBaseline(
 
 			return sb.String()[:sb.Len()-1]
 		}
+		typesOpts.IsSubmoduleAccepted = len(program.UnsupportedExtensions()) != 0 // TODO(jakebailey): read submoduleAccepted.txt
 
 		checkBaselines(t, baselinePath, allFiles, fullWalker, header, typesOpts, false /*isSymbolBaseline*/)
 	})