Compiler tests diff runner

.github/workflows/ci.yml

@@ -40,13 +40,19 @@ jobs:
           - os: windows-latest
           - os: macos-latest
           - os: ubuntu-latest
+            name: 'no submodules'
             no-submodules: true
-          - os: ubuntu-latest
+          - os: ['self-hosted', '1ES.Pool=TypeScript-1ES-GitHub-XL', '1ES.ImageOverride=mariner-2.0']
+            name: 'race mode'
             race: true
           - os: ubuntu-latest
+            name: 'noembed'
             noembed: true
+          - os: ubuntu-latest
+            name: 'concurrent test programs'
+            concurrent-test-program: true
 
-    name: test (${{ matrix.os }}${{ (matrix.no-submodules && ' without submodules') || '' }}${{ (matrix.race && ' in race mode') || '' }}${{ (matrix.noembed && ' with noembed') || '' }})
+    name: test (${{ matrix.name || matrix.os }})
 
     runs-on: ${{ matrix.os }}
 
@@ -58,6 +64,8 @@ jobs:
         with:
           submodules: ${{ !matrix.no-submodules }}
       - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
+        with:
+          node-version: 'lts/*'
       - uses: ./.github/actions/setup-go
         with:
           cache-name: test
@@ -71,10 +79,11 @@ jobs:
 
       - name: Tests
         id: test
-        run: npx hereby test:all ${RACE_FLAG:+"$RACE_FLAG"} ${NOEMBED_FLAG:+"$NOEMBED_FLAG"}
+        run: npx hereby test:all ${RACE_FLAG:+"$RACE_FLAG"} ${NOEMBED_FLAG:+"$NOEMBED_FLAG"} ${CONCURRENTTESTPROGRAM_FLAG:+"$CONCURRENTTESTPROGRAM_FLAG"}
         env:
           RACE_FLAG: ${{ (matrix.race && '--race') || '' }}
           NOEMBED_FLAG: ${{ (matrix.noembed && '--noembed') || '' }}
+          CONCURRENTTESTPROGRAM_FLAG: ${{ (matrix.concurrent-test-program && '--concurrentTestPrograms') || '' }}
 
       - run: git add .
       - run: git diff --staged --exit-code --stat
@@ -85,6 +94,11 @@ jobs:
           npx hereby baseline-accept
           git add testdata/baselines/reference
           git diff --staged --exit-code
+      - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        if: ${{ failure() && steps.test.conclusion == 'failure' }}
+        with:
+          name: ${{ matrix.os }}-${{ (matrix.race && 'race') || 'norace' }}-new-baselines-artifact
+          path: testdata/baselines/local
 
   lint:
     runs-on: ubuntu-latest

Herebyfile.mjs

@@ -28,12 +28,14 @@ const { values: options } = parseArgs({
         fix: { type: "boolean" },
         noembed: { type: "boolean" },
         debug: { type: "boolean" },
+        concurrentTestPrograms: { type: "boolean" },
     },
     strict: false,
     allowPositionals: true,
     allowNegative: true,
     noembed: false,
     debug: false,
+    concurrentTestPrograms: false,
 });
 
 const defaultGoBuildTags = [
@@ -203,6 +205,12 @@ const goTestFlags = [
     ...goBuildTags(),
 ];
 
+const goTestEnv = {
+    ...(options.concurrentTestPrograms ? { TS_TEST_PROGRAM_SINGLE_THREADED: "false" } : {}),
+};
+
+const $test = $({ env: goTestEnv });
+
 const gotestsum = memoize(() => {
     const args = isInstalled("gotestsum") ? ["gotestsum", "--format-hide-empty-pkg", "--"] : ["go", "test"];
     return args.concat(goTestFlags);
@@ -213,7 +221,7 @@ const goTest = memoize(() => {
 });
 
 async function runTests() {
-    await $`${gotestsum()} ./...`;
+    await $test`${gotestsum()} ./... ${isCI ? ["--timeout=45m"] : []}`;
 }
 
 export const test = task({
@@ -223,7 +231,7 @@ export const test = task({
 
 async function runTestBenchmarks() {
     // Run the benchmarks once to ensure they compile and run without errors.
-    await $`${goTest()} -run=- -bench=. -benchtime=1x ./...`;
+    await $test`${goTest()} -run=- -bench=. -benchtime=1x ./...`;
 }
 
 export const testBenchmarks = task({
@@ -232,7 +240,7 @@ export const testBenchmarks = task({
 });
 
 async function runTestTools() {
-    await $({ cwd: path.join(__dirname, "_tools") })`${gotestsum()} ./...`;
+    await $test({ cwd: path.join(__dirname, "_tools") })`${gotestsum()} ./...`;
 }
 
 export const testTools = task({

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/dlclark/regexp2 v1.11.4
 	github.com/go-json-experiment/json v0.0.0-20241127185351-9802db03f36a
 	github.com/google/go-cmp v0.6.0
+	github.com/pkg/diff v0.0.0-20241224192749-4e6772a4315c
 	golang.org/x/sys v0.27.0
 	golang.org/x/tools v0.27.0
 	gotest.tools/v3 v3.5.1

go.sum

@@ -4,6 +4,8 @@ github.com/go-json-experiment/json v0.0.0-20241127185351-9802db03f36a h1:W/o3DbE
 github.com/go-json-experiment/json v0.0.0-20241127185351-9802db03f36a/go.mod h1:BWmvoE1Xia34f3l/ibJweyhrT+aROb/FQ6d+37F0e2s=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/pkg/diff v0.0.0-20241224192749-4e6772a4315c h1:8TRxBMS/YsupXoOiGKHr9ZOXo+5DezGWPgBAhBHEHto=
+github.com/pkg/diff v0.0.0-20241224192749-4e6772a4315c/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
 golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=

internal/compiler/emitHost.go

@@ -51,6 +51,6 @@ func (host *emitHost) WriteFile(fileName string, text string, writeByteOrderMark
 }
 
 func (host *emitHost) GetEmitResolver(file *ast.SourceFile, skipDiagnostics bool) checker.EmitResolver {
-	checker := host.program.getTypeCheckerForFile(file)
+	checker := host.program.GetTypeCheckerForFile(file)
 	return checker.GetEmitResolver(file, skipDiagnostics)
 }

internal/compiler/program.go

@@ -190,7 +190,7 @@ func (p *Program) BindSourceFiles() {
 
 func (p *Program) CheckSourceFiles() {
 	p.createCheckers()
-	wg := core.NewWorkGroup(false)
+	wg := core.NewWorkGroup(p.programOptions.SingleThreaded)
 	for index, checker := range p.checkers {
 		wg.Queue(func() {
 			for i := index; i < len(p.files); i += len(p.checkers) {
@@ -227,7 +227,7 @@ func (p *Program) GetTypeChecker() *checker.Checker {
 // method returns the checker that was tasked with checking the file. Note that it isn't possible to mix
 // types obtained from different checkers, so only non-type data (such as diagnostics or string
 // representations of types) should be obtained from checkers returned by this method.
-func (p *Program) getTypeCheckerForFile(file *ast.SourceFile) *checker.Checker {
+func (p *Program) GetTypeCheckerForFile(file *ast.SourceFile) *checker.Checker {
 	p.createCheckers()
 	return p.checkersByFile[file]
 }
@@ -299,7 +299,7 @@ func (p *Program) getBindDiagnosticsForFile(sourceFile *ast.SourceFile) []*ast.D
 }
 
 func (p *Program) getSemanticDiagnosticsForFile(sourceFile *ast.SourceFile) []*ast.Diagnostic {
-	diags := core.Concatenate(sourceFile.BindDiagnostics(), p.getTypeCheckerForFile(sourceFile).GetDiagnostics(sourceFile))
+	diags := core.Concatenate(sourceFile.BindDiagnostics(), p.GetTypeCheckerForFile(sourceFile).GetDiagnostics(sourceFile))
 	if len(sourceFile.CommentDirectives) == 0 {
 		return diags
 	}

internal/testutil/baseline/baseline.go

@@ -4,21 +4,90 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strings"
 	"testing"
 
 	"github.com/microsoft/typescript-go/internal/repo"
+	"github.com/pkg/diff"
 )
 
 type Options struct {
-	Subfolder string
+	Subfolder   string
+	IsSubmodule bool
 }
 
-const NoContent = "<no content>"
+const (
+	NoContent       = "<no content>"
+	submoduleFolder = "submodule"
+)
 
 func Run(t *testing.T, fileName string, actual string, opts Options) {
-	writeComparison(t, actual, fileName, false /*useSubmodule*/, opts)
+	if opts.IsSubmodule {
+		diff := getBaselineDiff(t, actual, fileName)
+		diffFileName := fileName + ".diff"
+		opts.Subfolder = filepath.Join(submoduleFolder, opts.Subfolder)
+		referenceFileName := referencePath(fileName, opts.Subfolder)
+		expected := NoContent
+		if content, err := os.ReadFile(referenceFileName); err == nil {
+			expected = string(content)
+		}
+		// Write original baseline in addition to diff baseline, if needed, but don't fail if they don't match
+		if actual != expected {
+			localFileName := localPath(fileName, opts.Subfolder)
+			if err := os.MkdirAll(filepath.Dir(localFileName), 0o755); err != nil {
+				t.Fatal(fmt.Errorf("failed to create directories for the local baseline file %s: %w", localFileName, err))
+			}
+			if err := os.WriteFile(localFileName, []byte(actual), 0o644); err != nil {
+				t.Fatal(fmt.Errorf("failed to write the local baseline file %s: %w", localFileName, err))
+			}
+		}
+		writeComparison(t, diff, diffFileName, false /*useSubmodule*/, opts)
+	} else {
+		writeComparison(t, actual, fileName, false /*useSubmodule*/, opts)
+	}
+}
+
+func getBaselineDiff(t *testing.T, actual string, fileName string) string {
+	expected := NoContent
+	refFileName := submoduleReferencePath(fileName, "" /*subfolder*/)
+	if content, err := os.ReadFile(refFileName); err == nil {
+		expected = string(content)
+	}
+	if actual == expected {
+		return NoContent
+	}
+	var b strings.Builder
+	if err := diff.Text("old."+fileName, "new."+fileName, expected, actual, &b); err != nil {
+		t.Fatalf("failed to diff the actual and expected content: %v", err)
+	}
+
+	// Remove line numbers from unified diff headers; this avoids adding/deleting
+	// lines in our baselines from causing knock-on header changes later in the diff.
+	s := b.String()
+
+	aCurLine := 1
+	bCurLine := 1
+	s = fixUnifiedDiff.ReplaceAllStringFunc(s, func(match string) string {
+		var aLine, aLineCount, bLine, bLineCount int
+		if _, err := fmt.Sscanf(match, "@@ -%d,%d +%d,%d @@", &aLine, &aLineCount, &bLine, &bLineCount); err != nil {
+			panic(fmt.Sprintf("failed to parse unified diff header: %v", err))
+		}
+		aDiff := aLine - aCurLine
+		bDiff := bLine - bCurLine
+		aCurLine = aLine
+		bCurLine = bLine
+
+		// Keep surrounded by @@, to make GitHub's grammar happy.
+		// https://github.com/textmate/diff.tmbundle/blob/0593bb775eab1824af97ef2172fd38822abd97d7/Syntaxes/Diff.plist#L68
+		return fmt.Sprintf("@@= skipped -%d, +%d lines =@@", aDiff, bDiff)
+	})
+
+	return s
 }
 
+var fixUnifiedDiff = regexp.MustCompile(`@@ -\d+,\d+ \+\d+,\d+ @@`)
+
 func RunAgainstSubmodule(t *testing.T, fileName string, actual string, opts Options) {
 	writeComparison(t, actual, fileName, true /*useSubmodule*/, opts)
 }