Add test infrastructure (#23)

No real tests yet, but 🔜 

- Upgrade to Node v24 (maybe in 2025 the world is ready for top-level
await?)
- Remove `ts-node` in favor of Node native type stripping + `tsc`
- Set up `assertScopes`

Author: jtbandes

.github/workflows/ci.yml

@@ -7,15 +7,16 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/[email protected]
       - uses: actions/[email protected]
         with:
-          node-version: 20.9
+          node-version: 24
       - run: corepack enable yarn
       - run: yarn install --immutable
 
+      - run: yarn typecheck
       - run: yarn build
 
       - name: Ensure generated files are up to date
@@ -28,3 +29,5 @@ jobs:
           else
             echo "Generated files are up to date!"
           fi
+
+      - run: yarn test:ci

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
-.yarn
+.yarn/*
+!.yarn/patches
 .pnp.*
 yarn-error.log

.yarn/patches/node-test-github-reporter.patch

@@ -0,0 +1,11 @@
+diff --git a/index.js b/index.js
+index 396c5c36b966bffddd28417e6d49ce4ae985f4d0..1a2c69693163c5572fb923fbde5b8f8377e5a32e 100644
+--- a/index.js
++++ b/index.js
+@@ -136,5 +136,5 @@ export function getSafePath(path) {
+ 
+ export function getRelativeFilePath(path) {
+   const filePath = getSafePath(path)
+-  return new URL(filePath).pathname.replace(workspacePrefixRegex, '')
++  return new URL(filePath).pathname.replace(workspacePrefixRegex, '').replace(/^\//, '')
+ }

package.json

@@ -7,21 +7,27 @@
     "url": "https://github.com/jtbandes"
   },
   "license": "MIT",
-  "packageManager": "[email protected]",
+  "packageManager": "[email protected]",
+  "type": "module",
   "scripts": {
     "build": "yarn build:json && yarn build:plist",
-    "build:json": "ts-node scripts/yaml-to-json.ts -i Swift.tmLanguage.yaml -o Swift.tmLanguage.json",
-    "build:plist": "ts-node scripts/json-to-plist.ts -i Swift.tmLanguage.json -o Syntaxes/Swift.tmLanguage"
+    "build:json": "yarn node scripts/yaml-to-json.ts -i Swift.tmLanguage.yaml -o Swift.tmLanguage.json",
+    "build:plist": "yarn node scripts/json-to-plist.ts -i Swift.tmLanguage.json -o Syntaxes/Swift.tmLanguage",
+    "test": "yarn node --test test/*.test.*",
+    "test:ci": "yarn node --test --test-reporter spec --test-reporter-destination stdout --test-reporter node-test-github-reporter --test-reporter-destination stdout test/*.test.*",
+    "typecheck": "tsc --noEmit"
   },
   "devDependencies": {
-    "@tsconfig/node20": "20.1.2",
-    "@types/node": "20.8.9",
+    "@shikijs/vscode-textmate": "10.0.2",
+    "@tsconfig/node24": "24.0.0",
+    "@types/node": "22.15.21",
     "@types/plist": "3.0.4",
     "commander": "11.1.0",
+    "node-test-github-reporter": "patch:node-test-github-reporter@npm%3A1.3.0#~/.yarn/patches/node-test-github-reporter.patch",
     "plist": "3.1.0",
-    "prettier": "3.0.3",
-    "ts-node": "10.9.1",
-    "typescript": "5.2.2",
+    "prettier": "3.5.3",
+    "shiki": "3.4.2",
+    "typescript": "5.8.3",
     "yaml": "2.3.3"
   }
 }

scripts/yaml-to-json.ts

@@ -25,7 +25,11 @@ async function main({ input, output }: { input: string; output: string }) {
           node.items.unshift(new Pair("comment", comment));
           node.commentBefore = undefined;
         }
-      } else if (node instanceof YAMLSeq && node.items.length > 0 && node.items[0] instanceof YAMLMap) {
+      } else if (
+        node instanceof YAMLSeq &&
+        node.items.length > 0 &&
+        node.items[0] instanceof YAMLMap
+      ) {
         const map = node.items[0];
         if (map.has("comment")) {
           console.warn("warning: dropping comment", comment);

test/assert-scopes.test.ts

@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { $, _, createAssertScopes } from "./assert-scopes.ts";
+
+const assertScopes = await createAssertScopes({
+  name: "Test",
+  scopeName: "source.test",
+  repository: {},
+  patterns: [
+    { match: "foo", name: "example.foo" },
+    { match: "bar", name: "example.bar" },
+  ],
+});
+
+test("assertions", () => {
+  assert.throws(
+    () => {
+      assertScopes(_`~~~ test`);
+    },
+    { message: "Expected a source line before assertion" },
+  );
+
+  assert.throws(
+    () => {
+      assertScopes(
+        //
+        $`foo bar`,
+        _`~~~     example.foo`,
+      );
+    },
+    { message: "Not enough assertions" },
+  );
+
+  assert.throws(() => {
+    assertScopes(
+      //
+      $`foo bar`,
+      _`    ~~~ example.bar`,
+    );
+  }, /Skipped token should have no scopes/);
+
+  assert.throws(
+    () => {
+      assertScopes(
+        //
+        $`blah`,
+        _`~~~~~~ wrong.scope`,
+      );
+    },
+    { message: "No token found matching assertion (0-6: wrong.scope)" },
+  );
+});

test/assert-scopes.ts

@@ -0,0 +1,103 @@
+import type { IToken } from "@shikijs/vscode-textmate";
+import assert from "node:assert/strict";
+import { createHighlighter } from "shiki";
+import type { Grammar, LanguageRegistration } from "shiki";
+
+function assertScopes(
+  grammar: Grammar,
+  rootScopeName: string,
+  ...items: (string | ScopeAssertion)[]
+) {
+  function stripRootScope(scopes: string[]): string[] {
+    if (scopes[0] === rootScopeName) {
+      return scopes.slice(1);
+    }
+    return scopes;
+  }
+
+  let currentAssertion: ScopeAssertion | undefined;
+  try {
+    let state: ReturnType<Grammar["tokenizeLine"]> | undefined;
+    let tokenIndex = 0;
+    for (const item of items) {
+      if (typeof item === "string") {
+        if (state && tokenIndex < state.tokens.length) {
+          assert.fail("Not enough assertions");
+        }
+        state = grammar.tokenizeLine(item, state?.ruleStack ?? null);
+        tokenIndex = 0;
+        continue;
+      }
+      currentAssertion = item;
+      if (!state) {
+        assert.fail("Expected a source line before assertion");
+      }
+      while (
+        tokenIndex < state.tokens.length &&
+        (state.tokens[tokenIndex]!.startIndex !== item.startIndex ||
+          state.tokens[tokenIndex]!.endIndex !== item.endIndex)
+      ) {
+        const token: IToken = state.tokens[tokenIndex]!;
+        assert.deepEqual(stripRootScope(token.scopes), [], "Skipped token should have no scopes");
+        ++tokenIndex;
+      }
+      assert(
+        tokenIndex < state.tokens.length,
+        `No token found matching assertion (${item.startIndex}-${item.endIndex}: ${item.scopes.join(", ")})`,
+      );
+      const token = state.tokens[tokenIndex++]!;
+      assert.deepEqual(stripRootScope(token.scopes), item.scopes);
+    }
+    if (state && tokenIndex < state.tokens.length) {
+      assert.fail("Not enough assertions");
+    }
+  } catch (err) {
+    if (err instanceof assert.AssertionError) {
+      const newStack = currentAssertion?.stack;
+      if (newStack) {
+        const [firstLine] = err.stack?.split("\n", 1) ?? [];
+        err.stack = firstLine ? newStack.replace(/^.*$/m, firstLine) : newStack;
+      } else {
+        // Hide assertScopes from the stack trace
+        Error.captureStackTrace(err, assertScopes);
+      }
+    }
+    throw err;
+  }
+}
+
+const assertionPattern = /~+/g;
+class ScopeAssertion {
+  startIndex: number;
+  endIndex: number;
+  scopes: string[];
+  stack?: string;
+
+  constructor(str: string) {
+    Error.captureStackTrace(this, _); // hide stack frames below and including _
+    assertionPattern.lastIndex = 0;
+    const match = assertionPattern.exec(str);
+    if (!match) {
+      throw new Error("Expected one or more ~ in assertion string");
+    }
+    this.startIndex = match.index;
+    this.endIndex = assertionPattern.lastIndex;
+    this.scopes = str.substring(this.endIndex).trim().split(", ");
+  }
+}
+
+export function $(strings: TemplateStringsArray): string {
+  return strings[0]!;
+}
+
+export function _(strings: TemplateStringsArray): ScopeAssertion {
+  return new ScopeAssertion(strings[0]!);
+}
+
+export async function createAssertScopes(
+  rawGrammar: LanguageRegistration,
+): Promise<(...items: (string | ScopeAssertion)[]) => void> {
+  const highlighter = await createHighlighter({ langs: [rawGrammar], themes: [] });
+  const grammar = highlighter.getInternalContext().getLanguage(rawGrammar.name);
+  return assertScopes.bind(null, grammar, rawGrammar.scopeName);
+}

test/index.test.ts

@@ -0,0 +1,15 @@
+import test from "node:test";
+import swiftGrammar from "../Swift.tmLanguage.json" with { type: "json" };
+
+import { $, _, createAssertScopes } from "./assert-scopes.ts";
+import type { LanguageRegistration } from "shiki";
+
+const assertScopes = await createAssertScopes(swiftGrammar as unknown as LanguageRegistration);
+
+test("basic", () => {
+  assertScopes(
+    $`let x: String`,
+    _`~~~           keyword.other.declaration-specifier.swift`,
+    _`       ~~~~~~ support.type.swift`,
+  );
+});

tsconfig.json

@@ -1,6 +1,9 @@
 {
-  "extends": "@tsconfig/node20/tsconfig.json",
+  "extends": "@tsconfig/node24/tsconfig.json",
   "compilerOptions": {
-    "noUncheckedIndexedAccess": true
+    "noEmit": true,
+    "resolveJsonModule": true,
+    "noUncheckedIndexedAccess": true,
+    "allowImportingTsExtensions": true
   }
 }