Address review: drop runtime simple-html-tokenizer, fix PathExpression original setter

Two review comments from NullVoxPopuli on the cleanup commit:

1. rollup.config.mjs: "remove this. it's internal, and we don't want it.
   this grammar work should negate the need for simple-html-tokenizer
   entirely"

   - Vendor the HTML5 named-entity table as
     packages/@glimmer/syntax/lib/parser/html5-named-char-refs.js (the
     ~2200-entry map from simple-html-tokenizer's generated data, now
     internal to @glimmer/syntax).
   - Inline the entity decoder in tokenizer-event-handlers.ts: handles
     hex (&#xHH;), decimal (&#dd;), and named entities via the vendored
     table.
   - Drop simple-html-tokenizer from @glimmer/syntax/package.json and
     revert the rollup.config resolvePackages central externalization.
     (simple-html-tokenizer is still a testDependency because
     internal-test-helpers and integration-tests use it, but no runtime
     code does anymore.)
   - .prettierignore: exclude the vendored char-ref file (huge one-liner).

2. auto-import-builtins.ts: "this should not need changing. changing
   `original` should mean that the other nodes updates"

   The Peggy grammar was emitting PathExpression nodes as plain objects
   with `original` as a data field, bypassing buildLegacyPath (which
   installs `original` as a getter/setter tied to head/tail). That's why
   the plugin was having to manually rewrite head/tail after assigning
   to `.original`.

   - convertLocations now upgrades every PathExpression through
     buildLegacyPath, so `node.path.original = 'foo.bar'` correctly
     updates head/tail via the setter.
   - Revert the manual head/tail mutation in auto-import-builtins.ts
     back to the original one-line `node.path.original = rewritten`.

Verified: pnpm lint, pnpm build, pnpm vite build --mode=development,
pnpm test (9138/9138 browser tests pass), and Prettier handlebars smoke
test (163/163 pass, 151 snapshots).

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

Author: NullVoxPopuli

.prettierignore

@@ -12,5 +12,6 @@ package.json
 pnpm-lock.yaml
 internal-docs/**/*.md
 packages/@glimmer/syntax/lib/hbs-parser/parser.js
+packages/@glimmer/syntax/lib/parser/html5-named-char-refs.js
 *.peggy
 tracerbench-testing/

packages/@ember/template-compiler/lib/plugins/auto-import-builtins.ts

@@ -51,21 +51,7 @@ function rewriteKeyword(
       name,
     });
   } else if (env.meta?.emberRuntime) {
-    const rewritten = env.meta.emberRuntime.lookupKeyword(name);
-    node.path.original = rewritten;
-    // Also update head and tail so the v2 normalizer resolves correctly.
-    // e.g., "__ember_keywords__.on" → head="__ember_keywords__", tail=["on"]
-    const parts = rewritten.split('.');
-    if (parts.length >= 2) {
-      const headName = parts[0] as string;
-      node.path.head = {
-        type: 'VarHead',
-        name: headName,
-        original: headName,
-        loc: node.path.head.loc,
-      } as AST.VarHead;
-      node.path.tail = parts.slice(1);
-    }
+    node.path.original = env.meta.emberRuntime.lookupKeyword(name);
   }
 }
 

packages/@glimmer/syntax/lib/parser/html5-named-char-refs.js

@@ -1,7 +1,7 @@
 import type { Nullable } from '@glimmer/interfaces';
 import { assign } from '@glimmer/util';
-import { EntityParser, HTML5NamedCharRefs } from 'simple-html-tokenizer';
 import { parseTemplate } from '../hbs-parser/index.js';
+import HTML5NamedCharRefs from './html5-named-char-refs.js';
 
 import type { NodeVisitor } from '../traversal/visitor';
 import type * as ASTv1 from '../v1/api';
@@ -11,20 +11,25 @@ import * as src from '../source/api';
 import { generateSyntaxError } from '../syntax-error';
 import traverse from '../traversal/traverse';
 import Walker from '../traversal/walker';
+import { buildLegacyPath } from '../v1/legacy-interop';
 import publicBuilder from '../v1/public-builders';
 
 // ============================================================================
 // Entity decoding helper
 // ============================================================================
 
-// Full HTML5 entity decoding via simple-html-tokenizer's EntityParser so we
-// match the behavior of the previous tokenizer-based pipeline (~2200 named
-// entities plus numeric &#dd; and &#xHH; forms).
-const entityParser = new EntityParser(HTML5NamedCharRefs);
-const ENTITY_RE = /&(#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);/g;
+// Full HTML5 entity decoding, covering ~2200 named entities plus numeric
+// &#dd; and &#xHH; forms. The named-entity table is vendored from the HTML5
+// spec (see ./html5-named-char-refs.js).
+const ENTITY_RE = /&(#[xX]([0-9a-fA-F]+)|#([0-9]+)|([A-Za-z][A-Za-z0-9]*));/g;
+const NAMED: Record<string, string> = HTML5NamedCharRefs as Record<string, string>;
 
 function decodeEntities(text: string): string {
-  return text.replace(ENTITY_RE, (match, body) => entityParser.parse(body) ?? match);
+  return text.replace(ENTITY_RE, (match, _body, hex, dec, name) => {
+    if (hex !== undefined) return String.fromCodePoint(parseInt(hex, 16));
+    if (dec !== undefined) return String.fromCodePoint(parseInt(dec, 10));
+    return NAMED[name] ?? match;
+  });
 }
 
 // ============================================================================
@@ -469,6 +474,12 @@ function isPeggySpan(
 
 /**
  * Recursively convert plain {start, end} location objects to SourceSpan instances.
+ *
+ * PathExpression nodes are additionally upgraded via `buildLegacyPath` so that
+ * `node.path.original` is a live getter/setter backed by `head` and `tail`.
+ * This way downstream AST plugins (e.g. auto-import-builtins) can assign
+ * `node.path.original = 'foo.bar'` and have `head`/`tail` update automatically,
+ * matching the behavior of the legacy parser-builders-produced nodes.
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function convertLocations(node: any, source: src.Source): any {
@@ -482,6 +493,10 @@ function convertLocations(node: any, source: src.Source): any {
   let result: any = {};
 
   for (let key of Object.keys(node)) {
+    // PathExpression owns its own `original` via buildLegacyPath's getter.
+    // Drop the grammar-emitted data field so the getter wins.
+    if (node.type === 'PathExpression' && key === 'original') continue;
+
     let val = node[key];
     if ((key === 'loc' || key === 'openTag' || key === 'closeTag') && isPeggySpan(val)) {
       result[key] = peggySpanToSourceSpan(source, val);
@@ -492,6 +507,10 @@ function convertLocations(node: any, source: src.Source): any {
     }
   }
 
+  if (result.type === 'PathExpression') {
+    return buildLegacyPath({ head: result.head, tail: result.tail, loc: result.loc });
+  }
+
   return result;
 }
 

packages/@glimmer/syntax/lib/parser/tokenizer-event-handlers.ts

@@ -34,8 +34,7 @@
   "dependencies": {
     "@glimmer/interfaces": "workspace:*",
     "@glimmer/util": "workspace:*",
-    "@glimmer/wire-format": "workspace:*",
-    "simple-html-tokenizer": "^0.5.11"
+    "@glimmer/wire-format": "workspace:*"
   },
   "devDependencies": {
     "@glimmer/debug-util": "workspace:*",

packages/@glimmer/syntax/package.json

@@ -23,6 +23,7 @@ const testDependencies = [
   '@simple-dom/serializer',
   '@simple-dom/void-map',
   'expect-type',
+  'simple-html-tokenizer',
 ];
 
 let configs = [
@@ -427,13 +428,6 @@ export function resolvePackages(deps, params) {
           return { external: true, id: pkgName };
         }
 
-        // simple-html-tokenizer is a real runtime dep (listed in ember-source
-        // package.json and @glimmer/syntax package.json). Keep it external so
-        // consumers resolve it from their own node_modules.
-        if (pkgName === 'simple-html-tokenizer') {
-          return { external: true, id: source };
-        }
-
         if (isExternal?.(source)) {
           return { external: true, id: source };
         }