refactor: split TextReplacer into HtmlTokens and StoredForms (M1)

- HtmlTokens (static, with the S45 docblock): SPLIT, RAW_OPEN, tokenize(),
  structure(), closes(), unterminated() moved unchanged; new RAW_ELEMENTS list.
- StoredForms: of() (was TextReplacer::forms), mentions(), unicodeEscaped() –
  the DB prefilter forms; TextReplacer::mentions() delegates, the ability's
  candidate query uses StoredForms::of().
- TextReplacer keeps problems(), apply() and the splice/safe() verification;
  safe() still runs the tokenizer and HtmlContext as two independent scans, in
  the same order.
HtmlTokensTest asserts RAW_ELEMENTS === HtmlContext::RAW and that RAW_OPEN opens
exactly those elements. TextReplacerTest and HtmlContextTest unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01At3V6SjwbMe6eMzVGW3Hs7

src/Integrations/Divi/Compose/HtmlTokens.php

@@ -0,0 +1,134 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LIT\MCP\Integrations\Divi\Compose;
+
+/**
+ * Ruling S45 tokenizer of lit/divi-replace-text (spec §5.7, rulings S8/S45): splits a string into
+ * tags (quote-aware – a ">" inside a quoted attribute value does not end the tag), comments,
+ * entities and $variable(…)$ references ("token"), the bodies of raw-text elements such as
+ * <script> ("raw") and the rest ("text"). A string with an unterminated tag, quote, comment or
+ * reference has no tokens (null) – nobody can say where its text is. TextReplacer searches text
+ * tokens only and compares the non-text structure before and after; HtmlContext is the second,
+ * independently written scan. Linear time, pure PHP.
+ */
+final class HtmlTokens
+{
+    /**
+     * Rulings S8/S45: comments, tags (quote-aware: a ">" inside a quoted attribute value does not
+     * end the tag; an unquoted "<" does, so an unterminated tag cannot make PCRE rescan the
+     * rest from every "<"), entities and references are tokens, the rest is text. Possessive
+     * quantifiers keep it linear.
+     */
+    public const SPLIT = '/((?s:<!--.*?-->)|<(?:[^<>"\']++|"[^"]*+"|\'[^\']*+\')*+>|&[#A-Za-z0-9]++;'
+        . '|\$variable\(.+?\)\$)/';
+
+    /**
+     * Raw-text elements: their body is code, never text. Must equal HtmlContext::RAW (the second,
+     * independent scan) – HtmlTokensTest asserts it, and that RAW_OPEN opens exactly these.
+     */
+    public const RAW_ELEMENTS = ['script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', 'noscript', 'plaintext'];
+
+    /** Opening tag of a RAW_ELEMENTS element. */
+    private const RAW_OPEN = '/^<(script|style|xmp|iframe|noembed|noframes|noscript|plaintext)\b/i';
+
+    /**
+     * Ruling S45 tokenizer: [type, bytes, offset] with type "token" (tag – quote-aware, ">" in a
+     * quoted value does not end it –, comment, entity, reference), "raw" (body of script, style
+     * and the other raw-text elements) or "text". Null when the value has an unterminated tag,
+     * quote, comment or reference – then nobody can say where its text is.
+     *
+     * @return list<array{string, string, int}>|null
+     */
+    public static function tokenize(string $value): ?array
+    {
+        if (self::unterminated($value)) {
+            return null;
+        }
+
+        $parts = preg_split(self::SPLIT, $value, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE);
+        if (! is_array($parts)) {
+            return null;
+        }
+
+        $tokens = [];
+        $raw = null;
+        foreach ($parts as $i => [$text, $offset]) {
+            $text = (string) $text;
+            if ($i % 2 === 1) {
+                if ($raw === null && preg_match(self::RAW_OPEN, $text, $match) === 1) {
+                    $raw = strtolower($match[1]);
+                } elseif ($raw !== null && self::closes($raw, $text)) {
+                    $raw = null;
+                }
+                $tokens[] = ['token', $text, (int) $offset];
+                continue;
+            }
+
+            if ($text === '') {
+                continue;
+            }
+
+            // A "<" that starts markup but matched no token: an unterminated tag or quote.
+            if ($raw === null && preg_match('#<[A-Za-z/!?]#', $text) === 1) {
+                return null;
+            }
+            $tokens[] = [$raw === null ? 'text' : 'raw', $text, (int) $offset];
+        }
+
+        return $tokens;
+    }
+
+    /** Whether token $text closes raw-text element $raw ("</script-x>" does not; plaintext never ends). */
+    private static function closes(string $raw, string $text): bool
+    {
+        return $raw !== 'plaintext' && preg_match('#^</' . $raw . '[\s/>]#i', $text) === 1;
+    }
+
+    /**
+     * Cheap checks that keep the tokenizer linear: a comment opened after the last "-->", or a
+     * reference marker after the last ")$" of its line, can never be closed – PCRE would retry
+     * it from every later opener.
+     */
+    private static function unterminated(string $value): bool
+    {
+        $open = strrpos($value, '<!--');
+        if ($open !== false) {
+            $close = strrpos($value, '-->');
+            if ($close === false || $close < $open + 2) {
+                return true;
+            }
+        }
+
+        if (! str_contains($value, VariableRef::MARKER)) {
+            return false;
+        }
+
+        foreach (explode("\n", $value) as $line) {
+            $marker = strrpos($line, VariableRef::MARKER);
+            $end = $marker === false ? false : strrpos($line, VariableRef::END);
+            if ($marker !== false && ($end === false || $end < $marker)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @param list<array{string, string, int}> $tokens
+     * @return list<array{string, string}> The non-text tokens (type and bytes), in order.
+     */
+    public static function structure(array $tokens): array
+    {
+        $structure = [];
+        foreach ($tokens as [$type, $text]) {
+            if ($type !== 'text') {
+                $structure[] = [$type, $text];
+            }
+        }
+
+        return $structure;
+    }
+}