refactor: split TextReplacer into HtmlTokens and StoredForms (M1)

- HtmlTokens (static, with the S45 docblock): SPLIT, RAW_OPEN, tokenize(),
  structure(), closes(), unterminated() moved unchanged; new RAW_ELEMENTS list.
- StoredForms: of() (was TextReplacer::forms), mentions(), unicodeEscaped() –
  the DB prefilter forms; TextReplacer::mentions() delegates, the ability's
  candidate query uses StoredForms::of().
- TextReplacer keeps problems(), apply() and the splice/safe() verification;
  safe() still runs the tokenizer and HtmlContext as two independent scans, in
  the same order.
HtmlTokensTest asserts RAW_ELEMENTS === HtmlContext::RAW and that RAW_OPEN opens
exactly those elements. TextReplacerTest and HtmlContextTest unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01At3V6SjwbMe6eMzVGW3Hs7

src/Integrations/Divi/Compose/HtmlContext.php

@@ -6,7 +6,7 @@ namespace LIT\MCP\Integrations\Divi\Compose;
 
 /**
  * An independent second scan of an HTML fragment (ruling S45): a small state machine after the
- * HTML tokenizer rules, written separately from TextReplacer's regex tokenizer so a mistake in
+ * HTML tokenizer rules, written separately from the regex tokenizer HtmlTokens so a mistake in
  * one is caught by the other. It lists the byte spans that are NOT text: tags (with quoted
  * attribute values, where ">" does not end the tag), comments, bogus comments and the bodies of
  * raw-text elements (script, style, …). Where it is unsure it errs towards "not text": an
@@ -15,7 +15,7 @@ namespace LIT\MCP\Integrations\Divi\Compose;
  */
 final class HtmlContext
 {
-    /** Elements whose body the browser does not parse as text/markup. */
+    /** Elements whose body the browser does not parse as text/markup (= HtmlTokens::RAW_ELEMENTS). */
     public const RAW = ['script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', 'noscript', 'plaintext'];
 
     /** @return list<array{int, int}> [start, end) byte spans that are markup or raw bodies. */

src/Integrations/Divi/Compose/HtmlTokens.php

@@ -0,0 +1,134 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LIT\MCP\Integrations\Divi\Compose;
+
+/**
+ * Ruling S45 tokenizer of lit/divi-replace-text (spec §5.7, rulings S8/S45): splits a string into
+ * tags (quote-aware – a ">" inside a quoted attribute value does not end the tag), comments,
+ * entities and $variable(…)$ references ("token"), the bodies of raw-text elements such as
+ * <script> ("raw") and the rest ("text"). A string with an unterminated tag, quote, comment or
+ * reference has no tokens (null) – nobody can say where its text is. TextReplacer searches text
+ * tokens only and compares the non-text structure before and after; HtmlContext is the second,
+ * independently written scan. Linear time, pure PHP.
+ */
+final class HtmlTokens
+{
+    /**
+     * Rulings S8/S45: comments, tags (quote-aware: a ">" inside a quoted attribute value does not
+     * end the tag; an unquoted "<" does, so an unterminated tag cannot make PCRE rescan the
+     * rest from every "<"), entities and references are tokens, the rest is text. Possessive
+     * quantifiers keep it linear.
+     */
+    public const SPLIT = '/((?s:<!--.*?-->)|<(?:[^<>"\']++|"[^"]*+"|\'[^\']*+\')*+>|&[#A-Za-z0-9]++;'
+        . '|\$variable\(.+?\)\$)/';
+
+    /**
+     * Raw-text elements: their body is code, never text. Must equal HtmlContext::RAW (the second,
+     * independent scan) – HtmlTokensTest asserts it, and that RAW_OPEN opens exactly these.
+     */
+    public const RAW_ELEMENTS = ['script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', 'noscript', 'plaintext'];
+
+    /** Opening tag of a RAW_ELEMENTS element. */
+    private const RAW_OPEN = '/^<(script|style|xmp|iframe|noembed|noframes|noscript|plaintext)\b/i';
+
+    /**
+     * Ruling S45 tokenizer: [type, bytes, offset] with type "token" (tag – quote-aware, ">" in a
+     * quoted value does not end it –, comment, entity, reference), "raw" (body of script, style
+     * and the other raw-text elements) or "text". Null when the value has an unterminated tag,
+     * quote, comment or reference – then nobody can say where its text is.
+     *
+     * @return list<array{string, string, int}>|null
+     */
+    public static function tokenize(string $value): ?array
+    {
+        if (self::unterminated($value)) {
+            return null;
+        }
+
+        $parts = preg_split(self::SPLIT, $value, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE);
+        if (! is_array($parts)) {
+            return null;
+        }
+
+        $tokens = [];
+        $raw = null;
+        foreach ($parts as $i => [$text, $offset]) {
+            $text = (string) $text;
+            if ($i % 2 === 1) {
+                if ($raw === null && preg_match(self::RAW_OPEN, $text, $match) === 1) {
+                    $raw = strtolower($match[1]);
+                } elseif ($raw !== null && self::closes($raw, $text)) {
+                    $raw = null;
+                }
+                $tokens[] = ['token', $text, (int) $offset];
+                continue;
+            }
+
+            if ($text === '') {
+                continue;
+            }
+
+            // A "<" that starts markup but matched no token: an unterminated tag or quote.
+            if ($raw === null && preg_match('#<[A-Za-z/!?]#', $text) === 1) {
+                return null;
+            }
+            $tokens[] = [$raw === null ? 'text' : 'raw', $text, (int) $offset];
+        }
+
+        return $tokens;
+    }
+
+    /** Whether token $text closes raw-text element $raw ("</script-x>" does not; plaintext never ends). */
+    private static function closes(string $raw, string $text): bool
+    {
+        return $raw !== 'plaintext' && preg_match('#^</' . $raw . '[\s/>]#i', $text) === 1;
+    }
+
+    /**
+     * Cheap checks that keep the tokenizer linear: a comment opened after the last "-->", or a
+     * reference marker after the last ")$" of its line, can never be closed – PCRE would retry
+     * it from every later opener.
+     */
+    private static function unterminated(string $value): bool
+    {
+        $open = strrpos($value, '<!--');
+        if ($open !== false) {
+            $close = strrpos($value, '-->');
+            if ($close === false || $close < $open + 2) {
+                return true;
+            }
+        }
+
+        if (! str_contains($value, VariableRef::MARKER)) {
+            return false;
+        }
+
+        foreach (explode("\n", $value) as $line) {
+            $marker = strrpos($line, VariableRef::MARKER);
+            $end = $marker === false ? false : strrpos($line, VariableRef::END);
+            if ($marker !== false && ($end === false || $end < $marker)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @param list<array{string, string, int}> $tokens
+     * @return list<array{string, string}> The non-text tokens (type and bytes), in order.
+     */
+    public static function structure(array $tokens): array
+    {
+        $structure = [];
+        foreach ($tokens as [$type, $text]) {
+            if ($type !== 'text') {
+                $structure[] = [$type, $text];
+            }
+        }
+
+        return $structure;
+    }
+}

src/Integrations/Divi/Compose/StoredForms.php

@@ -0,0 +1,58 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LIT\MCP\Integrations\Divi\Compose;
+
+use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
+
+/**
+ * The forms a search text can take in stored block markup – the database prefilter of
+ * lit/divi-replace-text and the check for global modules it does not edit. Only a prefilter:
+ * the exact, safe match happens in the tree (TextReplacer). Pure PHP.
+ */
+final class StoredForms
+{
+    /**
+     * The search text as it can be stored in block markup: raw, as serialize_block_attributes()
+     * escapes it (<, >, &, " and --; the JS builder keeps non-ASCII raw) and with \uXXXX for
+     * non-ASCII (WordPress' PHP serializer). Case-insensitive adds the lower and upper case
+     * forms, since the hex digits of \u escapes do not fold.
+     *
+     * @return list<string>
+     */
+    public static function of(string $search, bool $caseSensitive): array
+    {
+        $variants = $caseSensitive ? [$search] : [$search, mb_strtolower($search), mb_strtoupper($search)];
+        $forms = [];
+        foreach ($variants as $variant) {
+            $escaped = substr(Serializer::attributes(['s' => $variant]), 6, -2);
+            array_push($forms, $variant, $escaped, self::unicodeEscaped($escaped));
+        }
+
+        return array_values(array_unique($forms));
+    }
+
+    /** Whether block markup contains $search in one of its stored forms. */
+    public static function mentions(string $markup, string $search, bool $caseSensitive): bool
+    {
+        foreach (self::of($search, $caseSensitive) as $form) {
+            $found = $caseSensitive ? str_contains($markup, $form) : mb_stripos($markup, $form) !== false;
+            if ($found) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /** Non-ASCII characters as JSON \uXXXX escapes (UTF-16, lower-case hex like json_encode()). */
+    private static function unicodeEscaped(string $value): string
+    {
+        return (string) preg_replace_callback('/[^\x00-\x7F]/u', static function (array $match): string {
+            $units = unpack('n*', (string) mb_convert_encoding($match[0], 'UTF-16BE', 'UTF-8'));
+
+            return implode('', array_map(static fn (int $unit): string => sprintf('\\u%04x', $unit), (array) $units));
+        }, $value);
+    }
+}

src/Integrations/Divi/Compose/TextReplacer.php

@@ -7,7 +7,6 @@ namespace LIT\MCP\Integrations\Divi\Compose;
 use InvalidArgumentException;
 use LIT\MCP\Integrations\Divi\BlockTree\AttrMap;
 use LIT\MCP\Integrations\Divi\BlockTree\NodeSummary;
-use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
 
 /**
  * Replaces visible text in the attributes of one block (lit/divi-replace-text, spec §5.7, D10):
@@ -15,8 +14,8 @@ use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
  * title, label, alt, titleText) – never in URLs, IDs, styles or $variable(…)$ references.
  * Pure PHP.
  *
- * Rulings S8/S45: each string is tokenized (quote-aware tags, comments, entities, references,
+ * Rulings S8/S45: each string is tokenized by HtmlTokens (quote-aware tags, comments, entities,
- * raw-text bodies such as <script>); only text tokens are searched. A string with an
+ * references, raw-text bodies such as <script>); only text tokens are searched. A string with an
  * unterminated tag, quote, comment or reference and a hit is refused. The inserted text is the
  * replacement as the $clean callback returns it for that place (ValueCleaner::textReplacementFor()
  * in the ability: sanitized by the field kind, markers rejected – rulings S2/S28). A string with no
@@ -28,20 +27,8 @@ use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
  */
 final class TextReplacer
 {
-    /**
-     * Rulings S8/S45: comments, tags (quote-aware: a ">" inside a quoted attribute value does not
-     * end the tag; an unquoted "<" does, so an unterminated tag cannot make PCRE rescan the
-     * rest from every "<"), entities and references are tokens, the rest is text. Possessive
-     * quantifiers keep it linear.
-     */
-    public const SPLIT = '/((?s:<!--.*?-->)|<(?:[^<>"\']++|"[^"]*+"|\'[^\']*+\')*+>|&[#A-Za-z0-9]++;'
-        . '|\$variable\(.+?\)\$)/';
-
     private const TEXT_KEYS = ['text', 'title', 'label', 'alt', 'titleText'];
 
-    /** Raw-text elements (HtmlContext::RAW): their body is code, never text. */
-    private const RAW_OPEN = '/^<(script|style|xmp|iframe|noembed|noframes|noscript|plaintext)\b/i';
-
     private const CONTEXT = 30;
 
     private string $pattern;
@@ -89,47 +76,10 @@ final class TextReplacer
         return $problems;
     }
 
-    /**
-     * The search text as it can be stored in block markup: raw, as serialize_block_attributes()
-     * escapes it (<, >, &, " and --; the JS builder keeps non-ASCII raw) and with \uXXXX for
-     * non-ASCII (WordPress' PHP serializer). Case-insensitive adds the lower and upper case
-     * forms, since the hex digits of \u escapes do not fold.
-     *
-     * @return list<string>
-     */
-    public static function forms(string $search, bool $caseSensitive): array
-    {
-        $variants = $caseSensitive ? [$search] : [$search, mb_strtolower($search), mb_strtoupper($search)];
-        $forms = [];
-        foreach ($variants as $variant) {
-            $escaped = substr(Serializer::attributes(['s' => $variant]), 6, -2);
-            array_push($forms, $variant, $escaped, self::unicodeEscaped($escaped));
-        }
-
-        return array_values(array_unique($forms));
-    }
-
     /** Whether block markup (e.g. of a global module this tool does not edit) contains the search text. */
     public function mentions(string $markup): bool
     {
-        foreach (self::forms($this->search, $this->caseSensitive) as $form) {
+        return StoredForms::mentions($markup, $this->search, $this->caseSensitive);
-            $found = $this->caseSensitive ? str_contains($markup, $form) : mb_stripos($markup, $form) !== false;
-            if ($found) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    /** Non-ASCII characters as JSON \uXXXX escapes (UTF-16, lower-case hex like json_encode()). */
-    private static function unicodeEscaped(string $value): string
-    {
-        return (string) preg_replace_callback('/[^\x00-\x7F]/u', static function (array $match): string {
-            $units = unpack('n*', (string) mb_convert_encoding($match[0], 'UTF-16BE', 'UTF-8'));
-
-            return implode('', array_map(static fn (int $unit): string => sprintf('\\u%04x', $unit), (array) $units));
-        }, $value);
     }
 
     /**
@@ -199,7 +149,7 @@ final class TextReplacer
         array &$samples
     ): string {
         $place = $path . ($key === null ? '' : '.' . $key);
-        $tokens = self::tokenize($value);
+        $tokens = HtmlTokens::tokenize($value);
         if ($tokens === null) {
             // Unterminated markup, quotes, comments or references: where does the text end? Never guess.
             if (preg_match($this->pattern, $value) === 1) {
@@ -226,7 +176,7 @@ final class TextReplacer
             return $value;
         }
 
-        $inserted = self::tokenize($replacement);
+        $inserted = HtmlTokens::tokenize($replacement);
         if ($inserted === null) {
             throw self::refused($place);
         }
@@ -290,7 +240,7 @@ final class TextReplacer
         string $replacement,
         array $inserted
     ): array {
-        $own = self::structure($inserted);
+        $own = HtmlTokens::structure($inserted);
         $replaced = '';
         $ranges = [];
         $expected = [];
@@ -343,9 +293,9 @@ final class TextReplacer
             $tokens,
             static fn (array $token): bool => $token[0] === 'token' && str_starts_with($token[1], VariableRef::MARKER)
         ));
-        $after = self::tokenize($replaced);
+        $after = HtmlTokens::tokenize($replaced);
         $sameMarkers = $markers === $references && substr_count($replaced, VariableRef::MARKER) === $markers;
-        if (! $sameMarkers || $after === null || self::structure($after) !== $expected) {
+        if (! $sameMarkers || $after === null || HtmlTokens::structure($after) !== $expected) {
             return false;
         }
 
@@ -366,105 +316,6 @@ final class TextReplacer
         return true;
     }
 
-    /**
-     * @param list<array{string, string, int}> $tokens
-     * @return list<array{string, string}> The non-text tokens (type and bytes), in order.
-     */
-    private static function structure(array $tokens): array
-    {
-        $structure = [];
-        foreach ($tokens as [$type, $text]) {
-            if ($type !== 'text') {
-                $structure[] = [$type, $text];
-            }
-        }
-
-        return $structure;
-    }
-
-    /**
-     * Ruling S45 tokenizer: [type, bytes, offset] with type "token" (tag – quote-aware, ">" in a
-     * quoted value does not end it –, comment, entity, reference), "raw" (body of script, style
-     * and the other raw-text elements) or "text". Null when the value has an unterminated tag,
-     * quote, comment or reference – then nobody can say where its text is.
-     *
-     * @return list<array{string, string, int}>|null
-     */
-    private static function tokenize(string $value): ?array
-    {
-        if (self::unterminated($value)) {
-            return null;
-        }
-
-        $parts = preg_split(self::SPLIT, $value, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE);
-        if (! is_array($parts)) {
-            return null;
-        }
-
-        $tokens = [];
-        $raw = null;
-        foreach ($parts as $i => [$text, $offset]) {
-            $text = (string) $text;
-            if ($i % 2 === 1) {
-                if ($raw === null && preg_match(self::RAW_OPEN, $text, $match) === 1) {
-                    $raw = strtolower($match[1]);
-                } elseif ($raw !== null && self::closes($raw, $text)) {
-                    $raw = null;
-                }
-                $tokens[] = ['token', $text, (int) $offset];
-                continue;
-            }
-
-            if ($text === '') {
-                continue;
-            }
-
-            // A "<" that starts markup but matched no token: an unterminated tag or quote.
-            if ($raw === null && preg_match('#<[A-Za-z/!?]#', $text) === 1) {
-                return null;
-            }
-            $tokens[] = [$raw === null ? 'text' : 'raw', $text, (int) $offset];
-        }
-
-        return $tokens;
-    }
-
-    /** Whether token $text closes raw-text element $raw ("</script-x>" does not; plaintext never ends). */
-    private static function closes(string $raw, string $text): bool
-    {
-        return $raw !== 'plaintext' && preg_match('#^</' . $raw . '[\s/>]#i', $text) === 1;
-    }
-
-    /**
-     * Cheap checks that keep the tokenizer linear: a comment opened after the last "-->", or a
-     * reference marker after the last ")$" of its line, can never be closed – PCRE would retry
-     * it from every later opener.
-     */
-    private static function unterminated(string $value): bool
-    {
-        $open = strrpos($value, '<!--');
-        if ($open !== false) {
-            $close = strrpos($value, '-->');
-            if ($close === false || $close < $open + 2) {
-                return true;
-            }
-        }
-
-        if (! str_contains($value, VariableRef::MARKER)) {
-            return false;
-        }
-
-        foreach (explode("\n", $value) as $line) {
-            $marker = strrpos($line, VariableRef::MARKER);
-            $end = $marker === false ? false : strrpos($line, VariableRef::END);
-            if ($marker !== false && ($end === false || $end < $marker)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
     /** @param callable(string, ?string, string): ?string $clean */
     private function replacement(string $path, ?string $key, callable $clean): ?string
     {

src/Integrations/Divi/Pages/DiviReplaceTextAbility.php

@@ -12,6 +12,7 @@ use LIT\MCP\Abilities\ToolResult;
 use LIT\MCP\Abilities\WriteFailed;
 use LIT\MCP\Integrations\Divi\BlockTree\Block;
 use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
+use LIT\MCP\Integrations\Divi\Compose\StoredForms;
 use LIT\MCP\Integrations\Divi\Compose\TextReplacer;
 use LIT\MCP\Integrations\WordPress\PostGuard;
 use LIT\MCP\Integrations\WordPress\Schema\ContentSchema;
@@ -426,14 +427,14 @@ final class DiviReplaceTextAbility implements Ability
 
     /**
      * IDs of Divi entries whose content contains the search text in one of its stored forms
-     * (TextReplacer::forms()). The exact match happens in the tree; this is only a prefilter.
+     * (StoredForms::of()). The exact match happens in the tree; this is only a prefilter.
      *
      * @return list<int>
      */
     private function candidates(string $postType, string $search, bool $caseSensitive, int $limit, int $offset): array
     {
         $db = $this->db();
-        $forms = TextReplacer::forms($search, $caseSensitive);
+        $forms = StoredForms::of($search, $caseSensitive);
         $likes = array_map(
             static fn (string $form): string => '%' . $db->esc_like($form) . '%',
             $forms

tests/Unit/Integrations/Divi/Compose/HtmlTokensTest.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LIT\MCP\Tests\Unit\Integrations\Divi\Compose;
+
+use LIT\MCP\Integrations\Divi\Compose\HtmlContext;
+use LIT\MCP\Integrations\Divi\Compose\HtmlTokens;
+use PHPUnit\Framework\TestCase;
+
+/** Ruling S45: the tokenizer and the independent scan (HtmlContext) share one raw-text element list. */
+final class HtmlTokensTest extends TestCase
+{
+    public function testRawElementsAreHtmlContextsList(): void
+    {
+        self::assertSame(HtmlContext::RAW, HtmlTokens::RAW_ELEMENTS);
+    }
+
+    /** RAW_OPEN opens exactly the RAW_ELEMENTS: their body is "raw", other elements' body is text. */
+    public function testRawOpenMatchesTheList(): void
+    {
+        foreach (HtmlTokens::RAW_ELEMENTS as $element) {
+            $tokens = HtmlTokens::tokenize('<' . strtoupper($element) . ' a="1">x</' . $element . '>y');
+            // plaintext never ends: everything after it stays raw.
+            $after = $element === 'plaintext' ? 'raw' : 'text';
+            self::assertSame(['token', 'raw', 'token', $after], array_column((array) $tokens, 0), $element);
+        }
+
+        foreach (['div', 'scripts', 'styles', 'textarea', 'title'] as $element) {
+            $tokens = HtmlTokens::tokenize('<' . $element . '>x</' . $element . '>');
+            self::assertSame(['token', 'text', 'token'], array_column((array) $tokens, 0), $element);
+        }
+    }
+}