refactor: split TextReplacer into HtmlTokens and StoredForms (M1)
- HtmlTokens (static, with the S45 docblock): SPLIT, RAW_OPEN, tokenize(), structure(), closes(), unterminated() moved unchanged; new RAW_ELEMENTS list. - StoredForms: of() (was TextReplacer::forms), mentions(), unicodeEscaped() – the DB prefilter forms; TextReplacer::mentions() delegates, the ability's candidate query uses StoredForms::of(). - TextReplacer keeps problems(), apply() and the splice/safe() verification; safe() still runs the tokenizer and HtmlContext as two independent scans, in the same order. HtmlTokensTest asserts RAW_ELEMENTS === HtmlContext::RAW and that RAW_OPEN opens exactly those elements. TextReplacerTest and HtmlContextTest unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01At3V6SjwbMe6eMzVGW3Hs7
This commit is contained in:
Claude Opus 5.5
@@ -7,7 +7,6 @@ namespace LIT\MCP\Integrations\Divi\Compose;
|
||||
use InvalidArgumentException;
|
||||
use LIT\MCP\Integrations\Divi\BlockTree\AttrMap;
|
||||
use LIT\MCP\Integrations\Divi\BlockTree\NodeSummary;
|
||||
use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
|
||||
|
||||
/**
|
||||
* Replaces visible text in the attributes of one block (lit/divi-replace-text, spec §5.7, D10):
|
||||
@@ -15,8 +14,8 @@ use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
|
||||
* title, label, alt, titleText) – never in URLs, IDs, styles or $variable(…)$ references.
|
||||
* Pure PHP.
|
||||
*
|
||||
* Rulings S8/S45: each string is tokenized (quote-aware tags, comments, entities, references,
|
||||
* raw-text bodies such as <script>); only text tokens are searched. A string with an
|
||||
* Rulings S8/S45: each string is tokenized by HtmlTokens (quote-aware tags, comments, entities,
|
||||
* references, raw-text bodies such as <script>); only text tokens are searched. A string with an
|
||||
* unterminated tag, quote, comment or reference and a hit is refused. The inserted text is the
|
||||
* replacement as the $clean callback returns it for that place (ValueCleaner::textReplacementFor()
|
||||
* in the ability: sanitized by the field kind, markers rejected – rulings S2/S28). A string with no
|
||||
@@ -28,20 +27,8 @@ use LIT\MCP\Integrations\Divi\BlockTree\Serializer;
|
||||
*/
|
||||
final class TextReplacer
|
||||
{
|
||||
/**
|
||||
* Rulings S8/S45: comments, tags (quote-aware: a ">" inside a quoted attribute value does not
|
||||
* end the tag; an unquoted "<" does, so an unterminated tag cannot make PCRE rescan the
|
||||
* rest from every "<"), entities and references are tokens, the rest is text. Possessive
|
||||
* quantifiers keep it linear.
|
||||
*/
|
||||
public const SPLIT = '/((?s:<!--.*?-->)|<(?:[^<>"\']++|"[^"]*+"|\'[^\']*+\')*+>|&[#A-Za-z0-9]++;'
|
||||
. '|\$variable\(.+?\)\$)/';
|
||||
|
||||
private const TEXT_KEYS = ['text', 'title', 'label', 'alt', 'titleText'];
|
||||
|
||||
/** Raw-text elements (HtmlContext::RAW): their body is code, never text. */
|
||||
private const RAW_OPEN = '/^<(script|style|xmp|iframe|noembed|noframes|noscript|plaintext)\b/i';
|
||||
|
||||
private const CONTEXT = 30;
|
||||
|
||||
private string $pattern;
|
||||
@@ -89,47 +76,10 @@ final class TextReplacer
|
||||
return $problems;
|
||||
}
|
||||
|
||||
/**
|
||||
* The search text as it can be stored in block markup: raw, as serialize_block_attributes()
|
||||
* escapes it (<, >, &, " and --; the JS builder keeps non-ASCII raw) and with \uXXXX for
|
||||
* non-ASCII (WordPress' PHP serializer). Case-insensitive adds the lower and upper case
|
||||
* forms, since the hex digits of \u escapes do not fold.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public static function forms(string $search, bool $caseSensitive): array
|
||||
{
|
||||
$variants = $caseSensitive ? [$search] : [$search, mb_strtolower($search), mb_strtoupper($search)];
|
||||
$forms = [];
|
||||
foreach ($variants as $variant) {
|
||||
$escaped = substr(Serializer::attributes(['s' => $variant]), 6, -2);
|
||||
array_push($forms, $variant, $escaped, self::unicodeEscaped($escaped));
|
||||
}
|
||||
|
||||
return array_values(array_unique($forms));
|
||||
}
|
||||
|
||||
/** Whether block markup (e.g. of a global module this tool does not edit) contains the search text. */
|
||||
public function mentions(string $markup): bool
|
||||
{
|
||||
foreach (self::forms($this->search, $this->caseSensitive) as $form) {
|
||||
$found = $this->caseSensitive ? str_contains($markup, $form) : mb_stripos($markup, $form) !== false;
|
||||
if ($found) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Non-ASCII characters as JSON \uXXXX escapes (UTF-16, lower-case hex like json_encode()). */
|
||||
private static function unicodeEscaped(string $value): string
|
||||
{
|
||||
return (string) preg_replace_callback('/[^\x00-\x7F]/u', static function (array $match): string {
|
||||
$units = unpack('n*', (string) mb_convert_encoding($match[0], 'UTF-16BE', 'UTF-8'));
|
||||
|
||||
return implode('', array_map(static fn (int $unit): string => sprintf('\\u%04x', $unit), (array) $units));
|
||||
}, $value);
|
||||
return StoredForms::mentions($markup, $this->search, $this->caseSensitive);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -199,7 +149,7 @@ final class TextReplacer
|
||||
array &$samples
|
||||
): string {
|
||||
$place = $path . ($key === null ? '' : '.' . $key);
|
||||
$tokens = self::tokenize($value);
|
||||
$tokens = HtmlTokens::tokenize($value);
|
||||
if ($tokens === null) {
|
||||
// Unterminated markup, quotes, comments or references: where does the text end? Never guess.
|
||||
if (preg_match($this->pattern, $value) === 1) {
|
||||
@@ -226,7 +176,7 @@ final class TextReplacer
|
||||
return $value;
|
||||
}
|
||||
|
||||
$inserted = self::tokenize($replacement);
|
||||
$inserted = HtmlTokens::tokenize($replacement);
|
||||
if ($inserted === null) {
|
||||
throw self::refused($place);
|
||||
}
|
||||
@@ -290,7 +240,7 @@ final class TextReplacer
|
||||
string $replacement,
|
||||
array $inserted
|
||||
): array {
|
||||
$own = self::structure($inserted);
|
||||
$own = HtmlTokens::structure($inserted);
|
||||
$replaced = '';
|
||||
$ranges = [];
|
||||
$expected = [];
|
||||
@@ -343,9 +293,9 @@ final class TextReplacer
|
||||
$tokens,
|
||||
static fn (array $token): bool => $token[0] === 'token' && str_starts_with($token[1], VariableRef::MARKER)
|
||||
));
|
||||
$after = self::tokenize($replaced);
|
||||
$after = HtmlTokens::tokenize($replaced);
|
||||
$sameMarkers = $markers === $references && substr_count($replaced, VariableRef::MARKER) === $markers;
|
||||
if (! $sameMarkers || $after === null || self::structure($after) !== $expected) {
|
||||
if (! $sameMarkers || $after === null || HtmlTokens::structure($after) !== $expected) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -366,105 +316,6 @@ final class TextReplacer
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{string, string, int}> $tokens
|
||||
* @return list<array{string, string}> The non-text tokens (type and bytes), in order.
|
||||
*/
|
||||
private static function structure(array $tokens): array
|
||||
{
|
||||
$structure = [];
|
||||
foreach ($tokens as [$type, $text]) {
|
||||
if ($type !== 'text') {
|
||||
$structure[] = [$type, $text];
|
||||
}
|
||||
}
|
||||
|
||||
return $structure;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ruling S45 tokenizer: [type, bytes, offset] with type "token" (tag – quote-aware, ">" in a
|
||||
* quoted value does not end it –, comment, entity, reference), "raw" (body of script, style
|
||||
* and the other raw-text elements) or "text". Null when the value has an unterminated tag,
|
||||
* quote, comment or reference – then nobody can say where its text is.
|
||||
*
|
||||
* @return list<array{string, string, int}>|null
|
||||
*/
|
||||
private static function tokenize(string $value): ?array
|
||||
{
|
||||
if (self::unterminated($value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$parts = preg_split(self::SPLIT, $value, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE);
|
||||
if (! is_array($parts)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$tokens = [];
|
||||
$raw = null;
|
||||
foreach ($parts as $i => [$text, $offset]) {
|
||||
$text = (string) $text;
|
||||
if ($i % 2 === 1) {
|
||||
if ($raw === null && preg_match(self::RAW_OPEN, $text, $match) === 1) {
|
||||
$raw = strtolower($match[1]);
|
||||
} elseif ($raw !== null && self::closes($raw, $text)) {
|
||||
$raw = null;
|
||||
}
|
||||
$tokens[] = ['token', $text, (int) $offset];
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($text === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// A "<" that starts markup but matched no token: an unterminated tag or quote.
|
||||
if ($raw === null && preg_match('#<[A-Za-z/!?]#', $text) === 1) {
|
||||
return null;
|
||||
}
|
||||
$tokens[] = [$raw === null ? 'text' : 'raw', $text, (int) $offset];
|
||||
}
|
||||
|
||||
return $tokens;
|
||||
}
|
||||
|
||||
/** Whether token $text closes raw-text element $raw ("</script-x>" does not; plaintext never ends). */
|
||||
private static function closes(string $raw, string $text): bool
|
||||
{
|
||||
return $raw !== 'plaintext' && preg_match('#^</' . $raw . '[\s/>]#i', $text) === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap checks that keep the tokenizer linear: a comment opened after the last "-->", or a
|
||||
* reference marker after the last ")$" of its line, can never be closed – PCRE would retry
|
||||
* it from every later opener.
|
||||
*/
|
||||
private static function unterminated(string $value): bool
|
||||
{
|
||||
$open = strrpos($value, '<!--');
|
||||
if ($open !== false) {
|
||||
$close = strrpos($value, '-->');
|
||||
if ($close === false || $close < $open + 2) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (! str_contains($value, VariableRef::MARKER)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (explode("\n", $value) as $line) {
|
||||
$marker = strrpos($line, VariableRef::MARKER);
|
||||
$end = $marker === false ? false : strrpos($line, VariableRef::END);
|
||||
if ($marker !== false && ($end === false || $end < $marker)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/** @param callable(string, ?string, string): ?string $clean */
|
||||
private function replacement(string $path, ?string $key, callable $clean): ?string
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user