38542847508

Committed 14 Oct 2026 09:30AM UTC coverage: 84.066% (+4.2%) from 79.825%

Build # 38542847508

Build Type

Pull #108

github

Committed by

koertho

Commit Message

address review comments on resolver and tokenizer

- Look up stemmer and normalizer in explicit methods that check the service
  against its interface, instead of a generic helper
- Transliterate letters without a decomposition into base letter and accent
  (æ, ø, œ, ð, þ, ł, đ, ı), accented letters were already folded

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Pull Request Pull Request #108: add search vector field

Coverage Stats

392 of 407 new or added lines in 16 files covered. (96.31%)

1588 of 1889 relevant lines covered (84.07%)

8.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.67

/src/SearchVector/SearchVectorIndexer.php

<?php

namespace HeimrichHannot\UtilsBundle\SearchVector;

use Contao\CoreBundle\DataContainer\VirtualFieldsHandler;
use Contao\DataContainer;
use Contao\StringUtil;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Exception as DbalException;
use HeimrichHannot\UtilsBundle\Event\BuildSearchVectorEvent;
use HeimrichHannot\UtilsBundle\SearchVector\Messenger\RebuildSearchVectorMessage;
use HeimrichHannot\UtilsBundle\Util\FormatterUtil;
use HeimrichHannot\UtilsBundle\Util\FormatterUtil\FormatDcaFieldValueOptions;
use Psr\Container\ContainerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\Messenger\MessageBusInterface;
use Symfony\Component\Messenger\Stamp\TransportNamesStamp;
use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;

/**
 * Builds and stores the search vector of records whose table registered a SearchVectorField.
 *
 * Format: the tokens of one value are joined by a space, values are separated by ` | ` and the whole string is
 * wrapped in `| … |`, e.g. `| fuhrung durch die altstadt | sommer | stadtrundgange |` (without stemmer). The space
 * keeps word adjacency for quoted phrase searches, the pipe stops a phrase from matching across two unrelated values.
 *
 * Option labels are resolved and words are stemmed in the language of the record (see
 * SearchVectorFieldConfiguration::setLanguage() and setLanguageField()), never in the language of the current user.
 */
class SearchVectorIndexer
{
    /**
     * The messenger transport of asynchronous rebuilds, configured by the Contao Managed Edition.
     */
    public const TRANSPORT = 'contao_prio_normal';

    private const VALUE_SEPARATOR = ' | ';

    /**
     * Joins array values in the formatter output, so each array value stays a separate value of the vector.
     */
    private const ARRAY_JOINER = "\x1F";

    public function __construct(
        private readonly Connection $connection,
        private readonly SearchVectorConfigurationResolver $configurationResolver,
        private readonly FormatterUtil $formatterUtil,
        private readonly EventDispatcherInterface $eventDispatcher,
        private readonly LoggerInterface $logger,
        private readonly SearchVectorLanguageSwitcher $languageSwitcher,
        private readonly ?MessageBusInterface $messageBus = null,
        private readonly ?ContainerInterface $messengerTransports = null,
        private readonly ?VirtualFieldsHandler $virtualFieldsHandler = null,
    ) {}

    /**
     * Rebuilds the search vector of a single record asynchronously.
     *
     * Sends a message to the contao_prio_normal transport, which a messenger worker or, without a running worker,
     * Contao's web worker at the end of the request processes. Rebuilds the vector right away if the transport is not
     * available.
     *
     * @throws \InvalidArgumentException if the table has no search vector field
     */
    public function queue(string $table, int $id): void
    {
        // Fail early instead of in the worker
        $this->configurationResolver->getConfiguration($table);

        if ($id < 1) {
            return;
        }

        if (!$this->messageBus || !$this->messengerTransports?->has(self::TRANSPORT)) {
            $this->index($table, $id);

            return;
        }

        // A stamp instead of routing configuration, as the Contao routing interfaces are deprecated since Contao 5.6
        // and the #[AsMessage] attribute needs Symfony 7.2
        $this->messageBus->dispatch(new RebuildSearchVectorMessage($table, $id), [new TransportNamesStamp([self::TRANSPORT])]);
    }

    /**
     * Rebuilds the search vector of a single record.
     *
     * The record is read from the database, so this can be called from any save path after the record was written.
     * The record is locked while its vector is rebuilt, so a concurrent rebuild of the same record waits and then
     * reads the newer data instead of being overwritten with an outdated vector.
     *
     * @param DataContainer|null $dc The data container passed to options callbacks. If omitted, a minimal one
     *                               holding table, id and record is used.
     *
     * @return bool Whether the stored value changed
     */
    public function index(string $table, int $id, ?DataContainer $dc = null): bool
    {
        $column = $this->configurationResolver->getConfiguration($table)->getFieldName();

        if ($id < 1) {
            return false;
        }

        return $this->connection->transactional(function () use ($table, $id, $column, $dc): bool {
            $platform = $this->connection->getDatabasePlatform();

            $query = $this->connection->createQueryBuilder()
                ->select('*')
                ->from($platform->quoteSingleIdentifier($table))
                ->where('id = :id');
            $sql = $query->getSQL();

            try {
                $sql = $query->forUpdate()->getSQL();
            } catch (DbalException) {
                // No row locks, e.g. SQLite, which locks the whole database for writes anyway
            }

            $row = $this->connection->fetchAssociative($sql, ['id' => $id]);

            if (!$row) {
                return false;
            }

            $currentVector = $row[$column] ?? null;

            // Values of virtual fields (Contao 5.7+) are stored in a JSON column, not under their field name
            if ($this->virtualFieldsHandler) {
                $row = $this->virtualFieldsHandler->expandFields($row, $table);
            }

            $vector = $this->buildVector($table, $row, $dc);

            if ($currentVector === $vector) {
                return false;
            }

            $this->connection->update(
                $platform->quoteSingleIdentifier($table),
                [$platform->quoteSingleIdentifier($column) => $vector],
                ['id' => $id]
            );

            return true;
        });
    }

    /**
     * Rebuilds the search vector of every record of a table.
     *
     * @param (callable(int $id, bool $changed): void)|null $onProgress
     *
     * @return int The number of records whose vector changed
     */
    public function indexAll(string $table, ?callable $onProgress = null): int
    {
        $this->configurationResolver->getConfiguration($table);

        $ids = $this->connection->fetchFirstColumn(
            \sprintf('SELECT id FROM %s ORDER BY id', $this->connection->getDatabasePlatform()->quoteSingleIdentifier($table))
        );

        $changed = 0;

        foreach ($ids as $id) {
            $didChange = $this->index($table, (int) $id);

            if ($didChange) {
                ++$changed;
            }

            if ($onProgress) {
                $onProgress((int) $id, $didChange);
            }
        }

        return $changed;
    }

    /**
     * Builds the search vector from a raw database row.
     *
     * @param array<string, mixed> $row
     */
    public function buildVector(string $table, array $row, ?DataContainer $dc = null): string
    {
        $configuration = $this->configurationResolver->getConfiguration($table);
        $stemmer = $this->configurationResolver->getStemmer($configuration);
        $normalizer = $this->configurationResolver->getNormalizer($configuration);
        $locale = $this->configurationResolver->getRecordLocale($configuration, $row);
        $dc ??= new SearchVectorDataContainer($table, $row);

        $values = $this->languageSwitcher->run($locale, $table, fn (): array => $this->formatValues($table, $row, $configuration->getFields(), $dc));

        $event = $this->eventDispatcher->dispatch(new BuildSearchVectorEvent($table, $row, $values, $locale));

        $parts = [];

        foreach ($event->getValues() as $value) {
            if ($tokens = Tokenizer::tokenize($value, $locale, $stemmer, $normalizer)) {
                $parts[] = \implode(' ', $tokens);
            }
        }

        if (!$parts = \array_values(\array_unique($parts))) {
            return '';
        }

        return '| ' . \implode(self::VALUE_SEPARATOR, $parts) . ' |';
    }

    /**
     * Resolves the field values like the backend does: option labels, foreign keys, plain text instead of HTML.
     *
     * @param array<string, mixed> $row
     * @param list<string>         $fields
     *
     * @return list<string>
     */
    private function formatValues(string $table, array $row, array $fields, DataContainer $dc): array
    {
        $values = [];

        foreach ($fields as $field) {
            $raw = $row[$field] ?? null;

            if (null === $raw || '' === $raw) {
                continue;
            }

            $raw = \is_scalar($raw) ? (string) $raw : $raw;

            try {
                $formatted = $this->formatterUtil->formatDcaFieldValue(
                    $dc,
                    $field,
                    $raw,
                    FormatDcaFieldValueOptions::create()
                        ->setLoadDca(false)
                        ->setReplaceInsertTags(false)
                        ->setArrayJoiner(self::ARRAY_JOINER)
                );
            } catch (\Throwable $e) {
                $this->logger->warning(
                    \sprintf('Could not resolve field "%s.%s" for the search vector, indexing the raw value: %s', $table, $field, $e->getMessage()),
                    ['exception' => $e]
                );

                $formatted = \is_array($raw) ? \implode(self::ARRAY_JOINER, $raw) : $raw;
            }

            foreach (\explode(self::ARRAY_JOINER, $this->toPlainText((string) $formatted)) as $value) {
                $values[] = $value;
            }
        }

        return $values;
    }

    private function toPlainText(string $value): string
    {
        $value = StringUtil::decodeEntities(StringUtil::restoreBasicEntities($value));

        // Remove insert tags, nested ones from the inside out
        do {
            $value = (string) \preg_replace('/\{\{[^{}]*}}/', ' ', $value, -1, $count);
        } while ($count > 0);

        // Replace every tag with a space, so "<p>Wort.</p><p>Wort</p>" does not collapse into one word
        return (string) \preg_replace('/<[^>]*+>/', ' ', $value);
    }
}

1	<?php
2
3	namespace HeimrichHannot\UtilsBundle\SearchVector;
4
5	use Contao\CoreBundle\DataContainer\VirtualFieldsHandler;
6	use Contao\DataContainer;
7	use Contao\StringUtil;
8	use Doctrine\DBAL\Connection;
9	use Doctrine\DBAL\Exception as DbalException;
10	use HeimrichHannot\UtilsBundle\Event\BuildSearchVectorEvent;
11	use HeimrichHannot\UtilsBundle\SearchVector\Messenger\RebuildSearchVectorMessage;
12	use HeimrichHannot\UtilsBundle\Util\FormatterUtil;
13	use HeimrichHannot\UtilsBundle\Util\FormatterUtil\FormatDcaFieldValueOptions;
14	use Psr\Container\ContainerInterface;
15	use Psr\Log\LoggerInterface;
16	use Symfony\Component\Messenger\MessageBusInterface;
17	use Symfony\Component\Messenger\Stamp\TransportNamesStamp;
18	use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;
19
20	/**
21	* Builds and stores the search vector of records whose table registered a SearchVectorField.
22	*
23	* Format: the tokens of one value are joined by a space, values are separated by ` \| ` and the whole string is
24	* wrapped in `\| … \|`, e.g. `\| fuhrung durch die altstadt \| sommer \| stadtrundgange \|` (without stemmer). The space
25	* keeps word adjacency for quoted phrase searches, the pipe stops a phrase from matching across two unrelated values.
26	*
27	* Option labels are resolved and words are stemmed in the language of the record (see
28	* SearchVectorFieldConfiguration::setLanguage() and setLanguageField()), never in the language of the current user.
29	*/
30	class SearchVectorIndexer
31	{
32	/**
33	* The messenger transport of asynchronous rebuilds, configured by the Contao Managed Edition.
34	*/
35	public const TRANSPORT = 'contao_prio_normal';
36
37	private const VALUE_SEPARATOR = ' \| ';
38
39	/**
40	* Joins array values in the formatter output, so each array value stays a separate value of the vector.
41	*/
42	private const ARRAY_JOINER = "\x1F";
43
44	public function __construct(
45	private readonly Connection $connection,
46	private readonly SearchVectorConfigurationResolver $configurationResolver,
47	private readonly FormatterUtil $formatterUtil,
48	private readonly EventDispatcherInterface $eventDispatcher,
49	private readonly LoggerInterface $logger,
50	private readonly SearchVectorLanguageSwitcher $languageSwitcher,
51	private readonly ?MessageBusInterface $messageBus = null,
52	private readonly ?ContainerInterface $messengerTransports = null,
53	private readonly ?VirtualFieldsHandler $virtualFieldsHandler = null,
54	) {}	18 ✔
55
56	/**
57	* Rebuilds the search vector of a single record asynchronously.
58	*
59	* Sends a message to the contao_prio_normal transport, which a messenger worker or, without a running worker,
60	* Contao's web worker at the end of the request processes. Rebuilds the vector right away if the transport is not
61	* available.
62	*
63	* @throws \InvalidArgumentException if the table has no search vector field
64	*/
65	public function queue(string $table, int $id): void
66	{
67	// Fail early instead of in the worker
68	$this->configurationResolver->getConfiguration($table);	3 ✔
69
70	if ($id < 1) {	2 ✔
NEW 71	return;	×
72	}
73
74	if (!$this->messageBus \|\| !$this->messengerTransports?->has(self::TRANSPORT)) {	2 ✔
75	$this->index($table, $id);	1 ✔
76
77	return;	1 ✔
78	}
79
80	// A stamp instead of routing configuration, as the Contao routing interfaces are deprecated since Contao 5.6
81	// and the #[AsMessage] attribute needs Symfony 7.2
82	$this->messageBus->dispatch(new RebuildSearchVectorMessage($table, $id), [new TransportNamesStamp([self::TRANSPORT])]);	1 ✔
83	}
84
85	/**
86	* Rebuilds the search vector of a single record.
87	*
88	* The record is read from the database, so this can be called from any save path after the record was written.
89	* The record is locked while its vector is rebuilt, so a concurrent rebuild of the same record waits and then
90	* reads the newer data instead of being overwritten with an outdated vector.
91	*
92	* @param DataContainer\|null $dc The data container passed to options callbacks. If omitted, a minimal one
93	* holding table, id and record is used.
94	*
95	* @return bool Whether the stored value changed
96	*/
97	public function index(string $table, int $id, ?DataContainer $dc = null): bool
98	{
99	$column = $this->configurationResolver->getConfiguration($table)->getFieldName();	6 ✔
100
101	if ($id < 1) {	5 ✔
NEW 102	return false;	×
103	}
104
105	return $this->connection->transactional(function () use ($table, $id, $column, $dc): bool {	5 ✔
106	$platform = $this->connection->getDatabasePlatform();	5 ✔
107
108	$query = $this->connection->createQueryBuilder()	5 ✔
109	->select('*')	5 ✔
110	->from($platform->quoteSingleIdentifier($table))	5 ✔
111	->where('id = :id');	5 ✔
112	$sql = $query->getSQL();	5 ✔
113
114	try {
115	$sql = $query->forUpdate()->getSQL();	5 ✔
116	} catch (DbalException) {	5 ✔
117	// No row locks, e.g. SQLite, which locks the whole database for writes anyway
118	}
119
120	$row = $this->connection->fetchAssociative($sql, ['id' => $id]);	5 ✔
121
122	if (!$row) {	5 ✔
123	return false;	1 ✔
124	}
125
126	$currentVector = $row[$column] ?? null;	5 ✔
127
128	// Values of virtual fields (Contao 5.7+) are stored in a JSON column, not under their field name
129	if ($this->virtualFieldsHandler) {	5 ✔
NEW 130	$row = $this->virtualFieldsHandler->expandFields($row, $table);	×
131	}
132
133	$vector = $this->buildVector($table, $row, $dc);	5 ✔
134
135	if ($currentVector === $vector) {	5 ✔
136	return false;	2 ✔
137	}
138
139	$this->connection->update(	5 ✔
140	$platform->quoteSingleIdentifier($table),	5 ✔
141	[$platform->quoteSingleIdentifier($column) => $vector],	5 ✔
142	['id' => $id]	5 ✔
143	);	5 ✔
144
145	return true;	5 ✔
146	});	5 ✔
147	}
148
149	/**
150	* Rebuilds the search vector of every record of a table.
151	*
152	* @param (callable(int $id, bool $changed): void)\|null $onProgress
153	*
154	* @return int The number of records whose vector changed
155	*/
156	public function indexAll(string $table, ?callable $onProgress = null): int
157	{
158	$this->configurationResolver->getConfiguration($table);	1 ✔
159
160	$ids = $this->connection->fetchFirstColumn(	1 ✔
161	\sprintf('SELECT id FROM %s ORDER BY id', $this->connection->getDatabasePlatform()->quoteSingleIdentifier($table))	1 ✔
162	);	1 ✔
163
164	$changed = 0;	1 ✔
165
166	foreach ($ids as $id) {	1 ✔
167	$didChange = $this->index($table, (int) $id);	1 ✔
168
169	if ($didChange) {	1 ✔
170	++$changed;	1 ✔
171	}
172
173	if ($onProgress) {	1 ✔
174	$onProgress((int) $id, $didChange);	1 ✔
175	}
176	}
177
178	return $changed;	1 ✔
179	}
180
181	/**
182	* Builds the search vector from a raw database row.
183	*
184	* @param array<string, mixed> $row
185	*/
186	public function buildVector(string $table, array $row, ?DataContainer $dc = null): string
187	{
188	$configuration = $this->configurationResolver->getConfiguration($table);	16 ✔
189	$stemmer = $this->configurationResolver->getStemmer($configuration);	16 ✔
190	$normalizer = $this->configurationResolver->getNormalizer($configuration);	16 ✔
191	$locale = $this->configurationResolver->getRecordLocale($configuration, $row);	16 ✔
192	$dc ??= new SearchVectorDataContainer($table, $row);	16 ✔
193
194	$values = $this->languageSwitcher->run($locale, $table, fn (): array => $this->formatValues($table, $row, $configuration->getFields(), $dc));	16 ✔
195
196	$event = $this->eventDispatcher->dispatch(new BuildSearchVectorEvent($table, $row, $values, $locale));	16 ✔
197
198	$parts = [];	16 ✔
199
200	foreach ($event->getValues() as $value) {	16 ✔
201	if ($tokens = Tokenizer::tokenize($value, $locale, $stemmer, $normalizer)) {	16 ✔
202	$parts[] = \implode(' ', $tokens);	16 ✔
203	}
204	}
205
206	if (!$parts = \array_values(\array_unique($parts))) {	16 ✔
207	return '';	1 ✔
208	}
209
210	return '\| ' . \implode(self::VALUE_SEPARATOR, $parts) . ' \|';	16 ✔
211	}
212
213	/**
214	* Resolves the field values like the backend does: option labels, foreign keys, plain text instead of HTML.
215	*
216	* @param array<string, mixed> $row
217	* @param list<string> $fields
218	*
219	* @return list<string>
220	*/
221	private function formatValues(string $table, array $row, array $fields, DataContainer $dc): array
222	{
223	$values = [];	16 ✔
224
225	foreach ($fields as $field) {	16 ✔
226	$raw = $row[$field] ?? null;	16 ✔
227
228	if (null === $raw \|\| '' === $raw) {	16 ✔
229	continue;	16 ✔
230	}
231
232	$raw = \is_scalar($raw) ? (string) $raw : $raw;	16 ✔
233
234	try {
235	$formatted = $this->formatterUtil->formatDcaFieldValue(	16 ✔
236	$dc,	16 ✔
237	$field,	16 ✔
238	$raw,	16 ✔
239	FormatDcaFieldValueOptions::create()	16 ✔
240	->setLoadDca(false)	16 ✔
241	->setReplaceInsertTags(false)	16 ✔
242	->setArrayJoiner(self::ARRAY_JOINER)	16 ✔
243	);	16 ✔
244	} catch (\Throwable $e) {	1 ✔
245	$this->logger->warning(	1 ✔
246	\sprintf('Could not resolve field "%s.%s" for the search vector, indexing the raw value: %s', $table, $field, $e->getMessage()),	1 ✔
247	['exception' => $e]	1 ✔
248	);	1 ✔
249
250	$formatted = \is_array($raw) ? \implode(self::ARRAY_JOINER, $raw) : $raw;	1 ✔
251	}
252
253	foreach (\explode(self::ARRAY_JOINER, $this->toPlainText((string) $formatted)) as $value) {	16 ✔
254	$values[] = $value;	16 ✔
255	}
256	}
257
258	return $values;	16 ✔
259	}
260
261	private function toPlainText(string $value): string
262	{
263	$value = StringUtil::decodeEntities(StringUtil::restoreBasicEntities($value));	16 ✔
264
265	// Remove insert tags, nested ones from the inside out
266	do {
267	$value = (string) \preg_replace('/\{\{[^{}]*}}/', ' ', $value, -1, $count);	16 ✔
268	} while ($count > 0);	16 ✔
269
270	// Replace every tag with a space, so "<p>Wort.</p><p>Wort</p>" does not collapse into one word
271	return (string) \preg_replace('/<[^>]*+>/', ' ', $value);	16 ✔
272	}
273	}

heimrichhannot / contao-utils-bundle / 38542847508

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous