38531955264

Committed 14 Oct 2026 07:59AM UTC coverage: 83.985% (+4.2%) from 79.825%

Build # 38531955264

Build Type

Pull #108

github

Committed by

koertho

Commit Message

reject several search vector fields per table

A second registration for the same table silently replaced the first one.
Now loading the DCA throws an exception that names the conflicting fields.
Loading a DCA several times in one process, each time registering the field
once, is still allowed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Pull Request Pull Request #108: add search vector field

Coverage Stats

372 of 385 new or added lines in 16 files covered. (96.62%)

1568 of 1867 relevant lines covered (83.99%)

8.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.7

/src/SearchVector/SearchVectorIndexer.php

<?php

namespace HeimrichHannot\UtilsBundle\SearchVector;

use Contao\DataContainer;
use Contao\StringUtil;
use Doctrine\DBAL\Connection;
use HeimrichHannot\UtilsBundle\Event\BuildSearchVectorEvent;
use HeimrichHannot\UtilsBundle\SearchVector\Messenger\RebuildSearchVectorMessage;
use HeimrichHannot\UtilsBundle\Util\FormatterUtil;
use HeimrichHannot\UtilsBundle\Util\FormatterUtil\FormatDcaFieldValueOptions;
use Psr\Container\ContainerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\Messenger\MessageBusInterface;
use Symfony\Component\Messenger\Stamp\TransportNamesStamp;
use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;

/**
 * Builds and stores the search vector of records whose table registered a SearchVectorField.
 *
 * Format: the tokens of one value are joined by a space, values are separated by ` | ` and the whole string is
 * wrapped in `| … |`, e.g. `| fuhrung durch die altstadt | sommer | stadtrundgange |` (without stemmer). The space
 * keeps word adjacency for quoted phrase searches, the pipe stops a phrase from matching across two unrelated values.
 *
 * Option labels are resolved and words are stemmed in the language of the record (see
 * SearchVectorFieldConfiguration::setLanguage() and setLanguageField()), never in the language of the current user.
 */
class SearchVectorIndexer
{
    /**
     * The messenger transport of asynchronous rebuilds, configured by the Contao Managed Edition.
     */
    public const TRANSPORT = 'contao_prio_normal';

    private const VALUE_SEPARATOR = ' | ';

    /**
     * Joins array values in the formatter output, so each array value stays a separate value of the vector.
     */
    private const ARRAY_JOINER = "\x1F";

    public function __construct(
        private readonly Connection $connection,
        private readonly SearchVectorConfigurationResolver $configurationResolver,
        private readonly FormatterUtil $formatterUtil,
        private readonly EventDispatcherInterface $eventDispatcher,
        private readonly LoggerInterface $logger,
        private readonly SearchVectorLanguageSwitcher $languageSwitcher,
        private readonly ?MessageBusInterface $messageBus = null,
        private readonly ?ContainerInterface $messengerTransports = null,
    ) {}

    /**
     * Rebuilds the search vector of a single record asynchronously.
     *
     * Sends a message to the contao_prio_normal transport, which a messenger worker or, without a running worker,
     * Contao's web worker at the end of the request processes. Rebuilds the vector right away if the transport is not
     * available.
     *
     * @throws \InvalidArgumentException if the table has no search vector field
     */
    public function queue(string $table, int $id): void
    {
        // Fail early instead of in the worker
        $this->configurationResolver->getConfiguration($table);

        if ($id < 1) {
            return;
        }

        if (!$this->messageBus || !$this->messengerTransports?->has(self::TRANSPORT)) {
            $this->index($table, $id);

            return;
        }

        // A stamp instead of routing configuration, as the Contao routing interfaces are deprecated since Contao 5.6
        // and the #[AsMessage] attribute needs Symfony 7.2
        $this->messageBus->dispatch(new RebuildSearchVectorMessage($table, $id), [new TransportNamesStamp([self::TRANSPORT])]);
    }

    /**
     * Rebuilds the search vector of a single record.
     *
     * The record is read from the database, so this can be called from any save path after the record was written.
     *
     * @param DataContainer|null $dc The data container passed to options callbacks. If omitted, a minimal one
     *                               holding table, id and record is used.
     *
     * @return bool Whether the stored value changed
     */
    public function index(string $table, int $id, ?DataContainer $dc = null): bool
    {
        $column = $this->configurationResolver->getConfiguration($table)->getFieldName();

        if ($id < 1 || !$row = $this->connection->fetchAssociative(
            \sprintf('SELECT * FROM %s WHERE id = ?', $this->connection->quoteIdentifier($table)),
            [$id]
        )) {
            return false;
        }

        $vector = $this->buildVector($table, $row, $dc);

        if (($row[$column] ?? null) === $vector) {
            return false;
        }

        $this->connection->update(
            $this->connection->quoteIdentifier($table),
            [$this->connection->quoteIdentifier($column) => $vector],
            ['id' => $id]
        );

        return true;
    }

    /**
     * Rebuilds the search vector of every record of a table.
     *
     * @param (callable(int $id, bool $changed): void)|null $onProgress
     *
     * @return int The number of records whose vector changed
     */
    public function indexAll(string $table, ?callable $onProgress = null): int
    {
        $this->configurationResolver->getConfiguration($table);

        $ids = $this->connection->fetchFirstColumn(
            \sprintf('SELECT id FROM %s ORDER BY id', $this->connection->quoteIdentifier($table))
        );

        $changed = 0;

        foreach ($ids as $id) {
            $didChange = $this->index($table, (int) $id);

            if ($didChange) {
                ++$changed;
            }

            if ($onProgress) {
                $onProgress((int) $id, $didChange);
            }
        }

        return $changed;
    }

    /**
     * Builds the search vector from a raw database row.
     *
     * @param array<string, mixed> $row
     */
    public function buildVector(string $table, array $row, ?DataContainer $dc = null): string
    {
        $configuration = $this->configurationResolver->getConfiguration($table);
        $stemmer = $this->configurationResolver->getStemmer($configuration);
        $normalizer = $this->configurationResolver->getNormalizer($configuration);
        $locale = $this->configurationResolver->getRecordLocale($configuration, $row);
        $dc ??= new SearchVectorDataContainer($table, $row);

        $values = $this->languageSwitcher->run($locale, $table, fn (): array => $this->formatValues($table, $row, $configuration->getFields(), $dc));

        $event = $this->eventDispatcher->dispatch(new BuildSearchVectorEvent($table, $row, $values, $locale));

        $parts = [];

        foreach ($event->getValues() as $value) {
            if ($tokens = Tokenizer::tokenize($value, $locale, $stemmer, $normalizer)) {
                $parts[] = \implode(' ', $tokens);
            }
        }

        if (!$parts = \array_values(\array_unique($parts))) {
            return '';
        }

        return '| ' . \implode(self::VALUE_SEPARATOR, $parts) . ' |';
    }

    /**
     * Resolves the field values like the backend does: option labels, foreign keys, plain text instead of HTML.
     *
     * @param array<string, mixed> $row
     * @param list<string>         $fields
     *
     * @return list<string>
     */
    private function formatValues(string $table, array $row, array $fields, DataContainer $dc): array
    {
        $values = [];

        foreach ($fields as $field) {
            $raw = $row[$field] ?? null;

            if (null === $raw || '' === $raw) {
                continue;
            }

            $raw = \is_scalar($raw) ? (string) $raw : $raw;

            try {
                $formatted = $this->formatterUtil->formatDcaFieldValue(
                    $dc,
                    $field,
                    $raw,
                    FormatDcaFieldValueOptions::create()
                        ->setLoadDca(false)
                        ->setReplaceInsertTags(false)
                        ->setArrayJoiner(self::ARRAY_JOINER)
                );
            } catch (\Throwable $e) {
                $this->logger->warning(
                    \sprintf('Could not resolve field "%s.%s" for the search vector, indexing the raw value: %s', $table, $field, $e->getMessage()),
                    ['exception' => $e]
                );

                $formatted = \is_array($raw) ? \implode(self::ARRAY_JOINER, $raw) : $raw;
            }

            foreach (\explode(self::ARRAY_JOINER, $this->toPlainText((string) $formatted)) as $value) {
                $values[] = $value;
            }
        }

        return $values;
    }

    private function toPlainText(string $value): string
    {
        $value = StringUtil::decodeEntities(StringUtil::restoreBasicEntities($value));

        // Remove insert tags, nested ones from the inside out
        do {
            $value = (string) \preg_replace('/\{\{[^{}]*}}/', ' ', $value, -1, $count);
        } while ($count > 0);

        // Replace every tag with a space, so "<p>Wort.</p><p>Wort</p>" does not collapse into one word
        return (string) \preg_replace('/<[^>]*+>/', ' ', $value);
    }
}

1	<?php
2
3	namespace HeimrichHannot\UtilsBundle\SearchVector;
4
5	use Contao\DataContainer;
6	use Contao\StringUtil;
7	use Doctrine\DBAL\Connection;
8	use HeimrichHannot\UtilsBundle\Event\BuildSearchVectorEvent;
9	use HeimrichHannot\UtilsBundle\SearchVector\Messenger\RebuildSearchVectorMessage;
10	use HeimrichHannot\UtilsBundle\Util\FormatterUtil;
11	use HeimrichHannot\UtilsBundle\Util\FormatterUtil\FormatDcaFieldValueOptions;
12	use Psr\Container\ContainerInterface;
13	use Psr\Log\LoggerInterface;
14	use Symfony\Component\Messenger\MessageBusInterface;
15	use Symfony\Component\Messenger\Stamp\TransportNamesStamp;
16	use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;
17
18	/**
19	* Builds and stores the search vector of records whose table registered a SearchVectorField.
20	*
21	* Format: the tokens of one value are joined by a space, values are separated by ` \| ` and the whole string is
22	* wrapped in `\| … \|`, e.g. `\| fuhrung durch die altstadt \| sommer \| stadtrundgange \|` (without stemmer). The space
23	* keeps word adjacency for quoted phrase searches, the pipe stops a phrase from matching across two unrelated values.
24	*
25	* Option labels are resolved and words are stemmed in the language of the record (see
26	* SearchVectorFieldConfiguration::setLanguage() and setLanguageField()), never in the language of the current user.
27	*/
28	class SearchVectorIndexer
29	{
30	/**
31	* The messenger transport of asynchronous rebuilds, configured by the Contao Managed Edition.
32	*/
33	public const TRANSPORT = 'contao_prio_normal';
34
35	private const VALUE_SEPARATOR = ' \| ';
36
37	/**
38	* Joins array values in the formatter output, so each array value stays a separate value of the vector.
39	*/
40	private const ARRAY_JOINER = "\x1F";
41
42	public function __construct(
43	private readonly Connection $connection,
44	private readonly SearchVectorConfigurationResolver $configurationResolver,
45	private readonly FormatterUtil $formatterUtil,
46	private readonly EventDispatcherInterface $eventDispatcher,
47	private readonly LoggerInterface $logger,
48	private readonly SearchVectorLanguageSwitcher $languageSwitcher,
49	private readonly ?MessageBusInterface $messageBus = null,
50	private readonly ?ContainerInterface $messengerTransports = null,
51	) {}	17 ✔
52
53	/**
54	* Rebuilds the search vector of a single record asynchronously.
55	*
56	* Sends a message to the contao_prio_normal transport, which a messenger worker or, without a running worker,
57	* Contao's web worker at the end of the request processes. Rebuilds the vector right away if the transport is not
58	* available.
59	*
60	* @throws \InvalidArgumentException if the table has no search vector field
61	*/
62	public function queue(string $table, int $id): void
63	{
64	// Fail early instead of in the worker
65	$this->configurationResolver->getConfiguration($table);	3 ✔
66
67	if ($id < 1) {	2 ✔
NEW 68	return;	×
69	}
70
71	if (!$this->messageBus \|\| !$this->messengerTransports?->has(self::TRANSPORT)) {	2 ✔
72	$this->index($table, $id);	1 ✔
73
74	return;	1 ✔
75	}
76
77	// A stamp instead of routing configuration, as the Contao routing interfaces are deprecated since Contao 5.6
78	// and the #[AsMessage] attribute needs Symfony 7.2
79	$this->messageBus->dispatch(new RebuildSearchVectorMessage($table, $id), [new TransportNamesStamp([self::TRANSPORT])]);	1 ✔
80	}
81
82	/**
83	* Rebuilds the search vector of a single record.
84	*
85	* The record is read from the database, so this can be called from any save path after the record was written.
86	*
87	* @param DataContainer\|null $dc The data container passed to options callbacks. If omitted, a minimal one
88	* holding table, id and record is used.
89	*
90	* @return bool Whether the stored value changed
91	*/
92	public function index(string $table, int $id, ?DataContainer $dc = null): bool
93	{
94	$column = $this->configurationResolver->getConfiguration($table)->getFieldName();	5 ✔
95
96	if ($id < 1 \|\| !$row = $this->connection->fetchAssociative(	4 ✔
97	\sprintf('SELECT * FROM %s WHERE id = ?', $this->connection->quoteIdentifier($table)),	4 ✔
98	[$id]	4 ✔
99	)) {	4 ✔
100	return false;	1 ✔
101	}
102
103	$vector = $this->buildVector($table, $row, $dc);	4 ✔
104
105	if (($row[$column] ?? null) === $vector) {	4 ✔
106	return false;	2 ✔
107	}
108
109	$this->connection->update(	4 ✔
110	$this->connection->quoteIdentifier($table),	4 ✔
111	[$this->connection->quoteIdentifier($column) => $vector],	4 ✔
112	['id' => $id]	4 ✔
113	);	4 ✔
114
115	return true;	4 ✔
116	}
117
118	/**
119	* Rebuilds the search vector of every record of a table.
120	*
121	* @param (callable(int $id, bool $changed): void)\|null $onProgress
122	*
123	* @return int The number of records whose vector changed
124	*/
125	public function indexAll(string $table, ?callable $onProgress = null): int
126	{
127	$this->configurationResolver->getConfiguration($table);	1 ✔
128
129	$ids = $this->connection->fetchFirstColumn(	1 ✔
130	\sprintf('SELECT id FROM %s ORDER BY id', $this->connection->quoteIdentifier($table))	1 ✔
131	);	1 ✔
132
133	$changed = 0;	1 ✔
134
135	foreach ($ids as $id) {	1 ✔
136	$didChange = $this->index($table, (int) $id);	1 ✔
137
138	if ($didChange) {	1 ✔
139	++$changed;	1 ✔
140	}
141
142	if ($onProgress) {	1 ✔
143	$onProgress((int) $id, $didChange);	1 ✔
144	}
145	}
146
147	return $changed;	1 ✔
148	}
149
150	/**
151	* Builds the search vector from a raw database row.
152	*
153	* @param array<string, mixed> $row
154	*/
155	public function buildVector(string $table, array $row, ?DataContainer $dc = null): string
156	{
157	$configuration = $this->configurationResolver->getConfiguration($table);	15 ✔
158	$stemmer = $this->configurationResolver->getStemmer($configuration);	15 ✔
159	$normalizer = $this->configurationResolver->getNormalizer($configuration);	15 ✔
160	$locale = $this->configurationResolver->getRecordLocale($configuration, $row);	15 ✔
161	$dc ??= new SearchVectorDataContainer($table, $row);	15 ✔
162
163	$values = $this->languageSwitcher->run($locale, $table, fn (): array => $this->formatValues($table, $row, $configuration->getFields(), $dc));	15 ✔
164
165	$event = $this->eventDispatcher->dispatch(new BuildSearchVectorEvent($table, $row, $values, $locale));	15 ✔
166
167	$parts = [];	15 ✔
168
169	foreach ($event->getValues() as $value) {	15 ✔
170	if ($tokens = Tokenizer::tokenize($value, $locale, $stemmer, $normalizer)) {	15 ✔
171	$parts[] = \implode(' ', $tokens);	15 ✔
172	}
173	}
174
175	if (!$parts = \array_values(\array_unique($parts))) {	15 ✔
176	return '';	1 ✔
177	}
178
179	return '\| ' . \implode(self::VALUE_SEPARATOR, $parts) . ' \|';	15 ✔
180	}
181
182	/**
183	* Resolves the field values like the backend does: option labels, foreign keys, plain text instead of HTML.
184	*
185	* @param array<string, mixed> $row
186	* @param list<string> $fields
187	*
188	* @return list<string>
189	*/
190	private function formatValues(string $table, array $row, array $fields, DataContainer $dc): array
191	{
192	$values = [];	15 ✔
193
194	foreach ($fields as $field) {	15 ✔
195	$raw = $row[$field] ?? null;	15 ✔
196
197	if (null === $raw \|\| '' === $raw) {	15 ✔
198	continue;	15 ✔
199	}
200
201	$raw = \is_scalar($raw) ? (string) $raw : $raw;	15 ✔
202
203	try {
204	$formatted = $this->formatterUtil->formatDcaFieldValue(	15 ✔
205	$dc,	15 ✔
206	$field,	15 ✔
207	$raw,	15 ✔
208	FormatDcaFieldValueOptions::create()	15 ✔
209	->setLoadDca(false)	15 ✔
210	->setReplaceInsertTags(false)	15 ✔
211	->setArrayJoiner(self::ARRAY_JOINER)	15 ✔
212	);	15 ✔
213	} catch (\Throwable $e) {	1 ✔
214	$this->logger->warning(	1 ✔
215	\sprintf('Could not resolve field "%s.%s" for the search vector, indexing the raw value: %s', $table, $field, $e->getMessage()),	1 ✔
216	['exception' => $e]	1 ✔
217	);	1 ✔
218
219	$formatted = \is_array($raw) ? \implode(self::ARRAY_JOINER, $raw) : $raw;	1 ✔
220	}
221
222	foreach (\explode(self::ARRAY_JOINER, $this->toPlainText((string) $formatted)) as $value) {	15 ✔
223	$values[] = $value;	15 ✔
224	}
225	}
226
227	return $values;	15 ✔
228	}
229
230	private function toPlainText(string $value): string
231	{
232	$value = StringUtil::decodeEntities(StringUtil::restoreBasicEntities($value));	15 ✔
233
234	// Remove insert tags, nested ones from the inside out
235	do {
236	$value = (string) \preg_replace('/\{\{[^{}]*}}/', ' ', $value, -1, $count);	15 ✔
237	} while ($count > 0);	15 ✔
238
239	// Replace every tag with a space, so "<p>Wort.</p><p>Wort</p>" does not collapse into one word
240	return (string) \preg_replace('/<[^>]*+>/', ' ', $value);	15 ✔
241	}
242	}

heimrichhannot / contao-utils-bundle / 38531955264

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous