28931069285

Committed 08 Jul 2026 09:07AM UTC coverage: 20.893% (-0.2%) from 21.137%

Build # 28931069285

Build Type

Pull #675

github

Committed by

RinyVT

Commit Message

[BUGFIX] Optimize prominent words and linking suggestions performance, prevent prominent words to be copied when pages are copied

Pull Request Pull Request #675: [BUGFIX] Optimize prominent words and linking suggestions performance, prevent prominent words to be copied when pages are copied

Coverage Stats

16 of 17 new or added lines in 2 files covered. (94.12%)

571 of 2733 relevant lines covered (20.89%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.24

/Classes/Service/LinkingSuggestionsService.php

<?php

/**
 * This file is part of the "yoast_seo" extension for TYPO3 CMS.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 */

declare(strict_types=1);

namespace YoastSeoForTypo3\YoastSeo\Service;

use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Context\LanguageAspect;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Domain\Repository\PageRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use YoastSeoForTypo3\YoastSeo\Constants\TableNames;
use YoastSeoForTypo3\YoastSeo\Traits\LanguageServiceTrait;

class LinkingSuggestionsService
{
    use LanguageServiceTrait;

    protected int $excludePageId;
    protected int $site;
    protected int $languageId;

    /**
     * Document frequency of every stem of the current site and language,
     * loaded once per scoring run by {@see self::loadDocumentFrequencies()}.
     *
     * @var array<string, int>
     */
    protected array $documentFrequencyCache = [];

    public function __construct(
        protected ConnectionPool $connectionPool,
        protected PageRepository $pageRepository,
        protected SiteService $siteService,
        protected int $batchSize = 100,
    ) {}

    /**
     * @param array<array{occurrences: int, stem: string}> $words
     * @return array<array<string, mixed>>
     */
    public function getLinkingSuggestions(
        array $words,
        int $excludePageId,
        int $languageId,
        string $content
    ): array {
        if ($words === []) {
            return [];
        }
        $this->excludePageId = $excludePageId;
        $this->site = $this->siteService->getSiteRootPageId($excludePageId);
        $this->languageId = $languageId;
        $this->documentFrequencyCache = [];

        $scores = $this->collectScores(array_column($words, 'occurrences', 'stem'));

        // Return the empty list if no suggestions have been found.
        if ($scores === []) {
            return [];
        }

        return $this->linkRecords($scores, $this->getCurrentContentLinks($content));
    }

    /**
     * Score every candidate record that shares prominent word stems with the request,
     * processing the candidates in chunks of {@see self::$batchSize} record groups.
     *
     * @param array<string, int|string> $words stem => occurrences
     * @return array<string, float|int> record key (uid_foreign-tablenames) => normalized score
     */
    protected function collectScores(array $words): array
    {
        $this->loadDocumentFrequencies();

        // Combine stems, weights and DFs from request
        $requestData = $this->composeRequestData($words);
        if ($requestData === []) {
            return [];
        }

        // Calculate vector length of the request set (needed for score normalization later)
        $requestVectorLength = $this->computeVectorLength($requestData);

        // Retrieve all (pid, tablenames) record groups that share stems with the request
        $recordGroups = $this->findRecordsByStems(array_keys($requestData));

        $scores = [];
        foreach (array_chunk($recordGroups, max(1, $this->batchSize)) as $recordGroupBatch) {
            // Expand the groups into their prominent words and index them by record
            $candidatesWords = $this->findStemsByRecords($recordGroupBatch);
            $candidatesWordsByRecord = $this->groupWordsByRecord($candidatesWords);

            foreach ($candidatesWordsByRecord as $id => $candidateData) {
                $scores[$id] = $this->calculateScoreForIndexable($requestData, $requestVectorLength, $candidateData);
            }

            // Sort the list of scores and keep only the top of the scores
            $scores = $this->getTopSuggestions($scores);
        }

        return $scores;
    }

    /**
     * @param array<string, int|string> $requestWords
     * @return array<string, array{weight: int, df: int}>
     */
    protected function composeRequestData(array $requestWords): array
    {
        $requestDocFrequencies = $this->countDocumentFrequencies(array_keys($requestWords));
        $combinedRequestData = [];
        foreach ($requestWords as $stem => $weight) {
            if (!isset($requestDocFrequencies[$stem])) {
                continue;
            }

            $combinedRequestData[$stem] = [
                'weight' => (int)$weight,
                'df' => $requestDocFrequencies[$stem],
            ];
        }
        return $combinedRequestData;
    }

    /**
     * Load all stems and their document frequencies for the current site and language in one call.
     * This used to be done per batch of candidate records, but that was very expensive on large sites with many stems.
     * Doing this in one pass is much cheaper, as it can be done with a single index-covered aggregation query.
     * Using fetchAssociative() to stream the rows instead of buffering them in memory, which saves a lot of memory on large sites.
     */
    protected function loadDocumentFrequencies(): void
    {
        $this->documentFrequencyCache = [];

        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);
        $result = $queryBuilder->select('stem')->addSelectLiteral('COUNT(stem) AS document_frequency')
            ->from(TableNames::PROMINENT_WORD)
            ->where(
                $queryBuilder->expr()->eq('sys_language_uid', $this->languageId),
                $queryBuilder->expr()->eq('site', $this->site)
            )->groupBy('stem')->executeQuery();

        while ($rawDocFrequency = $result->fetchAssociative()) {
            $this->documentFrequencyCache[(string)$rawDocFrequency['stem']] = (int)$rawDocFrequency['document_frequency'];
        }
    }

    /**
     * @param string[] $stems
     * @return array<string, int>
     */
    protected function countDocumentFrequencies(array $stems): array
    {
        $docFrequencies = [];
        foreach ($stems as $stem) {
            if (isset($this->documentFrequencyCache[$stem])) {
                $docFrequencies[$stem] = $this->documentFrequencyCache[$stem];
            }
        }
        return $docFrequencies;
    }

    /**
     * @param array<string, array{weight: int, df?: int}> $prominentWords
     */
    protected function computeVectorLength(array $prominentWords): float
    {
        $sumOfSquares = 0;
        foreach ($prominentWords as $word) {
            $docFrequency = 1;
            if (array_key_exists('df', $word)) {
                $docFrequency = $word['df'];
            }

            $tfIdf = $this->computeTfIdfScore($word['weight'], $docFrequency);
            $sumOfSquares += ($tfIdf ** 2);
        }
        return sqrt($sumOfSquares);
    }

    protected function computeTfIdfScore(int $termFrequency, int $docFrequency): float
    {
        $docFrequency = max(1, $docFrequency);
        return $termFrequency * (1 / $docFrequency);
    }

    /**
     * @param array<array{pid: int, tablenames: string}> $records
     * @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}>
     */
    protected function findStemsByRecords(array $records): array
    {
        if ($records === []) {
            return [];
        }

        $prominentWords = $this->getProminentWords($records);
        $prominentStems = array_unique(array_column($prominentWords, 'stem'));

        $stemCounts = $this->countDocumentFrequencies($prominentStems);

        foreach ($prominentWords as &$prominentWord) {
            if (!isset($stemCounts[$prominentWord['stem']])) {
                continue;
            }
            $prominentWord['df'] = $stemCounts[$prominentWord['stem']];
        }
        return $prominentWords;
    }

    /**
     * @param string[] $stems
     * @return array<array{pid: int, tablenames: string}>
     */
    protected function findRecordsByStems(array $stems): array
    {
        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);
        $queryBuilder->select('pid', 'tablenames')->from(TableNames::PROMINENT_WORD)->where(
            $queryBuilder->expr()->in(
                'stem',
                $queryBuilder->createNamedParameter($stems, Connection::PARAM_STR_ARRAY)
            ),
            $queryBuilder->expr()->eq('sys_language_uid', $this->languageId),
            $queryBuilder->expr()->eq('site', $this->site)
        )->groupBy('pid', 'tablenames')
            ->orderBy('pid')
            ->orderBy('tablenames');
        /** @var array<array{pid: int, tablenames: string}> $records */
        $records = $queryBuilder->executeQuery()->fetchAllAssociative();
        return $records;
    }

    /**
     * @param array<array{pid: int, tablenames: string}> $records
     * @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}>
     */
    protected function getProminentWords(array $records): array
    {
        // Group pids by tablename to use efficient IN() clauses instead of OR chains
        $pidsByTable = [];
        foreach ($records as $record) {
            $pidsByTable[$record['tablenames']][] = (int)$record['pid'];
        }

        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);
        $tableConditions = [];
        foreach ($pidsByTable as $tablename => $pids) {
            $tableConditions[] = $queryBuilder->expr()->and(
                $queryBuilder->expr()->eq('tablenames', $queryBuilder->createNamedParameter($tablename)),
                $queryBuilder->expr()->in('pid', $queryBuilder->createNamedParameter($pids, Connection::PARAM_INT_ARRAY))
            );
        }

        $queryBuilder->select('stem', 'weight', 'pid', 'tablenames', 'uid_foreign')->from(TableNames::PROMINENT_WORD)
            ->where(
                $queryBuilder->expr()->eq('sys_language_uid', $this->languageId),
                $queryBuilder->expr()->or(...$tableConditions)
            );
        /** @var array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}> $prominentWords */
        $prominentWords = $queryBuilder->executeQuery()->fetchAllAssociative();
        return $prominentWords;
    }

    /**
     * @param array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}> $candidateWords
     * @return array<string, array<string, array{weight: int, df: int}>>
     */
    protected function groupWordsByRecord(array $candidateWords): array
    {
        $candidateWordsByRecords = [];
        foreach ($candidateWords as $candidateWord) {
            if (!isset($candidateWord['df'])) {
                continue;
            }
            $recordKey = $candidateWord['uid_foreign'] . '-' . $candidateWord['tablenames'];
            $candidateWordsByRecords[$recordKey][$candidateWord['stem']] = [
                'weight' => (int)$candidateWord['weight'],
                'df' => (int)$candidateWord['df'],
            ];
        }
        return $candidateWordsByRecords;
    }

    /**
     * @param array<string, array{weight: int, df: int}> $requestData
     * @param array<string, array{weight: int, df: int}> $candidateData
     */
    protected function calculateScoreForIndexable(
        array $requestData,
        float $requestVectorLength,
        array $candidateData
    ): float {
        $rawScore = $this->computeRawScore($requestData, $candidateData);
        $candidateVectorLength = $this->computeVectorLength($candidateData);
        return $this->normalizeScore($rawScore, $candidateVectorLength, $requestVectorLength);
    }

    /**
     * @param array<string, array{weight: int, df: int}> $requestData
     * @param array<string, array{weight: int, df: int}> $candidateData
     */
    protected function computeRawScore(array $requestData, array $candidateData): float
    {
        $rawScore = 0;
        foreach ($candidateData as $stem => $candidateWordData) {
            if (!array_key_exists($stem, $requestData)) {
                continue;
            }

            $wordFromRequestWeight = $requestData[$stem]['weight'];
            $wordFromRequestDf = $requestData[$stem]['df'];
            $candidateWeight = $candidateWordData['weight'];
            $canidateDf = $candidateWordData['df'];

            $tfIdfFromRequest = $this->computeTfIdfScore($wordFromRequestWeight, $wordFromRequestDf);
            $tfIdfFromDatabase = $this->computeTfIdfScore($candidateWeight, $canidateDf);

            $rawScore += ($tfIdfFromRequest * $tfIdfFromDatabase);
        }
        return (float)$rawScore;
    }

    protected function normalizeScore(float $rawScore, float $vectorLengthCandidate, float $vectorLengthRequest): float
    {
        $normalizingFactor = $vectorLengthRequest * $vectorLengthCandidate;
        if ($normalizingFactor === 0.0) {
            // We can't divide by 0, so set the score to 0 instead.
            return 0;
        }
        return (float)($rawScore / $normalizingFactor);
    }

    /**
     * @param array<string, float|int> $scores
     * @return array<string, float|int>
     */
    protected function getTopSuggestions(array $scores): array
    {
        // Sort the indexables by descending score.
        uasort(
            $scores,
            static function ($score1, $score2) {
                if ($score1 === $score2) {
                    return 0;
                }
                return ($score1 < $score2) ? 1 : -1;
            }
        );

        // Take the top $limit suggestions, while preserving their ids specified in the keys of the array elements.
        return \array_slice($scores, 0, 20, true);
    }

    /**
     * @param array<string, float|int> $scores
     * @param array<string, bool> $currentLinks
     * @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
     */
    protected function linkRecords(array $scores, array $currentLinks): array
    {
        $links = [];
        foreach ($scores as $record => $score) {
            [$uid, $table] = explode('-', $record);
            if ($table === TableNames::PAGES && (int)$uid === $this->excludePageId) {
                continue;
            }

            $data = BackendUtility::getRecord($table, $uid);
            if ($data === null) {
                continue;
            }
            if ($this->languageId > 0 && ($overlay = $this->getRecordOverlay($table, $data, $this->languageId))) {
                $data = $overlay;
            }

            $labelField = $GLOBALS['TCA'][$table]['ctrl']['label'];

            $links[$record] = [
                'label' => $data[$labelField],
                'recordType' => $this->getRecordType($table),
                'id' => (int)$uid,
                'table' => $table,
                'cornerstone' => (int)($data['tx_yoastseo_cornerstone'] ?? 0),
                'score' => $score,
                'active' => isset($currentLinks[$record]),
            ];
        }
        $this->sortSuggestions($links);

        $cornerStoneSuggestions = $this->filterSuggestions($links, true);
        $nonCornerStoneSuggestions = $this->filterSuggestions($links, false);

        return array_merge_recursive([], $cornerStoneSuggestions, $nonCornerStoneSuggestions);
    }

    /**
     * @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
     */
    protected function sortSuggestions(array &$links): void
    {
        uasort(
            $links,
            static function ($suggestion1, $suggestion2) {
                if ($suggestion1['score'] === $suggestion2['score']) {
                    return 0;
                }

                return ($suggestion1['score'] < $suggestion2['score']) ? 1 : -1;
            }
        );
    }

    /**
     * @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
     * @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
     */
    protected function filterSuggestions(array $links, bool $cornerstone): array
    {
        return \array_filter(
            $links,
            static function ($suggestion) use ($cornerstone) {
                return (bool)$suggestion['cornerstone'] === $cornerstone;
            }
        );
    }

    /**
     * @return array<string, bool>
     */
    protected function getCurrentContentLinks(string $content): array
    {
        $currentLinks = [];
        preg_match_all('/<a href="t3:\/\/(.*)\?uid=([\d]+)/', $content, $matches, PREG_SET_ORDER);
        foreach ($matches as $match) {
            $key = (int)$match[2] . '-' . str_replace('page', TableNames::PAGES, $match[1]);
            $currentLinks[$key] = true;
        }
        return $currentLinks;
    }

    protected function getRecordType(string $table): string
    {
        return $this->getLanguageService()->sL(
            $GLOBALS['TCA'][$table]['ctrl']['title']
        );
    }

    /**
     * @param array<string, mixed> $data
     * @return array<string, mixed>|null
     */
    protected function getRecordOverlay(string $table, array $data, int $languageId): array|null
    {
        $languageAspect = GeneralUtility::makeInstance(LanguageAspect::class, $languageId, $languageId, 'mixed');
        return $this->pageRepository->getLanguageOverlay($table, $data, $languageAspect);
    }
}

1	<?php
2
3	/**
4	* This file is part of the "yoast_seo" extension for TYPO3 CMS.
5	*
6	* For the full copyright and license information, please read the
7	* LICENSE.txt file that was distributed with this source code.
8	*/
9
10	declare(strict_types=1);
11
12	namespace YoastSeoForTypo3\YoastSeo\Service;
13
14	use TYPO3\CMS\Backend\Utility\BackendUtility;
15	use TYPO3\CMS\Core\Context\LanguageAspect;
16	use TYPO3\CMS\Core\Database\Connection;
17	use TYPO3\CMS\Core\Database\ConnectionPool;
18	use TYPO3\CMS\Core\Domain\Repository\PageRepository;
19	use TYPO3\CMS\Core\Utility\GeneralUtility;
20	use YoastSeoForTypo3\YoastSeo\Constants\TableNames;
21	use YoastSeoForTypo3\YoastSeo\Traits\LanguageServiceTrait;
22
23	class LinkingSuggestionsService
24	{
25	use LanguageServiceTrait;
26
27	protected int $excludePageId;
28	protected int $site;
29	protected int $languageId;
30
31	/**
32	* Document frequency of every stem of the current site and language,
33	* loaded once per scoring run by {@see self::loadDocumentFrequencies()}.
34	*
35	* @var array<string, int>
36	*/
37	protected array $documentFrequencyCache = [];
38
39	public function __construct(
40	protected ConnectionPool $connectionPool,
41	protected PageRepository $pageRepository,
42	protected SiteService $siteService,
43	protected int $batchSize = 100,
44	) {}	3✔
45
46	/**
47	* @param array<array{occurrences: int, stem: string}> $words
48	* @return array<array<string, mixed>>
49	*/
50	public function getLinkingSuggestions(
51	array $words,
52	int $excludePageId,
53	int $languageId,
54	string $content
55	): array {
56	if ($words === []) {	×
57	return [];	×
58	}
59	$this->excludePageId = $excludePageId;	×
60	$this->site = $this->siteService->getSiteRootPageId($excludePageId);	×
61	$this->languageId = $languageId;	×
62	$this->documentFrequencyCache = [];	×
63
64	$scores = $this->collectScores(array_column($words, 'occurrences', 'stem'));	×
65
66	// Return the empty list if no suggestions have been found.
67	if ($scores === []) {	×
68	return [];	×
69	}
70
71	return $this->linkRecords($scores, $this->getCurrentContentLinks($content));	×
72	}
73
74	/**
75	* Score every candidate record that shares prominent word stems with the request,
76	* processing the candidates in chunks of {@see self::$batchSize} record groups.
77	*
78	* @param array<string, int\|string> $words stem => occurrences
79	* @return array<string, float\|int> record key (uid_foreign-tablenames) => normalized score
80	*/
81	protected function collectScores(array $words): array
82	{
83	$this->loadDocumentFrequencies();	3✔
84
85	// Combine stems, weights and DFs from request
86	$requestData = $this->composeRequestData($words);	3✔
87	if ($requestData === []) {	3✔
NEW 88	return [];	×
89	}
90
91	// Calculate vector length of the request set (needed for score normalization later)
92	$requestVectorLength = $this->computeVectorLength($requestData);	3✔
93
94	// Retrieve all (pid, tablenames) record groups that share stems with the request
95	$recordGroups = $this->findRecordsByStems(array_keys($requestData));	3✔
96
97	$scores = [];	3✔
98	foreach (array_chunk($recordGroups, max(1, $this->batchSize)) as $recordGroupBatch) {	3✔
99	// Expand the groups into their prominent words and index them by record
100	$candidatesWords = $this->findStemsByRecords($recordGroupBatch);	3✔
101	$candidatesWordsByRecord = $this->groupWordsByRecord($candidatesWords);	3✔
102
103	foreach ($candidatesWordsByRecord as $id => $candidateData) {	3✔
104	$scores[$id] = $this->calculateScoreForIndexable($requestData, $requestVectorLength, $candidateData);	3✔
105	}
106
107	// Sort the list of scores and keep only the top of the scores
108	$scores = $this->getTopSuggestions($scores);	3✔
109	}
110
111	return $scores;	3✔
112	}
113
114	/**
115	* @param array<string, int\|string> $requestWords
116	* @return array<string, array{weight: int, df: int}>
117	*/
118	protected function composeRequestData(array $requestWords): array
119	{
120	$requestDocFrequencies = $this->countDocumentFrequencies(array_keys($requestWords));	3✔
121	$combinedRequestData = [];	3✔
122	foreach ($requestWords as $stem => $weight) {	3✔
123	if (!isset($requestDocFrequencies[$stem])) {	3✔
124	continue;	×
125	}
126
127	$combinedRequestData[$stem] = [	3✔
128	'weight' => (int)$weight,	3✔
129	'df' => $requestDocFrequencies[$stem],	3✔
130	];	3✔
131	}
132	return $combinedRequestData;	3✔
133	}
134
135	/**
136	* Load all stems and their document frequencies for the current site and language in one call.
137	* This used to be done per batch of candidate records, but that was very expensive on large sites with many stems.
138	* Doing this in one pass is much cheaper, as it can be done with a single index-covered aggregation query.
139	* Using fetchAssociative() to stream the rows instead of buffering them in memory, which saves a lot of memory on large sites.
140	*/
141	protected function loadDocumentFrequencies(): void
142	{
143	$this->documentFrequencyCache = [];	3✔
144
145	$queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);	3✔
146	$result = $queryBuilder->select('stem')->addSelectLiteral('COUNT(stem) AS document_frequency')	3✔
147	->from(TableNames::PROMINENT_WORD)	3✔
148	->where(	3✔
149	$queryBuilder->expr()->eq('sys_language_uid', $this->languageId),	3✔
150	$queryBuilder->expr()->eq('site', $this->site)	3✔
151	)->groupBy('stem')->executeQuery();	3✔
152
153	while ($rawDocFrequency = $result->fetchAssociative()) {	3✔
154	$this->documentFrequencyCache[(string)$rawDocFrequency['stem']] = (int)$rawDocFrequency['document_frequency'];	3✔
155	}
156	}
157
158	/**
159	* @param string[] $stems
160	* @return array<string, int>
161	*/
162	protected function countDocumentFrequencies(array $stems): array
163	{
164	$docFrequencies = [];	3✔
165	foreach ($stems as $stem) {	3✔
166	if (isset($this->documentFrequencyCache[$stem])) {	3✔
167	$docFrequencies[$stem] = $this->documentFrequencyCache[$stem];	3✔
168	}
169	}
170	return $docFrequencies;	3✔
171	}
172
173	/**
174	* @param array<string, array{weight: int, df?: int}> $prominentWords
175	*/
176	protected function computeVectorLength(array $prominentWords): float
177	{
178	$sumOfSquares = 0;	3✔
179	foreach ($prominentWords as $word) {	3✔
180	$docFrequency = 1;	3✔
181	if (array_key_exists('df', $word)) {	3✔
182	$docFrequency = $word['df'];	3✔
183	}
184
185	$tfIdf = $this->computeTfIdfScore($word['weight'], $docFrequency);	3✔
186	$sumOfSquares += ($tfIdf ** 2);	3✔
187	}
188	return sqrt($sumOfSquares);	3✔
189	}
190
191	protected function computeTfIdfScore(int $termFrequency, int $docFrequency): float
192	{
193	$docFrequency = max(1, $docFrequency);	3✔
194	return $termFrequency * (1 / $docFrequency);	3✔
195	}
196
197	/**
198	* @param array<array{pid: int, tablenames: string}> $records
199	* @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}>
200	*/
201	protected function findStemsByRecords(array $records): array
202	{
203	if ($records === []) {	3✔
204	return [];	×
205	}
206
207	$prominentWords = $this->getProminentWords($records);	3✔
208	$prominentStems = array_unique(array_column($prominentWords, 'stem'));	3✔
209
210	$stemCounts = $this->countDocumentFrequencies($prominentStems);	3✔
211
212	foreach ($prominentWords as &$prominentWord) {	3✔
213	if (!isset($stemCounts[$prominentWord['stem']])) {	3✔
214	continue;	×
215	}
216	$prominentWord['df'] = $stemCounts[$prominentWord['stem']];	3✔
217	}
218	return $prominentWords;	3✔
219	}
220
221	/**
222	* @param string[] $stems
223	* @return array<array{pid: int, tablenames: string}>
224	*/
225	protected function findRecordsByStems(array $stems): array
226	{
227	$queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);	3✔
228	$queryBuilder->select('pid', 'tablenames')->from(TableNames::PROMINENT_WORD)->where(	3✔
229	$queryBuilder->expr()->in(	3✔
230	'stem',	3✔
231	$queryBuilder->createNamedParameter($stems, Connection::PARAM_STR_ARRAY)	3✔
232	),	3✔
233	$queryBuilder->expr()->eq('sys_language_uid', $this->languageId),	3✔
234	$queryBuilder->expr()->eq('site', $this->site)	3✔
235	)->groupBy('pid', 'tablenames')	3✔
236	->orderBy('pid')	3✔
237	->orderBy('tablenames');	3✔
238	/** @var array<array{pid: int, tablenames: string}> $records */
239	$records = $queryBuilder->executeQuery()->fetchAllAssociative();	3✔
240	return $records;	3✔
241	}
242
243	/**
244	* @param array<array{pid: int, tablenames: string}> $records
245	* @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}>
246	*/
247	protected function getProminentWords(array $records): array
248	{
249	// Group pids by tablename to use efficient IN() clauses instead of OR chains
250	$pidsByTable = [];	3✔
251	foreach ($records as $record) {	3✔
252	$pidsByTable[$record['tablenames']][] = (int)$record['pid'];	3✔
253	}
254
255	$queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);	3✔
256	$tableConditions = [];	3✔
257	foreach ($pidsByTable as $tablename => $pids) {	3✔
258	$tableConditions[] = $queryBuilder->expr()->and(	3✔
259	$queryBuilder->expr()->eq('tablenames', $queryBuilder->createNamedParameter($tablename)),	3✔
260	$queryBuilder->expr()->in('pid', $queryBuilder->createNamedParameter($pids, Connection::PARAM_INT_ARRAY))	3✔
261	);	3✔
262	}
263
264	$queryBuilder->select('stem', 'weight', 'pid', 'tablenames', 'uid_foreign')->from(TableNames::PROMINENT_WORD)	3✔
265	->where(	3✔
266	$queryBuilder->expr()->eq('sys_language_uid', $this->languageId),	3✔
267	$queryBuilder->expr()->or(...$tableConditions)	3✔
268	);	3✔
269	/** @var array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}> $prominentWords */
270	$prominentWords = $queryBuilder->executeQuery()->fetchAllAssociative();	3✔
271	return $prominentWords;	3✔
272	}
273
274	/**
275	* @param array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}> $candidateWords
276	* @return array<string, array<string, array{weight: int, df: int}>>
277	*/
278	protected function groupWordsByRecord(array $candidateWords): array
279	{
280	$candidateWordsByRecords = [];	3✔
281	foreach ($candidateWords as $candidateWord) {	3✔
282	if (!isset($candidateWord['df'])) {	3✔
283	continue;	×
284	}
285	$recordKey = $candidateWord['uid_foreign'] . '-' . $candidateWord['tablenames'];	3✔
286	$candidateWordsByRecords[$recordKey][$candidateWord['stem']] = [	3✔
287	'weight' => (int)$candidateWord['weight'],	3✔
288	'df' => (int)$candidateWord['df'],	3✔
289	];	3✔
290	}
291	return $candidateWordsByRecords;	3✔
292	}
293
294	/**
295	* @param array<string, array{weight: int, df: int}> $requestData
296	* @param array<string, array{weight: int, df: int}> $candidateData
297	*/
298	protected function calculateScoreForIndexable(
299	array $requestData,
300	float $requestVectorLength,
301	array $candidateData
302	): float {
303	$rawScore = $this->computeRawScore($requestData, $candidateData);	3✔
304	$candidateVectorLength = $this->computeVectorLength($candidateData);	3✔
305	return $this->normalizeScore($rawScore, $candidateVectorLength, $requestVectorLength);	3✔
306	}
307
308	/**
309	* @param array<string, array{weight: int, df: int}> $requestData
310	* @param array<string, array{weight: int, df: int}> $candidateData
311	*/
312	protected function computeRawScore(array $requestData, array $candidateData): float
313	{
314	$rawScore = 0;	3✔
315	foreach ($candidateData as $stem => $candidateWordData) {	3✔
316	if (!array_key_exists($stem, $requestData)) {	3✔
317	continue;	2✔
318	}
319
320	$wordFromRequestWeight = $requestData[$stem]['weight'];	3✔
321	$wordFromRequestDf = $requestData[$stem]['df'];	3✔
322	$candidateWeight = $candidateWordData['weight'];	3✔
323	$canidateDf = $candidateWordData['df'];	3✔
324
325	$tfIdfFromRequest = $this->computeTfIdfScore($wordFromRequestWeight, $wordFromRequestDf);	3✔
326	$tfIdfFromDatabase = $this->computeTfIdfScore($candidateWeight, $canidateDf);	3✔
327
328	$rawScore += ($tfIdfFromRequest * $tfIdfFromDatabase);	3✔
329	}
330	return (float)$rawScore;	3✔
331	}
332
333	protected function normalizeScore(float $rawScore, float $vectorLengthCandidate, float $vectorLengthRequest): float
334	{
335	$normalizingFactor = $vectorLengthRequest * $vectorLengthCandidate;	3✔
336	if ($normalizingFactor === 0.0) {	3✔
337	// We can't divide by 0, so set the score to 0 instead.
338	return 0;	×
339	}
340	return (float)($rawScore / $normalizingFactor);	3✔
341	}
342
343	/**
344	* @param array<string, float\|int> $scores
345	* @return array<string, float\|int>
346	*/
347	protected function getTopSuggestions(array $scores): array
348	{
349	// Sort the indexables by descending score.
350	uasort(	3✔
351	$scores,	3✔
352	static function ($score1, $score2) {	3✔
353	if ($score1 === $score2) {	3✔
354	return 0;	1✔
355	}
356	return ($score1 < $score2) ? 1 : -1;	2✔
357	}	3✔
358	);	3✔
359
360	// Take the top $limit suggestions, while preserving their ids specified in the keys of the array elements.
361	return \array_slice($scores, 0, 20, true);	3✔
362	}
363
364	/**
365	* @param array<string, float\|int> $scores
366	* @param array<string, bool> $currentLinks
367	* @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
368	*/
369	protected function linkRecords(array $scores, array $currentLinks): array
370	{
371	$links = [];	×
372	foreach ($scores as $record => $score) {	×
373	[$uid, $table] = explode('-', $record);	×
374	if ($table === TableNames::PAGES && (int)$uid === $this->excludePageId) {	×
375	continue;	×
376	}
377
378	$data = BackendUtility::getRecord($table, $uid);	×
379	if ($data === null) {	×
380	continue;	×
381	}
382	if ($this->languageId > 0 && ($overlay = $this->getRecordOverlay($table, $data, $this->languageId))) {	×
383	$data = $overlay;	×
384	}
385
386	$labelField = $GLOBALS['TCA'][$table]['ctrl']['label'];	×
387
388	$links[$record] = [	×
389	'label' => $data[$labelField],	×
390	'recordType' => $this->getRecordType($table),	×
391	'id' => (int)$uid,	×
392	'table' => $table,	×
393	'cornerstone' => (int)($data['tx_yoastseo_cornerstone'] ?? 0),	×
394	'score' => $score,	×
395	'active' => isset($currentLinks[$record]),	×
396	];	×
397	}
398	$this->sortSuggestions($links);	×
399
400	$cornerStoneSuggestions = $this->filterSuggestions($links, true);	×
401	$nonCornerStoneSuggestions = $this->filterSuggestions($links, false);	×
402
403	return array_merge_recursive([], $cornerStoneSuggestions, $nonCornerStoneSuggestions);	×
404	}
405
406	/**
407	* @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
408	*/
409	protected function sortSuggestions(array &$links): void
410	{
411	uasort(	×
412	$links,	×
413	static function ($suggestion1, $suggestion2) {	×
414	if ($suggestion1['score'] === $suggestion2['score']) {	×
415	return 0;	×
416	}
417
418	return ($suggestion1['score'] < $suggestion2['score']) ? 1 : -1;	×
419	}	×
420	);	×
421	}
422
423	/**
424	* @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
425	* @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
426	*/
427	protected function filterSuggestions(array $links, bool $cornerstone): array
428	{
429	return \array_filter(	×
430	$links,	×
431	static function ($suggestion) use ($cornerstone) {	×
432	return (bool)$suggestion['cornerstone'] === $cornerstone;	×
433	}	×
434	);	×
435	}
436
437	/**
438	* @return array<string, bool>
439	*/
440	protected function getCurrentContentLinks(string $content): array
441	{
442	$currentLinks = [];	×
443	preg_match_all('/<a href="t3:\/\/(.*)\?uid=([\d]+)/', $content, $matches, PREG_SET_ORDER);	×
444	foreach ($matches as $match) {	×
445	$key = (int)$match[2] . '-' . str_replace('page', TableNames::PAGES, $match[1]);	×
446	$currentLinks[$key] = true;	×
447	}
448	return $currentLinks;	×
449	}
450
451	protected function getRecordType(string $table): string
452	{
453	return $this->getLanguageService()->sL(	×
454	$GLOBALS['TCA'][$table]['ctrl']['title']	×
455	);	×
456	}
457
458	/**
459	* @param array<string, mixed> $data
460	* @return array<string, mixed>\|null
461	*/
462	protected function getRecordOverlay(string $table, array $data, int $languageId): array\|null
463	{
464	$languageAspect = GeneralUtility::makeInstance(LanguageAspect::class, $languageId, $languageId, 'mixed');	×
465	return $this->pageRepository->getLanguageOverlay($table, $data, $languageAspect);	×
466	}
467	}

Yoast / Yoast-SEO-for-TYPO3 / 28931069285

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous