27827943989

Committed 19 Jun 2026 01:15PM UTC coverage: 21.03% (+4.8%) from 16.198%

Build # 27827943989

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #673 from Yoast/bugfix/linking-suggestions

[BUGFIX] Updated LinkingSuggestionsService to sort correctly and base batchSize of group sizes

Coverage Stats

6 of 10 new or added lines in 1 file covered. (60.0%)

576 of 2739 relevant lines covered (21.03%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.33

/Classes/Service/LinkingSuggestionsService.php

<?php

/**
 * This file is part of the "yoast_seo" extension for TYPO3 CMS.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 */

declare(strict_types=1);

namespace YoastSeoForTypo3\YoastSeo\Service;

use TYPO3\CMS\Backend\Utility\BackendUtility;
use TYPO3\CMS\Core\Context\LanguageAspect;
use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Domain\Repository\PageRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use YoastSeoForTypo3\YoastSeo\Constants\TableNames;
use YoastSeoForTypo3\YoastSeo\Traits\LanguageServiceTrait;

class LinkingSuggestionsService
{
    use LanguageServiceTrait;

    protected int $excludePageId;
    protected int $site;
    protected int $languageId;

    /**
     * @var array<string, int>
     */
    protected array $documentFrequencyCache = [];

    public function __construct(
        protected ConnectionPool $connectionPool,
        protected PageRepository $pageRepository,
        protected SiteService $siteService,
        protected int $batchSize = 100,
    ) {}

    /**
     * @param array<array{occurrences: int, stem: string}> $words
     * @return array<array<string, mixed>>
     */
    public function getLinkingSuggestions(
        array $words,
        int $excludePageId,
        int $languageId,
        string $content
    ): array {
        if ($words === []) {
            return [];
        }
        $this->excludePageId = $excludePageId;
        $this->site = $this->siteService->getSiteRootPageId($excludePageId);
        $this->languageId = $languageId;
        $this->documentFrequencyCache = [];

        $scores = $this->collectScores(array_column($words, 'occurrences', 'stem'));

        // Return the empty list if no suggestions have been found.
        if ($scores === []) {
            return [];
        }

        return $this->linkRecords($scores, $this->getCurrentContentLinks($content));
    }

    /**
     * Score every candidate record that shares prominent word stems with the request, paging
     * through the prominent word table in batches of {@see self::$batchSize} record groups.
     *
     * @param array<string, int|string> $words stem => occurrences
     * @return array<string, float|int> record key (uid_foreign-tablenames) => normalized score
     */
    protected function collectScores(array $words): array
    {
        // Combine stems, weights and DFs from request
        $requestData = $this->composeRequestData($words);

        // Calculate vector length of the request set (needed for score normalization later)
        $requestVectorLength = $this->computeVectorLength($requestData);

        $requestStems = array_keys($requestData);
        $scores = [];
        $page = 1;

        do {
            // Retrieve the (pid, tablenames) record groups for this batch that share stems with the request
            $recordGroups = $this->findRecordsByStems($requestStems, $this->batchSize, $page);

            // Expand the groups into their prominent words and index them by record
            $candidatesWords = $this->findStemsByRecords($recordGroups);
            $candidatesWordsByRecord = $this->groupWordsByRecord($candidatesWords);

            foreach ($candidatesWordsByRecord as $id => $candidateData) {
                $scores[$id] = $this->calculateScoreForIndexable($requestData, $requestVectorLength, $candidateData);
            }

            // Sort the list of scores and keep only the top of the scores
            $scores = $this->getTopSuggestions($scores);

            ++$page;
            // Keep paging while the batch was full. The loop must count the paged unit
            // (record groups), not the scored records, because a single group can expand
            // into several records and would otherwise stop paging too early.
        } while (count($recordGroups) === $this->batchSize);

        return $scores;
    }

    /**
     * @param array<string, int|string> $requestWords
     * @return array<string, array{weight: int, df: int}>
     */
    protected function composeRequestData(array $requestWords): array
    {
        $requestDocFrequencies = $this->countDocumentFrequencies(array_keys($requestWords));
        $combinedRequestData = [];
        foreach ($requestWords as $stem => $weight) {
            if (!isset($requestDocFrequencies[$stem])) {
                continue;
            }

            $combinedRequestData[$stem] = [
                'weight' => (int)$weight,
                'df' => $requestDocFrequencies[$stem],
            ];
        }
        return $combinedRequestData;
    }

    /**
     * @param string[] $stems
     * @return array<string, int>
     */
    protected function countDocumentFrequencies(array $stems): array
    {
        if ($stems === []) {
            return [];
        }

        $uncachedStems = array_values(array_filter(
            $stems,
            fn(string $stem): bool => !isset($this->documentFrequencyCache[$stem])
        ));

        if ($uncachedStems !== []) {
            $queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);
            $rawDocFrequencies = $queryBuilder->select('stem')->addSelectLiteral('COUNT(stem) AS document_frequency')->from(
                TableNames::PROMINENT_WORD
            )->where(
                $queryBuilder->expr()->in(
                    'stem',
                    $queryBuilder->createNamedParameter($uncachedStems, Connection::PARAM_STR_ARRAY)
                ),
                $queryBuilder->expr()->eq('sys_language_uid', $this->languageId),
                $queryBuilder->expr()->eq('site', $this->site)
            )->groupBy('stem')->executeQuery()->fetchAllAssociative();

            foreach ($rawDocFrequencies as $rawDocFrequency) {
                $this->documentFrequencyCache[(string)$rawDocFrequency['stem']] = (int)$rawDocFrequency['document_frequency'];
            }
        }

        $docFrequencies = [];
        foreach ($stems as $stem) {
            if (isset($this->documentFrequencyCache[$stem])) {
                $docFrequencies[$stem] = $this->documentFrequencyCache[$stem];
            }
        }
        return $docFrequencies;
    }

    /**
     * @param array<string, array{weight: int, df?: int}> $prominentWords
     */
    protected function computeVectorLength(array $prominentWords): float
    {
        $sumOfSquares = 0;
        foreach ($prominentWords as $word) {
            $docFrequency = 1;
            if (array_key_exists('df', $word)) {
                $docFrequency = $word['df'];
            }

            $tfIdf = $this->computeTfIdfScore($word['weight'], $docFrequency);
            $sumOfSquares += ($tfIdf ** 2);
        }
        return sqrt($sumOfSquares);
    }

    protected function computeTfIdfScore(int $termFrequency, int $docFrequency): float
    {
        $docFrequency = max(1, $docFrequency);
        return $termFrequency * (1 / $docFrequency);
    }

    /**
     * @param array<array{pid: int, tablenames: string}> $records
     * @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}>
     */
    protected function findStemsByRecords(array $records): array
    {
        if ($records === []) {
            return [];
        }

        $prominentWords = $this->getProminentWords($records);
        $prominentStems = array_unique(array_column($prominentWords, 'stem'));

        $stemCounts = $this->countDocumentFrequencies($prominentStems);

        foreach ($prominentWords as &$prominentWord) {
            if (!isset($stemCounts[$prominentWord['stem']])) {
                continue;
            }
            $prominentWord['df'] = $stemCounts[$prominentWord['stem']];
        }
        return $prominentWords;
    }

    /**
     * @param string[] $stems
     * @return array<array{pid: int, tablenames: string}>
     */
    protected function findRecordsByStems(array $stems, int $batchSize, int $page): array
    {
        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);
        $queryBuilder->select('pid', 'tablenames')->from(TableNames::PROMINENT_WORD)->where(
            $queryBuilder->expr()->in(
                'stem',
                $queryBuilder->createNamedParameter($stems, Connection::PARAM_STR_ARRAY)
            ),
            $queryBuilder->expr()->eq('sys_language_uid', $this->languageId),
            $queryBuilder->expr()->eq('site', $this->site)
        )->groupBy('pid', 'tablenames')
            ->orderBy('pid')
            ->addOrderBy('tablenames')
            ->setMaxResults($batchSize)
            ->setFirstResult(($page - 1) * $batchSize);
        /** @var array<array{pid: int, tablenames: string}> $records */
        $records = $queryBuilder->executeQuery()->fetchAllAssociative();
        return $records;
    }

    /**
     * @param array<array{pid: int, tablenames: string}> $records
     * @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}>
     */
    protected function getProminentWords(array $records): array
    {
        // Group pids by tablename to use efficient IN() clauses instead of OR chains
        $pidsByTable = [];
        foreach ($records as $record) {
            $pidsByTable[$record['tablenames']][] = (int)$record['pid'];
        }

        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);
        $tableConditions = [];
        foreach ($pidsByTable as $tablename => $pids) {
            $tableConditions[] = $queryBuilder->expr()->and(
                $queryBuilder->expr()->eq('tablenames', $queryBuilder->createNamedParameter($tablename)),
                $queryBuilder->expr()->in('pid', $queryBuilder->createNamedParameter($pids, Connection::PARAM_INT_ARRAY))
            );
        }

        $queryBuilder->select('stem', 'weight', 'pid', 'tablenames', 'uid_foreign')->from(TableNames::PROMINENT_WORD)
            ->where(
                $queryBuilder->expr()->eq('sys_language_uid', $this->languageId),
                $queryBuilder->expr()->or(...$tableConditions)
            );
        /** @var array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}> $prominentWords */
        $prominentWords = $queryBuilder->executeQuery()->fetchAllAssociative();
        return $prominentWords;
    }

    /**
     * @param array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}> $candidateWords
     * @return array<string, array<string, array{weight: int, df: int}>>
     */
    protected function groupWordsByRecord(array $candidateWords): array
    {
        $candidateWordsByRecords = [];
        foreach ($candidateWords as $candidateWord) {
            if (!isset($candidateWord['df'])) {
                continue;
            }
            $recordKey = $candidateWord['uid_foreign'] . '-' . $candidateWord['tablenames'];
            $candidateWordsByRecords[$recordKey][$candidateWord['stem']] = [
                'weight' => (int)$candidateWord['weight'],
                'df' => (int)$candidateWord['df'],
            ];
        }
        return $candidateWordsByRecords;
    }

    /**
     * @param array<string, array{weight: int, df: int}> $requestData
     * @param array<string, array{weight: int, df: int}> $candidateData
     */
    protected function calculateScoreForIndexable(
        array $requestData,
        float $requestVectorLength,
        array $candidateData
    ): float {
        $rawScore = $this->computeRawScore($requestData, $candidateData);
        $candidateVectorLength = $this->computeVectorLength($candidateData);
        return $this->normalizeScore($rawScore, $candidateVectorLength, $requestVectorLength);
    }

    /**
     * @param array<string, array{weight: int, df: int}> $requestData
     * @param array<string, array{weight: int, df: int}> $candidateData
     */
    protected function computeRawScore(array $requestData, array $candidateData): float
    {
        $rawScore = 0;
        foreach ($candidateData as $stem => $candidateWordData) {
            if (!array_key_exists($stem, $requestData)) {
                continue;
            }

            $wordFromRequestWeight = $requestData[$stem]['weight'];
            $wordFromRequestDf = $requestData[$stem]['df'];
            $candidateWeight = $candidateWordData['weight'];
            $canidateDf = $candidateWordData['df'];

            $tfIdfFromRequest = $this->computeTfIdfScore($wordFromRequestWeight, $wordFromRequestDf);
            $tfIdfFromDatabase = $this->computeTfIdfScore($candidateWeight, $canidateDf);

            $rawScore += ($tfIdfFromRequest * $tfIdfFromDatabase);
        }
        return (float)$rawScore;
    }

    protected function normalizeScore(float $rawScore, float $vectorLengthCandidate, float $vectorLengthRequest): float
    {
        $normalizingFactor = $vectorLengthRequest * $vectorLengthCandidate;
        if ($normalizingFactor === 0.0) {
            // We can't divide by 0, so set the score to 0 instead.
            return 0;
        }
        return (float)($rawScore / $normalizingFactor);
    }

    /**
     * @param array<string, float|int> $scores
     * @return array<string, float|int>
     */
    protected function getTopSuggestions(array $scores): array
    {
        // Sort the indexables by descending score.
        uasort(
            $scores,
            static function ($score1, $score2) {
                if ($score1 === $score2) {
                    return 0;
                }
                return ($score1 < $score2) ? 1 : -1;
            }
        );

        // Take the top $limit suggestions, while preserving their ids specified in the keys of the array elements.
        return \array_slice($scores, 0, 20, true);
    }

    /**
     * @param array<string, float|int> $scores
     * @param array<string, bool> $currentLinks
     * @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
     */
    protected function linkRecords(array $scores, array $currentLinks): array
    {
        $links = [];
        foreach ($scores as $record => $score) {
            [$uid, $table] = explode('-', $record);
            if ($table === TableNames::PAGES && (int)$uid === $this->excludePageId) {
                continue;
            }

            $data = BackendUtility::getRecord($table, $uid);
            if ($data === null) {
                continue;
            }
            if ($this->languageId > 0 && ($overlay = $this->getRecordOverlay($table, $data, $this->languageId))) {
                $data = $overlay;
            }

            $labelField = $GLOBALS['TCA'][$table]['ctrl']['label'];

            $links[$record] = [
                'label' => $data[$labelField],
                'recordType' => $this->getRecordType($table),
                'id' => (int)$uid,
                'table' => $table,
                'cornerstone' => (int)($data['tx_yoastseo_cornerstone'] ?? 0),
                'score' => $score,
                'active' => isset($currentLinks[$record]),
            ];
        }
        $this->sortSuggestions($links);

        $cornerStoneSuggestions = $this->filterSuggestions($links, true);
        $nonCornerStoneSuggestions = $this->filterSuggestions($links, false);

        return array_merge_recursive([], $cornerStoneSuggestions, $nonCornerStoneSuggestions);
    }

    /**
     * @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
     */
    protected function sortSuggestions(array &$links): void
    {
        uasort(
            $links,
            static function ($suggestion1, $suggestion2) {
                if ($suggestion1['score'] === $suggestion2['score']) {
                    return 0;
                }

                return ($suggestion1['score'] < $suggestion2['score']) ? 1 : -1;
            }
        );
    }

    /**
     * @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
     * @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
     */
    protected function filterSuggestions(array $links, bool $cornerstone): array
    {
        return \array_filter(
            $links,
            static function ($suggestion) use ($cornerstone) {
                return (bool)$suggestion['cornerstone'] === $cornerstone;
            }
        );
    }

    /**
     * @return array<string, bool>
     */
    protected function getCurrentContentLinks(string $content): array
    {
        $currentLinks = [];
        preg_match_all('/<a href="t3:\/\/(.*)\?uid=([\d]+)/', $content, $matches, PREG_SET_ORDER);
        foreach ($matches as $match) {
            $key = (int)$match[2] . '-' . str_replace('page', TableNames::PAGES, $match[1]);
            $currentLinks[$key] = true;
        }
        return $currentLinks;
    }

    protected function getRecordType(string $table): string
    {
        return $this->getLanguageService()->sL(
            $GLOBALS['TCA'][$table]['ctrl']['title']
        );
    }

    /**
     * @param array<string, mixed> $data
     * @return array<string, mixed>|null
     */
    protected function getRecordOverlay(string $table, array $data, int $languageId): array|null
    {
        $languageAspect = GeneralUtility::makeInstance(LanguageAspect::class, $languageId, $languageId, 'mixed');
        return $this->pageRepository->getLanguageOverlay($table, $data, $languageAspect);
    }
}

1	<?php
2
3	/**
4	* This file is part of the "yoast_seo" extension for TYPO3 CMS.
5	*
6	* For the full copyright and license information, please read the
7	* LICENSE.txt file that was distributed with this source code.
8	*/
9
10	declare(strict_types=1);
11
12	namespace YoastSeoForTypo3\YoastSeo\Service;
13
14	use TYPO3\CMS\Backend\Utility\BackendUtility;
15	use TYPO3\CMS\Core\Context\LanguageAspect;
16	use TYPO3\CMS\Core\Database\Connection;
17	use TYPO3\CMS\Core\Database\ConnectionPool;
18	use TYPO3\CMS\Core\Domain\Repository\PageRepository;
19	use TYPO3\CMS\Core\Utility\GeneralUtility;
20	use YoastSeoForTypo3\YoastSeo\Constants\TableNames;
21	use YoastSeoForTypo3\YoastSeo\Traits\LanguageServiceTrait;
22
23	class LinkingSuggestionsService
24	{
25	use LanguageServiceTrait;
26
27	protected int $excludePageId;
28	protected int $site;
29	protected int $languageId;
30
31	/**
32	* @var array<string, int>
33	*/
34	protected array $documentFrequencyCache = [];
35
36	public function __construct(
37	protected ConnectionPool $connectionPool,
38	protected PageRepository $pageRepository,
39	protected SiteService $siteService,
40	protected int $batchSize = 100,
41	) {}	1✔
42
43	/**
44	* @param array<array{occurrences: int, stem: string}> $words
45	* @return array<array<string, mixed>>
46	*/
47	public function getLinkingSuggestions(
48	array $words,
49	int $excludePageId,
50	int $languageId,
51	string $content
52	): array {
53	if ($words === []) {	×
54	return [];	×
55	}
56	$this->excludePageId = $excludePageId;	×
57	$this->site = $this->siteService->getSiteRootPageId($excludePageId);	×
58	$this->languageId = $languageId;	×
59	$this->documentFrequencyCache = [];	×
60
NEW 61	$scores = $this->collectScores(array_column($words, 'occurrences', 'stem'));	×
62
63	// Return the empty list if no suggestions have been found.
NEW 64	if ($scores === []) {	×
NEW 65	return [];	×
66	}
67
NEW 68	return $this->linkRecords($scores, $this->getCurrentContentLinks($content));	×
69	}
70
71	/**
72	* Score every candidate record that shares prominent word stems with the request, paging
73	* through the prominent word table in batches of {@see self::$batchSize} record groups.
74	*
75	* @param array<string, int\|string> $words stem => occurrences
76	* @return array<string, float\|int> record key (uid_foreign-tablenames) => normalized score
77	*/
78	protected function collectScores(array $words): array
79	{
80	// Combine stems, weights and DFs from request
81	$requestData = $this->composeRequestData($words);	1✔
82
83	// Calculate vector length of the request set (needed for score normalization later)
84	$requestVectorLength = $this->computeVectorLength($requestData);	1✔
85
86	$requestStems = array_keys($requestData);	1✔
87	$scores = [];	1✔
88	$page = 1;	1✔
89
90	do {
91	// Retrieve the (pid, tablenames) record groups for this batch that share stems with the request
92	$recordGroups = $this->findRecordsByStems($requestStems, $this->batchSize, $page);	1✔
93
94	// Expand the groups into their prominent words and index them by record
95	$candidatesWords = $this->findStemsByRecords($recordGroups);	1✔
96	$candidatesWordsByRecord = $this->groupWordsByRecord($candidatesWords);	1✔
97
98	foreach ($candidatesWordsByRecord as $id => $candidateData) {	1✔
99	$scores[$id] = $this->calculateScoreForIndexable($requestData, $requestVectorLength, $candidateData);	1✔
100	}
101
102	// Sort the list of scores and keep only the top of the scores
103	$scores = $this->getTopSuggestions($scores);	1✔
104
105	++$page;	1✔
106	// Keep paging while the batch was full. The loop must count the paged unit
107	// (record groups), not the scored records, because a single group can expand
108	// into several records and would otherwise stop paging too early.
109	} while (count($recordGroups) === $this->batchSize);	1✔
110
111	return $scores;	1✔
112	}
113
114	/**
115	* @param array<string, int\|string> $requestWords
116	* @return array<string, array{weight: int, df: int}>
117	*/
118	protected function composeRequestData(array $requestWords): array
119	{
120	$requestDocFrequencies = $this->countDocumentFrequencies(array_keys($requestWords));	1✔
121	$combinedRequestData = [];	1✔
122	foreach ($requestWords as $stem => $weight) {	1✔
123	if (!isset($requestDocFrequencies[$stem])) {	1✔
124	continue;	×
125	}
126
127	$combinedRequestData[$stem] = [	1✔
128	'weight' => (int)$weight,	1✔
129	'df' => $requestDocFrequencies[$stem],	1✔
130	];	1✔
131	}
132	return $combinedRequestData;	1✔
133	}
134
135	/**
136	* @param string[] $stems
137	* @return array<string, int>
138	*/
139	protected function countDocumentFrequencies(array $stems): array
140	{
141	if ($stems === []) {	1✔
142	return [];	×
143	}
144
145	$uncachedStems = array_values(array_filter(	1✔
146	$stems,	1✔
147	fn(string $stem): bool => !isset($this->documentFrequencyCache[$stem])	1✔
148	));	1✔
149
150	if ($uncachedStems !== []) {	1✔
151	$queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);	1✔
152	$rawDocFrequencies = $queryBuilder->select('stem')->addSelectLiteral('COUNT(stem) AS document_frequency')->from(	1✔
153	TableNames::PROMINENT_WORD	1✔
154	)->where(	1✔
155	$queryBuilder->expr()->in(	1✔
156	'stem',	1✔
157	$queryBuilder->createNamedParameter($uncachedStems, Connection::PARAM_STR_ARRAY)	1✔
158	),	1✔
159	$queryBuilder->expr()->eq('sys_language_uid', $this->languageId),	1✔
160	$queryBuilder->expr()->eq('site', $this->site)	1✔
161	)->groupBy('stem')->executeQuery()->fetchAllAssociative();	1✔
162
163	foreach ($rawDocFrequencies as $rawDocFrequency) {	1✔
164	$this->documentFrequencyCache[(string)$rawDocFrequency['stem']] = (int)$rawDocFrequency['document_frequency'];	1✔
165	}
166	}
167
168	$docFrequencies = [];	1✔
169	foreach ($stems as $stem) {	1✔
170	if (isset($this->documentFrequencyCache[$stem])) {	1✔
171	$docFrequencies[$stem] = $this->documentFrequencyCache[$stem];	1✔
172	}
173	}
174	return $docFrequencies;	1✔
175	}
176
177	/**
178	* @param array<string, array{weight: int, df?: int}> $prominentWords
179	*/
180	protected function computeVectorLength(array $prominentWords): float
181	{
182	$sumOfSquares = 0;	1✔
183	foreach ($prominentWords as $word) {	1✔
184	$docFrequency = 1;	1✔
185	if (array_key_exists('df', $word)) {	1✔
186	$docFrequency = $word['df'];	1✔
187	}
188
189	$tfIdf = $this->computeTfIdfScore($word['weight'], $docFrequency);	1✔
190	$sumOfSquares += ($tfIdf ** 2);	1✔
191	}
192	return sqrt($sumOfSquares);	1✔
193	}
194
195	protected function computeTfIdfScore(int $termFrequency, int $docFrequency): float
196	{
197	$docFrequency = max(1, $docFrequency);	1✔
198	return $termFrequency * (1 / $docFrequency);	1✔
199	}
200
201	/**
202	* @param array<array{pid: int, tablenames: string}> $records
203	* @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}>
204	*/
205	protected function findStemsByRecords(array $records): array
206	{
207	if ($records === []) {	1✔
208	return [];	×
209	}
210
211	$prominentWords = $this->getProminentWords($records);	1✔
212	$prominentStems = array_unique(array_column($prominentWords, 'stem'));	1✔
213
214	$stemCounts = $this->countDocumentFrequencies($prominentStems);	1✔
215
216	foreach ($prominentWords as &$prominentWord) {	1✔
217	if (!isset($stemCounts[$prominentWord['stem']])) {	1✔
218	continue;	×
219	}
220	$prominentWord['df'] = $stemCounts[$prominentWord['stem']];	1✔
221	}
222	return $prominentWords;	1✔
223	}
224
225	/**
226	* @param string[] $stems
227	* @return array<array{pid: int, tablenames: string}>
228	*/
229	protected function findRecordsByStems(array $stems, int $batchSize, int $page): array
230	{
231	$queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);	1✔
232	$queryBuilder->select('pid', 'tablenames')->from(TableNames::PROMINENT_WORD)->where(	1✔
233	$queryBuilder->expr()->in(	1✔
234	'stem',	1✔
235	$queryBuilder->createNamedParameter($stems, Connection::PARAM_STR_ARRAY)	1✔
236	),	1✔
237	$queryBuilder->expr()->eq('sys_language_uid', $this->languageId),	1✔
238	$queryBuilder->expr()->eq('site', $this->site)	1✔
239	)->groupBy('pid', 'tablenames')	1✔
240	->orderBy('pid')	1✔
241	->addOrderBy('tablenames')	1✔
242	->setMaxResults($batchSize)	1✔
243	->setFirstResult(($page - 1) * $batchSize);	1✔
244	/** @var array<array{pid: int, tablenames: string}> $records */
245	$records = $queryBuilder->executeQuery()->fetchAllAssociative();	1✔
246	return $records;	1✔
247	}
248
249	/**
250	* @param array<array{pid: int, tablenames: string}> $records
251	* @return array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}>
252	*/
253	protected function getProminentWords(array $records): array
254	{
255	// Group pids by tablename to use efficient IN() clauses instead of OR chains
256	$pidsByTable = [];	1✔
257	foreach ($records as $record) {	1✔
258	$pidsByTable[$record['tablenames']][] = (int)$record['pid'];	1✔
259	}
260
261	$queryBuilder = $this->connectionPool->getQueryBuilderForTable(TableNames::PROMINENT_WORD);	1✔
262	$tableConditions = [];	1✔
263	foreach ($pidsByTable as $tablename => $pids) {	1✔
264	$tableConditions[] = $queryBuilder->expr()->and(	1✔
265	$queryBuilder->expr()->eq('tablenames', $queryBuilder->createNamedParameter($tablename)),	1✔
266	$queryBuilder->expr()->in('pid', $queryBuilder->createNamedParameter($pids, Connection::PARAM_INT_ARRAY))	1✔
267	);	1✔
268	}
269
270	$queryBuilder->select('stem', 'weight', 'pid', 'tablenames', 'uid_foreign')->from(TableNames::PROMINENT_WORD)	1✔
271	->where(	1✔
272	$queryBuilder->expr()->eq('sys_language_uid', $this->languageId),	1✔
273	$queryBuilder->expr()->or(...$tableConditions)	1✔
274	);	1✔
275	/** @var array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int}> $prominentWords */
276	$prominentWords = $queryBuilder->executeQuery()->fetchAllAssociative();	1✔
277	return $prominentWords;	1✔
278	}
279
280	/**
281	* @param array<array{stem: string, weight: int, pid: int, tablenames: string, uid_foreign: int, df?: int}> $candidateWords
282	* @return array<string, array<string, array{weight: int, df: int}>>
283	*/
284	protected function groupWordsByRecord(array $candidateWords): array
285	{
286	$candidateWordsByRecords = [];	1✔
287	foreach ($candidateWords as $candidateWord) {	1✔
288	if (!isset($candidateWord['df'])) {	1✔
289	continue;	×
290	}
291	$recordKey = $candidateWord['uid_foreign'] . '-' . $candidateWord['tablenames'];	1✔
292	$candidateWordsByRecords[$recordKey][$candidateWord['stem']] = [	1✔
293	'weight' => (int)$candidateWord['weight'],	1✔
294	'df' => (int)$candidateWord['df'],	1✔
295	];	1✔
296	}
297	return $candidateWordsByRecords;	1✔
298	}
299
300	/**
301	* @param array<string, array{weight: int, df: int}> $requestData
302	* @param array<string, array{weight: int, df: int}> $candidateData
303	*/
304	protected function calculateScoreForIndexable(
305	array $requestData,
306	float $requestVectorLength,
307	array $candidateData
308	): float {
309	$rawScore = $this->computeRawScore($requestData, $candidateData);	1✔
310	$candidateVectorLength = $this->computeVectorLength($candidateData);	1✔
311	return $this->normalizeScore($rawScore, $candidateVectorLength, $requestVectorLength);	1✔
312	}
313
314	/**
315	* @param array<string, array{weight: int, df: int}> $requestData
316	* @param array<string, array{weight: int, df: int}> $candidateData
317	*/
318	protected function computeRawScore(array $requestData, array $candidateData): float
319	{
320	$rawScore = 0;	1✔
321	foreach ($candidateData as $stem => $candidateWordData) {	1✔
322	if (!array_key_exists($stem, $requestData)) {	1✔
323	continue;	×
324	}
325
326	$wordFromRequestWeight = $requestData[$stem]['weight'];	1✔
327	$wordFromRequestDf = $requestData[$stem]['df'];	1✔
328	$candidateWeight = $candidateWordData['weight'];	1✔
329	$canidateDf = $candidateWordData['df'];	1✔
330
331	$tfIdfFromRequest = $this->computeTfIdfScore($wordFromRequestWeight, $wordFromRequestDf);	1✔
332	$tfIdfFromDatabase = $this->computeTfIdfScore($candidateWeight, $canidateDf);	1✔
333
334	$rawScore += ($tfIdfFromRequest * $tfIdfFromDatabase);	1✔
335	}
336	return (float)$rawScore;	1✔
337	}
338
339	protected function normalizeScore(float $rawScore, float $vectorLengthCandidate, float $vectorLengthRequest): float
340	{
341	$normalizingFactor = $vectorLengthRequest * $vectorLengthCandidate;	1✔
342	if ($normalizingFactor === 0.0) {	1✔
343	// We can't divide by 0, so set the score to 0 instead.
344	return 0;	×
345	}
346	return (float)($rawScore / $normalizingFactor);	1✔
347	}
348
349	/**
350	* @param array<string, float\|int> $scores
351	* @return array<string, float\|int>
352	*/
353	protected function getTopSuggestions(array $scores): array
354	{
355	// Sort the indexables by descending score.
356	uasort(	1✔
357	$scores,	1✔
358	static function ($score1, $score2) {	1✔
359	if ($score1 === $score2) {	1✔
360	return 0;	1✔
361	}
362	return ($score1 < $score2) ? 1 : -1;	×
363	}	1✔
364	);	1✔
365
366	// Take the top $limit suggestions, while preserving their ids specified in the keys of the array elements.
367	return \array_slice($scores, 0, 20, true);	1✔
368	}
369
370	/**
371	* @param array<string, float\|int> $scores
372	* @param array<string, bool> $currentLinks
373	* @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
374	*/
375	protected function linkRecords(array $scores, array $currentLinks): array
376	{
377	$links = [];	×
378	foreach ($scores as $record => $score) {	×
379	[$uid, $table] = explode('-', $record);	×
380	if ($table === TableNames::PAGES && (int)$uid === $this->excludePageId) {	×
381	continue;	×
382	}
383
384	$data = BackendUtility::getRecord($table, $uid);	×
385	if ($data === null) {	×
386	continue;	×
387	}
388	if ($this->languageId > 0 && ($overlay = $this->getRecordOverlay($table, $data, $this->languageId))) {	×
389	$data = $overlay;	×
390	}
391
392	$labelField = $GLOBALS['TCA'][$table]['ctrl']['label'];	×
393
394	$links[$record] = [	×
395	'label' => $data[$labelField],	×
396	'recordType' => $this->getRecordType($table),	×
397	'id' => (int)$uid,	×
398	'table' => $table,	×
399	'cornerstone' => (int)($data['tx_yoastseo_cornerstone'] ?? 0),	×
400	'score' => $score,	×
401	'active' => isset($currentLinks[$record]),	×
402	];	×
403	}
404	$this->sortSuggestions($links);	×
405
406	$cornerStoneSuggestions = $this->filterSuggestions($links, true);	×
407	$nonCornerStoneSuggestions = $this->filterSuggestions($links, false);	×
408
409	return array_merge_recursive([], $cornerStoneSuggestions, $nonCornerStoneSuggestions);	×
410	}
411
412	/**
413	* @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
414	*/
415	protected function sortSuggestions(array &$links): void
416	{
417	uasort(	×
418	$links,	×
419	static function ($suggestion1, $suggestion2) {	×
420	if ($suggestion1['score'] === $suggestion2['score']) {	×
421	return 0;	×
422	}
423
424	return ($suggestion1['score'] < $suggestion2['score']) ? 1 : -1;	×
425	}	×
426	);	×
427	}
428
429	/**
430	* @param array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}> $links
431	* @return array<string, array{label: string, recordType: string, id: int, table: string, cornerstone: int, score: float, active: bool}>
432	*/
433	protected function filterSuggestions(array $links, bool $cornerstone): array
434	{
435	return \array_filter(	×
436	$links,	×
437	static function ($suggestion) use ($cornerstone) {	×
438	return (bool)$suggestion['cornerstone'] === $cornerstone;	×
439	}	×
440	);	×
441	}
442
443	/**
444	* @return array<string, bool>
445	*/
446	protected function getCurrentContentLinks(string $content): array
447	{
448	$currentLinks = [];	×
449	preg_match_all('/<a href="t3:\/\/(.*)\?uid=([\d]+)/', $content, $matches, PREG_SET_ORDER);	×
450	foreach ($matches as $match) {	×
451	$key = (int)$match[2] . '-' . str_replace('page', TableNames::PAGES, $match[1]);	×
452	$currentLinks[$key] = true;	×
453	}
454	return $currentLinks;	×
455	}
456
457	protected function getRecordType(string $table): string
458	{
459	return $this->getLanguageService()->sL(	×
460	$GLOBALS['TCA'][$table]['ctrl']['title']	×
461	);	×
462	}
463
464	/**
465	* @param array<string, mixed> $data
466	* @return array<string, mixed>\|null
467	*/
468	protected function getRecordOverlay(string $table, array $data, int $languageId): array\|null
469	{
470	$languageAspect = GeneralUtility::makeInstance(LanguageAspect::class, $languageId, $languageId, 'mixed');	×
471	return $this->pageRepository->getLanguageOverlay($table, $data, $languageAspect);	×
472	}
473	}

Yoast / Yoast-SEO-for-TYPO3 / 27827943989

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous