systemsdk / phpcpd / #23

Committed 18 May 2025 05:01PM UTC coverage: 75.818% (-0.7%) from 76.552%

Build # #23

Build Type

push

Committed by DKravtsov

Commit Message

phpcpd 8.2.2 release. Improved progress bar.

Coverage Stats

9 of 34 new or added lines in 3 files covered. (26.47%)

718 of 947 relevant lines covered (75.82%)

3.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.89

/src/Detector/Strategy/SuffixTreeStrategy.php

<?php

declare(strict_types=1);

namespace Systemsdk\PhpCPD\Detector\Strategy;

use Systemsdk\PhpCPD\CodeClone;
use Systemsdk\PhpCPD\CodeCloneFile;
use Systemsdk\PhpCPD\CodeCloneMap;
use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\AbstractToken;
use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\ApproximateCloneDetectingSuffixTree;
use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\Sentinel;
use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\Token;
use Systemsdk\PhpCPD\Detector\Traits\ProgressBarTrait;
use Systemsdk\PhpCPD\Exceptions\MissingResultException;

use function array_key_exists;
use function array_keys;
use function file_get_contents;
use function is_array;
use function token_get_all;

use const T_ATTRIBUTE;

/**
 * The suffix tree strategy was implemented in PHP for PHPCPD by Olle Härstedt.
 *
 * This PHP implementation is based on the Java implementation archived that is available at
 * https://www.cqse.eu/en/news/blog/conqat-end-of-life/ under the Apache License 2.0.
 *
 * The aforementioned Java implementation is based on the algorithm described in
 * https://dl.acm.org/doi/10.1109/ICSE.2009.5070547. This paper is available at
 * https://www.cqse.eu/fileadmin/content/news/publications/2009-do-code-clones-matter.pdf.
 */
final class SuffixTreeStrategy extends AbstractStrategy
{
    use ProgressBarTrait;

    private const string PROGRESS_BAR_SEARCH_CLONES_TITLE = 'Search for clones';
    private const string PROGRESS_BAR_PROCESS_CLONES_TITLE = 'Clones processing';
    private const string PROGRESS_BAR_POST_PROCESS_DONE_TITLE = 'Post process done';

    /**
     * @var array<int, AbstractToken>
     */
    private array $word = [];

    /**
     * @var array<string, int>
     */
    private array $fileTokens = [];

    private ?CodeCloneMap $result = null;

    public function processFile(string $file, CodeCloneMap $result): void
    {
        $content = (string)file_get_contents($file);
        $tokens = token_get_all($content);
        $lastTokenLine = 0;
        $attributeStarted = false;
        $attributeStartedLine = 0;
        $this->fileTokens[$file] = 0;

        $result->addToNumberOfLines(substr_count($content, "\n"));

        unset($content);

        foreach (array_keys($tokens) as $key) {
            /** @var array{0: int, 1:string, 2:int}|string $token */
            $token = $tokens[$key];

            if (is_array($token)) {
                if ($attributeStarted === false && !isset($this->tokensIgnoreList[$token[0]])) {
                    $this->word[] = new Token(
                        $token[0],
                        token_name($token[0]),
                        $token[2],
                        $file,
                        $token[1]
                    );
                    $this->fileTokens[$file]++;
                }

                if ($token[0] === T_ATTRIBUTE) {
                    $attributeStarted = true;
                    $attributeStartedLine = $token[2];
                }

                $lastTokenLine = $token[2];
            } elseif (
                $attributeStarted === true && $token === ']'
                && (
                    $attributeStartedLine === $lastTokenLine
                    || (array_key_exists($key - 1, $tokens) && $tokens[$key - 1] === ')')
                )
            ) {
                $attributeStarted = false;
                $attributeStartedLine = 0;
            }
        }

        $this->result = $result;
    }

    /**
     * @throws MissingResultException
     */
    public function postProcess(bool $useProgressBar): void
    {
        if (empty($this->result)) {
            throw new MissingResultException('Missing result');
        }

        $totalSteps = 2;

        if ($useProgressBar) {
            $this->progressBar(0, $totalSteps, self::PROGRESS_BAR_SEARCH_CLONES_TITLE);
        }

        // Sentinel = End of word
        $this->word[] = new Sentinel();

        $cloneInfos = (new ApproximateCloneDetectingSuffixTree($this->word))->findClones(
            $this->config->minTokens(),
            $this->config->editDistance(),
            $this->config->headEquality()
        );

        if ($useProgressBar) {
            $this->progressBar(1, $totalSteps, self::PROGRESS_BAR_PROCESS_CLONES_TITLE);
        }

        foreach ($cloneInfos as $cloneInfo) {
            /** @var int[] $others */
            $others = $cloneInfo->otherClones->extractFirstList();
            $cloneLength = $this->processCloneLength($cloneInfo->length, $cloneInfo->token->file);
            $cloneInfoLastToken = $this->getLastToken($cloneInfo->position, $cloneLength);
            $lines = $cloneInfoLastToken->line + 1 - $cloneInfo->token->line;

            if ($lines >= $this->config->minLines()) {
                for ($j = 0, $count = count($others); $j < $count; $j++) {
                    $otherToken = $this->word[$others[$j]];
                    $otherCloneLength = $this->processCloneLength($cloneLength, $otherToken->file);
                    $otherLastToken = $this->getLastToken($others[$j], $otherCloneLength);

                    /** @phpstan-ignore method.nonObject */
                    $this->result->add(
                        new CodeClone(
                            new CodeCloneFile(
                                $cloneInfo->token->file,
                                $cloneInfo->token->line,
                                $cloneInfo->token->line + $lines
                            ),
                            new CodeCloneFile($otherToken->file, $otherToken->line, $otherLastToken->line + 1),
                            $lines,
                            $cloneLength
                        )
                    );
                }
            }
        }

        if ($useProgressBar) {
            $this->progressBar(2, $totalSteps, self::PROGRESS_BAR_POST_PROCESS_DONE_TITLE);
        }
    }

    private function processCloneLength(int $cloneLength, string $file): int
    {
        if ($cloneLength > $this->fileTokens[$file]) {
            $cloneLength = $this->fileTokens[$file];
        }

        return $cloneLength;
    }

    private function getLastToken(int $position, int $cloneLength): AbstractToken
    {
        $lastToken = $this->word[$position + $cloneLength - 1];
        // If we stumbled upon the Sentinel, rewind one step.
        if ($lastToken instanceof Sentinel) {
            $lastToken = $this->word[$position + $cloneLength - 2];
        }

        return $lastToken;
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace Systemsdk\PhpCPD\Detector\Strategy;
6
7	use Systemsdk\PhpCPD\CodeClone;
8	use Systemsdk\PhpCPD\CodeCloneFile;
9	use Systemsdk\PhpCPD\CodeCloneMap;
10	use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\AbstractToken;
11	use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\ApproximateCloneDetectingSuffixTree;
12	use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\Sentinel;
13	use Systemsdk\PhpCPD\Detector\Strategy\SuffixTree\Token;
14	use Systemsdk\PhpCPD\Detector\Traits\ProgressBarTrait;
15	use Systemsdk\PhpCPD\Exceptions\MissingResultException;
16
17	use function array_key_exists;
18	use function array_keys;
19	use function file_get_contents;
20	use function is_array;
21	use function token_get_all;
22
23	use const T_ATTRIBUTE;
24
25	/**
26	* The suffix tree strategy was implemented in PHP for PHPCPD by Olle Härstedt.
27	*
28	* This PHP implementation is based on the Java implementation archived that is available at
29	* https://www.cqse.eu/en/news/blog/conqat-end-of-life/ under the Apache License 2.0.
30	*
31	* The aforementioned Java implementation is based on the algorithm described in
32	* https://dl.acm.org/doi/10.1109/ICSE.2009.5070547. This paper is available at
33	* https://www.cqse.eu/fileadmin/content/news/publications/2009-do-code-clones-matter.pdf.
34	*/
35	final class SuffixTreeStrategy extends AbstractStrategy
36	{
37	use ProgressBarTrait;
38
39	private const string PROGRESS_BAR_SEARCH_CLONES_TITLE = 'Search for clones';
40	private const string PROGRESS_BAR_PROCESS_CLONES_TITLE = 'Clones processing';
41	private const string PROGRESS_BAR_POST_PROCESS_DONE_TITLE = 'Post process done';
42
43	/**
44	* @var array<int, AbstractToken>
45	*/
46	private array $word = [];
47
48	/**
49	* @var array<string, int>
50	*/
51	private array $fileTokens = [];
52
53	private ?CodeCloneMap $result = null;
54
55	public function processFile(string $file, CodeCloneMap $result): void
56	{
57	$content = (string)file_get_contents($file);	4✔
58	$tokens = token_get_all($content);	4✔
59	$lastTokenLine = 0;	4✔
60	$attributeStarted = false;	4✔
61	$attributeStartedLine = 0;	4✔
62	$this->fileTokens[$file] = 0;	4✔
63
64	$result->addToNumberOfLines(substr_count($content, "\n"));	4✔
65
66	unset($content);	4✔
67
68	foreach (array_keys($tokens) as $key) {	4✔
69	/** @var array{0: int, 1:string, 2:int}\|string $token */
70	$token = $tokens[$key];	4✔
71
72	if (is_array($token)) {	4✔
73	if ($attributeStarted === false && !isset($this->tokensIgnoreList[$token[0]])) {	4✔
74	$this->word[] = new Token(	4✔
75	$token[0],	4✔
76	token_name($token[0]),	4✔
77	$token[2],	4✔
78	$file,	4✔
79	$token[1]	4✔
80	);	4✔
81	$this->fileTokens[$file]++;	4✔
82	}
83
84	if ($token[0] === T_ATTRIBUTE) {	4✔
85	$attributeStarted = true;	1✔
86	$attributeStartedLine = $token[2];	1✔
87	}
88
89	$lastTokenLine = $token[2];	4✔
90	} elseif (
91	$attributeStarted === true && $token === ']'	4✔
92	&& (
93	$attributeStartedLine === $lastTokenLine	4✔
94	\|\| (array_key_exists($key - 1, $tokens) && $tokens[$key - 1] === ')')	4✔
95	)
96	) {
97	$attributeStarted = false;	1✔
98	$attributeStartedLine = 0;	1✔
99	}
100	}
101
102	$this->result = $result;	4✔
103	}
104
105	/**
106	* @throws MissingResultException
107	*/
108	public function postProcess(bool $useProgressBar): void
109	{
110	if (empty($this->result)) {	4✔
111	throw new MissingResultException('Missing result');	×
112	}
113
114	$totalSteps = 2;	4✔
115
116	if ($useProgressBar) {	4✔
NEW 117	$this->progressBar(0, $totalSteps, self::PROGRESS_BAR_SEARCH_CLONES_TITLE);	×
118	}
119
120	// Sentinel = End of word
121	$this->word[] = new Sentinel();	4✔
122
123	$cloneInfos = (new ApproximateCloneDetectingSuffixTree($this->word))->findClones(	4✔
124	$this->config->minTokens(),	4✔
125	$this->config->editDistance(),	4✔
126	$this->config->headEquality()	4✔
127	);	4✔
128
129	if ($useProgressBar) {	4✔
NEW 130	$this->progressBar(1, $totalSteps, self::PROGRESS_BAR_PROCESS_CLONES_TITLE);	×
131	}
132
133	foreach ($cloneInfos as $cloneInfo) {	4✔
134	/** @var int[] $others */
135	$others = $cloneInfo->otherClones->extractFirstList();	3✔
136	$cloneLength = $this->processCloneLength($cloneInfo->length, $cloneInfo->token->file);	3✔
137	$cloneInfoLastToken = $this->getLastToken($cloneInfo->position, $cloneLength);	3✔
138	$lines = $cloneInfoLastToken->line + 1 - $cloneInfo->token->line;	3✔
139
140	if ($lines >= $this->config->minLines()) {	3✔
141	for ($j = 0, $count = count($others); $j < $count; $j++) {	3✔
142	$otherToken = $this->word[$others[$j]];	3✔
143	$otherCloneLength = $this->processCloneLength($cloneLength, $otherToken->file);	3✔
144	$otherLastToken = $this->getLastToken($others[$j], $otherCloneLength);	3✔
145
146	/** @phpstan-ignore method.nonObject */
147	$this->result->add(	3✔
148	new CodeClone(	3✔
149	new CodeCloneFile(	3✔
150	$cloneInfo->token->file,	3✔
151	$cloneInfo->token->line,	3✔
152	$cloneInfo->token->line + $lines	3✔
153	),	3✔
154	new CodeCloneFile($otherToken->file, $otherToken->line, $otherLastToken->line + 1),	3✔
155	$lines,	3✔
156	$cloneLength	3✔
157	)	3✔
158	);	3✔
159	}
160	}
161	}
162
163	if ($useProgressBar) {	4✔
NEW 164	$this->progressBar(2, $totalSteps, self::PROGRESS_BAR_POST_PROCESS_DONE_TITLE);	×
165	}
166	}
167
168	private function processCloneLength(int $cloneLength, string $file): int
169	{
170	if ($cloneLength > $this->fileTokens[$file]) {	3✔
171	$cloneLength = $this->fileTokens[$file];	×
172	}
173
174	return $cloneLength;	3✔
175	}
176
177	private function getLastToken(int $position, int $cloneLength): AbstractToken
178	{
179	$lastToken = $this->word[$position + $cloneLength - 1];	3✔
180	// If we stumbled upon the Sentinel, rewind one step.
181	if ($lastToken instanceof Sentinel) {	3✔
182	$lastToken = $this->word[$position + $cloneLength - 2];	×
183	}
184
185	return $lastToken;	3✔
186	}
187	}

systemsdk / phpcpd / #23

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous