systemsdk / phpcpd / #31

Committed 16 Feb 2026 08:55PM UTC coverage: 78.378% (+2.6%) from 75.818%

Build # #31

Build Type

push

Committed by DKravtsov

Commit Message

### Added

* Added `--ignore-no-files option` to return a success exit code if no files were found.
* Added `#[SuppressCpd]` to ignore code clones inside a class or method (`use Systemsdk\PhpCPD\Attributes\SuppressCpd;`).

### Updated

* Improved Suffix Tree-based algorithm for code clone detection.
* Updated Dev environment: Updated XDebug, Phing, dev composer dependencies.

Run Details

129 of 150 new or added lines in 6 files covered. (86.0%)

2 existing lines in 2 files now uncovered.

841 of 1073 relevant lines covered (78.38%)

8.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.07

/src/Detector/Strategy/DefaultStrategy.php

<?php

declare(strict_types=1);

namespace Systemsdk\PhpCPD\Detector\Strategy;

use Systemsdk\PhpCPD\CodeClone;
use Systemsdk\PhpCPD\CodeCloneFile;
use Systemsdk\PhpCPD\CodeCloneMap;
use Systemsdk\PhpCPD\Exceptions\ProcessingResultException;

use function array_key_exists;
use function array_keys;
use function chr;
use function count;
use function crc32;
use function file_get_contents;
use function is_array;
use function md5;
use function pack;
use function substr;
use function substr_count;
use function token_get_all;
use function uniqid;

use const T_ATTRIBUTE;
use const T_VARIABLE;

/**
 *  This is a Rabin-Karp with an additional normalization steps before the hashing happens.
 *
 *  1. Tokenization
 *  2. Deletion of logic neutral tokens like T_CLOSE_TAG;T_COMMENT; T_DOC_COMMENT; T_INLINE_HTML; T_NS_SEPARATOR;
 *      T_OPEN_TAG; T_OPEN_TAG_WITH_ECHO; T_USE; T_WHITESPACE;
 *  3. If needed deletion of variable names
 *  4. Normalization of token + value using crc32
 *  5. Now the classic Rabin-Karp hashing takes place
 */
final class DefaultStrategy extends AbstractStrategy
{
    /**
     * @var array<string, array{0: string, 1: int}>
     */
    private array $hashes = [];

    /**
     * @throws ProcessingResultException
     */
    public function processFile(string $file, CodeCloneMap $result): void
    {
        $buffer = (string)file_get_contents($file);
        /** @var array<int, int> $currentTokenPositions */
        $currentTokenPositions = [];
        /** @var array<int, int> $currentTokenRealPositions */
        $currentTokenRealPositions = [];
        $currentSignature = '';
        $tokens = token_get_all($buffer);
        $tokenNr = 0;
        $lastTokenLine = 0;
        $attributeStarted = false;
        $attributeStartedLine = 0;
        $firstHash = '';
        $firstToken = 0;
        $wasSuppressed = false;

        $result->addToNumberOfLines(substr_count($buffer, "\n"));

        unset($buffer);

        foreach (array_keys($tokens) as $key) {
            /** @var array{0: int, 1:string, 2:int}|string $token */
            $token = $tokens[$key];

            if (is_array($token)) {
                $tokenLine = (int)$token[2];

                // If the current line is protected by #[SuppressCpd]
                if ($this->guard->isLineSuppressed($file, $tokenLine)) {
                    // If we JUST entered the suppressed zone, place a Barrier Wall
                    if (!$wasSuppressed) {
                        if ($tokenNr === 0) {
                            $currentTokenPositions[$tokenNr] = 0;
                        } else {
                            $currentTokenPositions[$tokenNr] = $currentTokenPositions[$tokenNr - 1];
                        }

                        $currentTokenRealPositions[$tokenNr] = $tokenLine;

                        // Create a globally unique token (Barrier) to poison any sliding window
                        // that tries to cross this area. This prevents wormhole bugs.
                        $currentSignature .= chr(255) . pack('N*', crc32(uniqid('barrier', true)));
                        $tokenNr++;
                        $wasSuppressed = true;
                    }

                    $lastTokenLine = $tokenLine;

                    continue; // Skip the actual token, keeping it out of the engine
                }

                // We are in normal code, reset the flag
                $wasSuppressed = false;

                if ($attributeStarted === false && !isset($this->tokensIgnoreList[$token[0]])) {
                    if ($tokenNr === 0) {
                        $currentTokenPositions[$tokenNr] = $tokenLine - $lastTokenLine;
                    } else {
                        $currentTokenPositions[$tokenNr] = $currentTokenPositions[$tokenNr - 1] + $tokenLine
                            - $lastTokenLine;
                    }

                    $currentTokenRealPositions[$tokenNr++] = $tokenLine;

                    if ($token[0] === T_VARIABLE && $this->config->fuzzy()) {
                        $token[1] = 'variable';
                    }

                    $currentSignature .= chr($token[0] & 255) . pack('N*', crc32($token[1]));
                }

                if ($token[0] === T_ATTRIBUTE) {
                    $attributeStarted = true;
                    $attributeStartedLine = $tokenLine;
                }

                $lastTokenLine = $tokenLine;
            } elseif (
                $attributeStarted === true && $token === ']'
                && (
                    $attributeStartedLine === $lastTokenLine
                    || (array_key_exists($key - 1, $tokens) && $tokens[$key - 1] === ')')
                )
            ) {
                $attributeStarted = false;
                $attributeStartedLine = 0;
            }
        }

        $count = count($currentTokenPositions);
        $firstLine = 0;
        $firstRealLine = 0;
        $found = false;
        $tokenNr = 0;

        while ($tokenNr <= $count - $this->config->minTokens()) {
            $line = $currentTokenPositions[$tokenNr];
            $realLine = $currentTokenRealPositions[$tokenNr];

            $hash = substr(md5(substr($currentSignature, $tokenNr * 5, $this->config->minTokens() * 5), true), 0, 8);

            if (isset($this->hashes[$hash])) {
                $found = true;

                if ($firstLine === 0) {
                    $firstLine = $line;
                    $firstRealLine = $realLine;
                    $firstHash = $hash;
                    $firstToken = $tokenNr;
                }
            } else {
                if ($found) {
                    $this->processResult(
                        $result,
                        $firstHash,
                        $tokenNr,
                        $currentTokenPositions,
                        $currentTokenRealPositions,
                        $firstLine,
                        $firstRealLine,
                        $file,
                        $firstToken
                    );
                    $found = false;
                    $firstLine = 0;
                }

                $this->hashes[$hash] = [$file, $realLine];
            }

            $tokenNr++;
        }

        if ($found) {
            $this->processResult(
                $result,
                $firstHash,
                $tokenNr,
                $currentTokenPositions,
                $currentTokenRealPositions,
                $firstLine,
                $firstRealLine,
                $file,
                $firstToken
            );
        }
    }

    /**
     * @param array<int, int> $currentTokenPositions
     * @param array<int, int> $currentTokenRealPositions
     *
     * @throws ProcessingResultException
     */
    private function processResult(
        CodeCloneMap $result,
        string $firstHash,
        int $tokenNr,
        array $currentTokenPositions,
        array $currentTokenRealPositions,
        int $firstLine,
        int $firstRealLine,
        string $file,
        int $firstToken
    ): void {
        [$fileA, $firstLineA] = $this->hashes[$firstHash];
        $lastToken = ($tokenNr - 1) + $this->config->minTokens() - 1;
        $lastLine = $currentTokenPositions[$lastToken];
        $lastRealLine = $currentTokenRealPositions[$lastToken];
        $numLines = $lastLine + 1 - $firstLine;
        $realNumLines = $lastRealLine + 1 - $firstRealLine;

        if (($fileA !== $file || $firstLineA !== $firstRealLine) && $numLines >= $this->config->minLines()) {
            $result->add(
                new CodeClone(
                    new CodeCloneFile($fileA, $firstLineA, $firstLineA + $realNumLines),
                    new CodeCloneFile($file, $firstRealLine, $firstRealLine + $realNumLines),
                    $realNumLines,
                    $lastToken + 1 - $firstToken
                )
            );
        }
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace Systemsdk\PhpCPD\Detector\Strategy;
6
7	use Systemsdk\PhpCPD\CodeClone;
8	use Systemsdk\PhpCPD\CodeCloneFile;
9	use Systemsdk\PhpCPD\CodeCloneMap;
10	use Systemsdk\PhpCPD\Exceptions\ProcessingResultException;
11
12	use function array_key_exists;
13	use function array_keys;
14	use function chr;
15	use function count;
16	use function crc32;
17	use function file_get_contents;
18	use function is_array;
19	use function md5;
20	use function pack;
21	use function substr;
22	use function substr_count;
23	use function token_get_all;
24	use function uniqid;
25
26	use const T_ATTRIBUTE;
27	use const T_VARIABLE;
28
29	/**
30	* This is a Rabin-Karp with an additional normalization steps before the hashing happens.
31	*
32	* 1. Tokenization
33	* 2. Deletion of logic neutral tokens like T_CLOSE_TAG;T_COMMENT; T_DOC_COMMENT; T_INLINE_HTML; T_NS_SEPARATOR;
34	* T_OPEN_TAG; T_OPEN_TAG_WITH_ECHO; T_USE; T_WHITESPACE;
35	* 3. If needed deletion of variable names
36	* 4. Normalization of token + value using crc32
37	* 5. Now the classic Rabin-Karp hashing takes place
38	*/
39	final class DefaultStrategy extends AbstractStrategy
40	{
41	/**
42	* @var array<string, array{0: string, 1: int}>
43	*/
44	private array $hashes = [];
45
46	/**
47	* @throws ProcessingResultException
48	*/
49	public function processFile(string $file, CodeCloneMap $result): void
50	{
51	$buffer = (string)file_get_contents($file);	15✔
52	/** @var array<int, int> $currentTokenPositions */
53	$currentTokenPositions = [];	15✔
54	/** @var array<int, int> $currentTokenRealPositions */
55	$currentTokenRealPositions = [];	15✔
56	$currentSignature = '';	15✔
57	$tokens = token_get_all($buffer);	15✔
58	$tokenNr = 0;	15✔
59	$lastTokenLine = 0;	15✔
60	$attributeStarted = false;	15✔
61	$attributeStartedLine = 0;	15✔
62	$firstHash = '';	15✔
63	$firstToken = 0;	15✔
64	$wasSuppressed = false;	15✔
65
66	$result->addToNumberOfLines(substr_count($buffer, "\n"));	15✔
67
68	unset($buffer);	15✔
69
70	foreach (array_keys($tokens) as $key) {	15✔
71	/** @var array{0: int, 1:string, 2:int}\|string $token */
72	$token = $tokens[$key];	15✔
73
74	if (is_array($token)) {	15✔
75	$tokenLine = (int)$token[2];	15✔
76
77	// If the current line is protected by #[SuppressCpd]
78	if ($this->guard->isLineSuppressed($file, $tokenLine)) {	15✔
79	// If we JUST entered the suppressed zone, place a Barrier Wall
80	if (!$wasSuppressed) {	6✔
81	if ($tokenNr === 0) {	6✔
NEW 82	$currentTokenPositions[$tokenNr] = 0;	×
83	} else {
84	$currentTokenPositions[$tokenNr] = $currentTokenPositions[$tokenNr - 1];	6✔
85	}
86
87	$currentTokenRealPositions[$tokenNr] = $tokenLine;	6✔
88
89	// Create a globally unique token (Barrier) to poison any sliding window
90	// that tries to cross this area. This prevents wormhole bugs.
91	$currentSignature .= chr(255) . pack('N*', crc32(uniqid('barrier', true)));	6✔
92	$tokenNr++;	6✔
93	$wasSuppressed = true;	6✔
94	}
95
96	$lastTokenLine = $tokenLine;	6✔
97
98	continue; // Skip the actual token, keeping it out of the engine	6✔
99	}
100
101	// We are in normal code, reset the flag
102	$wasSuppressed = false;	15✔
103
104	if ($attributeStarted === false && !isset($this->tokensIgnoreList[$token[0]])) {	15✔
105	if ($tokenNr === 0) {	15✔
106	$currentTokenPositions[$tokenNr] = $tokenLine - $lastTokenLine;	15✔
107	} else {
108	$currentTokenPositions[$tokenNr] = $currentTokenPositions[$tokenNr - 1] + $tokenLine	15✔
109	- $lastTokenLine;	15✔
110	}
111
112	$currentTokenRealPositions[$tokenNr++] = $tokenLine;	15✔
113
114	if ($token[0] === T_VARIABLE && $this->config->fuzzy()) {	15✔
115	$token[1] = 'variable';	2✔
116	}
117
118	$currentSignature .= chr($token[0] & 255) . pack('N*', crc32($token[1]));	15✔
119	}
120
121	if ($token[0] === T_ATTRIBUTE) {	15✔
122	$attributeStarted = true;	1✔
123	$attributeStartedLine = $tokenLine;	1✔
124	}
125
126	$lastTokenLine = $tokenLine;	15✔
127	} elseif (
128	$attributeStarted === true && $token === ']'	15✔
129	&& (
130	$attributeStartedLine === $lastTokenLine	15✔
131	\|\| (array_key_exists($key - 1, $tokens) && $tokens[$key - 1] === ')')	15✔
132	)
133	) {
134	$attributeStarted = false;	1✔
135	$attributeStartedLine = 0;	1✔
136	}
137	}
138
139	$count = count($currentTokenPositions);	15✔
140	$firstLine = 0;	15✔
141	$firstRealLine = 0;	15✔
142	$found = false;	15✔
143	$tokenNr = 0;	15✔
144
145	while ($tokenNr <= $count - $this->config->minTokens()) {	15✔
146	$line = $currentTokenPositions[$tokenNr];	13✔
147	$realLine = $currentTokenRealPositions[$tokenNr];	13✔
148
149	$hash = substr(md5(substr($currentSignature, $tokenNr * 5, $this->config->minTokens() * 5), true), 0, 8);	13✔
150
151	if (isset($this->hashes[$hash])) {	13✔
152	$found = true;	9✔
153
154	if ($firstLine === 0) {	9✔
155	$firstLine = $line;	9✔
156	$firstRealLine = $realLine;	9✔
157	$firstHash = $hash;	9✔
158	$firstToken = $tokenNr;	9✔
159	}
160	} else {
161	if ($found) {	13✔
162	$this->processResult(	1✔
163	$result,	1✔
164	$firstHash,	1✔
165	$tokenNr,	1✔
166	$currentTokenPositions,	1✔
167	$currentTokenRealPositions,	1✔
168	$firstLine,	1✔
169	$firstRealLine,	1✔
170	$file,	1✔
171	$firstToken	1✔
172	);	1✔
173	$found = false;	1✔
174	$firstLine = 0;	1✔
175	}
176
177	$this->hashes[$hash] = [$file, $realLine];	13✔
178	}
179
180	$tokenNr++;	13✔
181	}
182
183	if ($found) {	15✔
184	$this->processResult(	9✔
185	$result,	9✔
186	$firstHash,	9✔
187	$tokenNr,	9✔
188	$currentTokenPositions,	9✔
189	$currentTokenRealPositions,	9✔
190	$firstLine,	9✔
191	$firstRealLine,	9✔
192	$file,	9✔
193	$firstToken	9✔
194	);	9✔
195	}
196	}
197
198	/**
199	* @param array<int, int> $currentTokenPositions
200	* @param array<int, int> $currentTokenRealPositions
201	*
202	* @throws ProcessingResultException
203	*/
204	private function processResult(
205	CodeCloneMap $result,
206	string $firstHash,
207	int $tokenNr,
208	array $currentTokenPositions,
209	array $currentTokenRealPositions,
210	int $firstLine,
211	int $firstRealLine,
212	string $file,
213	int $firstToken
214	): void {
215	[$fileA, $firstLineA] = $this->hashes[$firstHash];	9✔
216	$lastToken = ($tokenNr - 1) + $this->config->minTokens() - 1;	9✔
217	$lastLine = $currentTokenPositions[$lastToken];	9✔
218	$lastRealLine = $currentTokenRealPositions[$lastToken];	9✔
219	$numLines = $lastLine + 1 - $firstLine;	9✔
220	$realNumLines = $lastRealLine + 1 - $firstRealLine;	9✔
221
222	if (($fileA !== $file \|\| $firstLineA !== $firstRealLine) && $numLines >= $this->config->minLines()) {	9✔
223	$result->add(	9✔
224	new CodeClone(	9✔
225	new CodeCloneFile($fileA, $firstLineA, $firstLineA + $realNumLines),	9✔
226	new CodeCloneFile($file, $firstRealLine, $firstRealLine + $realNumLines),	9✔
227	$realNumLines,	9✔
228	$lastToken + 1 - $firstToken	9✔
229	)	9✔
230	);	9✔
231	}
232	}
233	}

systemsdk / phpcpd / #31

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous