systemsdk / phpcpd / #4

Committed 30 Dec 2024 04:42PM UTC coverage: 75.789%. Remained the same

Build # #4

Build Type

push

Committed by DKravtsov

Commit Message

phpcpd 8.0.0 release. Made codebase refactoring. Updated requirements php 8.3, updated composer dependencies, updated tests to the PHPUnit 11. Updated dev environment to the php 8.4, Phing 3.0, added code quality tools: ecs, phpstan.

Run Details

271 of 379 new or added lines in 20 files covered. (71.5%)

65 existing lines in 13 files now uncovered.

648 of 855 relevant lines covered (75.79%)

2.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.02

/src/Detector/Strategy/DefaultStrategy.php

<?php

declare(strict_types=1);

namespace Systemsdk\PhpCPD\Detector\Strategy;

use Systemsdk\PhpCPD\CodeClone;
use Systemsdk\PhpCPD\CodeCloneFile;
use Systemsdk\PhpCPD\CodeCloneMap;
use Systemsdk\PhpCPD\Exceptions\ProcessingResultException;

use function array_key_exists;
use function array_keys;
use function chr;
use function count;
use function crc32;
use function file_get_contents;
use function is_array;
use function md5;
use function pack;
use function substr;
use function substr_count;
use function token_get_all;

use const T_ATTRIBUTE;
use const T_VARIABLE;

/**
 *  This is a Rabin-Karp with an additional normalization steps before the hashing happens.
 *
 *  1. Tokenization
 *  2. Deletion of logic neutral tokens like T_CLOSE_TAG;T_COMMENT; T_DOC_COMMENT; T_INLINE_HTML; T_NS_SEPARATOR;
 *      T_OPEN_TAG; T_OPEN_TAG_WITH_ECHO; T_USE; T_WHITESPACE;
 *  3. If needed deletion of variable names
 *  4. Normalization of token + value using crc32
 *  5. Now the classic Rabin-Karp hashing takes place
 */
final class DefaultStrategy extends AbstractStrategy
{
    /**
     * @var array<string, array{0: string, 1: int}>
     */
    private array $hashes = [];

    /**
     * @throws ProcessingResultException
     */
    public function processFile(string $file, CodeCloneMap $result): void
    {
        $buffer = (string)file_get_contents($file);
        /** @var array<int, int> $currentTokenPositions */
        $currentTokenPositions = [];
        /** @var array<int, int> $currentTokenRealPositions */
        $currentTokenRealPositions = [];
        $currentSignature = '';
        $tokens = token_get_all($buffer);
        $tokenNr = 0;
        $lastTokenLine = 0;
        $attributeStarted = false;
        $attributeStartedLine = 0;
        $firstHash = '';
        $firstToken = 0;

        $result->addToNumberOfLines(substr_count($buffer, "\n"));

        unset($buffer);

        foreach (array_keys($tokens) as $key) {
            /** @var array{0: int, 1:string, 2:int}|string $token */
            $token = $tokens[$key];

            if (is_array($token)) {
                if ($attributeStarted === false && !isset($this->tokensIgnoreList[$token[0]])) {
                    if ($tokenNr === 0) {
                        $currentTokenPositions[$tokenNr] = $token[2] - $lastTokenLine;
                    } else {
                        $currentTokenPositions[$tokenNr] = $currentTokenPositions[$tokenNr - 1] + $token[2]
                            - $lastTokenLine;
                    }

                    $currentTokenRealPositions[$tokenNr++] = (int)$token[2];

                    if ($token[0] === T_VARIABLE && $this->config->fuzzy()) {
                        $token[1] = 'variable';
                    }

                    $currentSignature .= chr($token[0] & 255) . pack('N*', crc32($token[1]));
                }

                if ($token[0] === T_ATTRIBUTE) {
                    $attributeStarted = true;
                    $attributeStartedLine = $token[2];
                }

                $lastTokenLine = $token[2];
            } elseif (
                $attributeStarted === true && $token === ']'
                && (
                    $attributeStartedLine === $lastTokenLine
                    || (array_key_exists($key - 1, $tokens) && $tokens[$key - 1] === ')')
                )
            ) {
                $attributeStarted = false;
                $attributeStartedLine = 0;
            }
        }

        $count = count($currentTokenPositions);
        $firstLine = 0;
        $firstRealLine = 0;
        $found = false;
        $tokenNr = 0;

        while ($tokenNr <= $count - $this->config->minTokens()) {
            $line = $currentTokenPositions[$tokenNr];
            $realLine = $currentTokenRealPositions[$tokenNr];

            $hash = substr(md5(substr($currentSignature, $tokenNr * 5, $this->config->minTokens() * 5), true), 0, 8);

            if (isset($this->hashes[$hash])) {
                $found = true;

                if ($firstLine === 0) {
                    $firstLine = $line;
                    $firstRealLine = $realLine;
                    $firstHash = $hash;
                    $firstToken = $tokenNr;
                }
            } else {
                if ($found) {
                    $this->processResult(
                        $result,
                        $firstHash,
                        $tokenNr,
                        $currentTokenPositions,
                        $currentTokenRealPositions,
                        $firstLine,
                        $firstRealLine,
                        $file,
                        $firstToken
                    );
                    $found = false;
                    $firstLine = 0;
                }

                $this->hashes[$hash] = [$file, $realLine];
            }

            $tokenNr++;
        }

        if ($found) {
            $this->processResult(
                $result,
                $firstHash,
                $tokenNr,
                $currentTokenPositions,
                $currentTokenRealPositions,
                $firstLine,
                $firstRealLine,
                $file,
                $firstToken
            );
        }
    }

    /**
     * @param array<int, int> $currentTokenPositions
     * @param array<int, int> $currentTokenRealPositions
     *
     * @throws ProcessingResultException
     */
    private function processResult(
        CodeCloneMap $result,
        string $firstHash,
        int $tokenNr,
        array $currentTokenPositions,
        array $currentTokenRealPositions,
        int $firstLine,
        int $firstRealLine,
        string $file,
        int $firstToken
    ): void {
        [$fileA, $firstLineA] = $this->hashes[$firstHash];
        $lastToken = ($tokenNr - 1) + $this->config->minTokens() - 1;
        $lastLine = $currentTokenPositions[$lastToken];
        $lastRealLine = $currentTokenRealPositions[$lastToken];
        $numLines = $lastLine + 1 - $firstLine;
        $realNumLines = $lastRealLine + 1 - $firstRealLine;

        if (($fileA !== $file || $firstLineA !== $firstRealLine) && $numLines >= $this->config->minLines()) {
            $result->add(
                new CodeClone(
                    new CodeCloneFile($fileA, $firstLineA),
                    new CodeCloneFile($file, $firstRealLine),
                    $realNumLines,
                    $lastToken + 1 - $firstToken
                )
            );
        }
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace Systemsdk\PhpCPD\Detector\Strategy;
6
7	use Systemsdk\PhpCPD\CodeClone;
8	use Systemsdk\PhpCPD\CodeCloneFile;
9	use Systemsdk\PhpCPD\CodeCloneMap;
10	use Systemsdk\PhpCPD\Exceptions\ProcessingResultException;
11
12	use function array_key_exists;
13	use function array_keys;
14	use function chr;
15	use function count;
16	use function crc32;
17	use function file_get_contents;
18	use function is_array;
19	use function md5;
20	use function pack;
21	use function substr;
22	use function substr_count;
23	use function token_get_all;
24
25	use const T_ATTRIBUTE;
26	use const T_VARIABLE;
27
28	/**
29	* This is a Rabin-Karp with an additional normalization steps before the hashing happens.
30	*
31	* 1. Tokenization
32	* 2. Deletion of logic neutral tokens like T_CLOSE_TAG;T_COMMENT; T_DOC_COMMENT; T_INLINE_HTML; T_NS_SEPARATOR;
33	* T_OPEN_TAG; T_OPEN_TAG_WITH_ECHO; T_USE; T_WHITESPACE;
34	* 3. If needed deletion of variable names
35	* 4. Normalization of token + value using crc32
36	* 5. Now the classic Rabin-Karp hashing takes place
37	*/
38	final class DefaultStrategy extends AbstractStrategy
39	{
40	/**
41	* @var array<string, array{0: string, 1: int}>
42	*/
43	private array $hashes = [];
44
45	/**
46	* @throws ProcessingResultException
47	*/
48	public function processFile(string $file, CodeCloneMap $result): void
49	{
50	$buffer = (string)file_get_contents($file);	8✔
51	/** @var array<int, int> $currentTokenPositions */
52	$currentTokenPositions = [];	8✔
53	/** @var array<int, int> $currentTokenRealPositions */
54	$currentTokenRealPositions = [];	8✔
55	$currentSignature = '';	8✔
56	$tokens = token_get_all($buffer);	8✔
57	$tokenNr = 0;	8✔
58	$lastTokenLine = 0;	8✔
59	$attributeStarted = false;	8✔
60	$attributeStartedLine = 0;	8✔
61	$firstHash = '';	8✔
62	$firstToken = 0;	8✔
63
64	$result->addToNumberOfLines(substr_count($buffer, "\n"));	8✔
65
66	unset($buffer);	8✔
67
68	foreach (array_keys($tokens) as $key) {	8✔
69	/** @var array{0: int, 1:string, 2:int}\|string $token */
70	$token = $tokens[$key];	8✔
71
72	if (is_array($token)) {	8✔
73	if ($attributeStarted === false && !isset($this->tokensIgnoreList[$token[0]])) {	8✔
74	if ($tokenNr === 0) {	8✔
75	$currentTokenPositions[$tokenNr] = $token[2] - $lastTokenLine;	8✔
76	} else {
77	$currentTokenPositions[$tokenNr] = $currentTokenPositions[$tokenNr - 1] + $token[2]	8✔
78	- $lastTokenLine;	8✔
79	}
80
81	$currentTokenRealPositions[$tokenNr++] = (int)$token[2];	8✔
82
83	if ($token[0] === T_VARIABLE && $this->config->fuzzy()) {	8✔
84	$token[1] = 'variable';	2✔
85	}
86
87	$currentSignature .= chr($token[0] & 255) . pack('N*', crc32($token[1]));	8✔
88	}
89
90	if ($token[0] === T_ATTRIBUTE) {	8✔
91	$attributeStarted = true;	1✔
92	$attributeStartedLine = $token[2];	1✔
93	}
94
95	$lastTokenLine = $token[2];	8✔
96	} elseif (
97	$attributeStarted === true && $token === ']'	8✔
98	&& (
99	$attributeStartedLine === $lastTokenLine	8✔
100	\|\| (array_key_exists($key - 1, $tokens) && $tokens[$key - 1] === ')')	8✔
101	)
102	) {
103	$attributeStarted = false;	1✔
104	$attributeStartedLine = 0;	1✔
105	}
106	}
107
108	$count = count($currentTokenPositions);	8✔
109	$firstLine = 0;	8✔
110	$firstRealLine = 0;	8✔
111	$found = false;	8✔
112	$tokenNr = 0;	8✔
113
114	while ($tokenNr <= $count - $this->config->minTokens()) {	8✔
115	$line = $currentTokenPositions[$tokenNr];	6✔
116	$realLine = $currentTokenRealPositions[$tokenNr];	6✔
117
118	$hash = substr(md5(substr($currentSignature, $tokenNr * 5, $this->config->minTokens() * 5), true), 0, 8);	6✔
119
120	if (isset($this->hashes[$hash])) {	6✔
121	$found = true;	5✔
122
123	if ($firstLine === 0) {	5✔
124	$firstLine = $line;	5✔
125	$firstRealLine = $realLine;	5✔
126	$firstHash = $hash;	5✔
127	$firstToken = $tokenNr;	5✔
128	}
129	} else {
130	if ($found) {	6✔
NEW 131	$this->processResult(	×
NEW 132	$result,	×
NEW 133	$firstHash,	×
NEW 134	$tokenNr,	×
NEW 135	$currentTokenPositions,	×
NEW 136	$currentTokenRealPositions,	×
NEW 137	$firstLine,	×
NEW 138	$firstRealLine,	×
NEW 139	$file,	×
NEW 140	$firstToken	×
NEW 141	);	×
NEW 142	$found = false;	×
UNCOV 143	$firstLine = 0;	×
144	}
145
146	$this->hashes[$hash] = [$file, $realLine];	6✔
147	}
148
149	$tokenNr++;	6✔
150	}
151
152	if ($found) {	8✔
153	$this->processResult(	5✔
154	$result,	5✔
155	$firstHash,	5✔
156	$tokenNr,	5✔
157	$currentTokenPositions,	5✔
158	$currentTokenRealPositions,	5✔
159	$firstLine,	5✔
160	$firstRealLine,	5✔
161	$file,	5✔
162	$firstToken	5✔
163	);	5✔
164	}
165	}
166
167	/**
168	* @param array<int, int> $currentTokenPositions
169	* @param array<int, int> $currentTokenRealPositions
170	*
171	* @throws ProcessingResultException
172	*/
173	private function processResult(
174	CodeCloneMap $result,
175	string $firstHash,
176	int $tokenNr,
177	array $currentTokenPositions,
178	array $currentTokenRealPositions,
179	int $firstLine,
180	int $firstRealLine,
181	string $file,
182	int $firstToken
183	): void {
184	[$fileA, $firstLineA] = $this->hashes[$firstHash];	5✔
185	$lastToken = ($tokenNr - 1) + $this->config->minTokens() - 1;	5✔
186	$lastLine = $currentTokenPositions[$lastToken];	5✔
187	$lastRealLine = $currentTokenRealPositions[$lastToken];	5✔
188	$numLines = $lastLine + 1 - $firstLine;	5✔
189	$realNumLines = $lastRealLine + 1 - $firstRealLine;	5✔
190
191	if (($fileA !== $file \|\| $firstLineA !== $firstRealLine) && $numLines >= $this->config->minLines()) {	5✔
192	$result->add(	5✔
193	new CodeClone(	5✔
194	new CodeCloneFile($fileA, $firstLineA),	5✔
195	new CodeCloneFile($file, $firstRealLine),	5✔
196	$realNumLines,	5✔
197	$lastToken + 1 - $firstToken	5✔
198	)	5✔
199	);	5✔
200	}
201	}
202	}

systemsdk / phpcpd / #4

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous