7392476149

Committed 12 Dec 2023 10:40PM UTC coverage: 83.803% (-0.05%) from 83.848%

Build # 7392476149

Build Type

push

github

Committed by

web-flow

Commit Message

Feature flag to add `timestamp_real` (#23)

Run Details

2 of 3 new or added lines in 1 file covered. (66.67%)

15 existing lines in 2 files now uncovered.

952 of 1136 relevant lines covered (83.8%)

138.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.01

/src/CliCommandMultiProc.php

<?php

/**
 * JBZoo Toolbox - Cli.
 *
 * This file is part of the JBZoo Toolbox project.
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 *
 * @license    MIT
 * @copyright  Copyright (C) JBZoo.com, All rights reserved.
 * @see        https://github.com/JBZoo/Cli
 */

declare(strict_types=1);

namespace JBZoo\Cli;

use BluePsyduck\SymfonyProcessManager\ProcessManager;
use JBZoo\Cli\ProgressBars\ProgressBarProcessManager;
use JBZoo\Utils\Cli as CliUtils;
use JBZoo\Utils\Sys;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Process\Process;

use function JBZoo\Utils\int;

abstract class CliCommandMultiProc extends CliCommand
{
    private const PM_DEFAULT_INTERVAL    = 100;
    private const PM_DEFAULT_START_DELAY = 1;
    private const PM_DEFAULT_TIMEOUT     = 7200;

    private array                      $procPool    = [];
    private ?ProgressBarProcessManager $progressBar = null;

    abstract protected function executeOneProcess(string $pmThreadId): int;

    /**
     * @return string[]
     */
    abstract protected function getListOfChildIds(): array;

    /**
     * {@inheritDoc}
     */
    protected function configure(): void
    {
        $this
            ->addOption(
                'pm-max',
                null,
                InputOption::VALUE_REQUIRED,
                'Process Manager. The number of processes to execute in parallel (os isolated processes)',
                'auto',
            )
            ->addOption(
                'pm-interval',
                null,
                InputOption::VALUE_REQUIRED,
                'Process Manager. The interval to use for polling the processes, in milliseconds',
                self::PM_DEFAULT_INTERVAL,
            )
            ->addOption(
                'pm-start-delay',
                null,
                InputOption::VALUE_REQUIRED,
                'Process Manager. The time to delay the start of processes to space them out, in milliseconds',
                self::PM_DEFAULT_START_DELAY,
            )
            ->addOption(
                'pm-max-timeout',
                null,
                InputOption::VALUE_REQUIRED,
                'Process Manager. The max timeout for each proccess, in seconds',
                self::PM_DEFAULT_TIMEOUT,
            )
            ->addOption(
                'pm-proc-id',
                null,
                InputOption::VALUE_REQUIRED,
                'Process Manager. Unique ID of process to execute one child proccess.',
                '',
            );

        parent::configure();
    }

    /**
     * @phan-suppress PhanPluginPossiblyStaticProtectedMethod
     */
    protected function beforeStartAllProcesses(): void
    {
        // noop
    }

    /**
     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
     * @phan-suppress PhanPluginPossiblyStaticProtectedMethod
     * @phan-suppress PhanUnusedProtectedNoOverrideMethodParameter
     */
    protected function afterFinishAllProcesses(array $procPool): void
    {
        // noop
    }

    protected function executeAction(): int
    {
        $pmProcId = $this->getOptString('pm-proc-id');
        if ($pmProcId !== '') {
            return $this->executeOneProcess($pmProcId);
        }

        return $this->executeMultiProcessAction();
    }

    protected function executeMultiProcessAction(): int
    {
        $procNum  = $this->getNumberOfProcesses();
        $cpuCores = CliHelper::getNumberOfCpuCores();
        $this->_("Max number of sub-processes: {$procNum}", OutLvl::DEBUG);
        if ($procNum > $cpuCores) {
            $this->_(
                "The specified number of processes (--pm-max={$procNum}) "
                . "is more than the found number of CPU cores in the system ({$cpuCores}).",
                OutLvl::WARNING,
            );
        }

        $procManager = $this->initProcManager($procNum, $this->gePmInterval(), $this->getPmStartDelay());

        $procListIds = $this->getListOfChildIds();

        if (!$this->outputMode->isProgressBarDisabled()) {
            $this->progressBar = new ProgressBarProcessManager($this->outputMode);
            $this->progressBar->setMax(\count($procListIds));
            $this->progressBar->start();
        }

        foreach ($procListIds as $procListId) {
            $childProcess = $this->createSubProcess($procListId);
            $procManager->addProcess($childProcess);
        }

        $this->beforeStartAllProcesses();
        $procManager->waitForAllProcesses();
        if ($this->progressBar !== null) {
            $this->progressBar->finish();
            $this->_('');
        }

        $this->afterFinishAllProcesses($this->procPool);

        $errorList = $this->getErrorList();
        if (\count($errorList) > 0) {
            throw new Exception(\implode("\n" . \str_repeat('-', 60) . "\n", $errorList));
        }

        $warningList = $this->getWarningList();
        if (\count($warningList) > 0) {
            $this->_(\implode("\n" . \str_repeat('-', 60) . "\n", $warningList), OutLvl::WARNING);
        }

        return 0;
    }

    private function initProcManager(
        int $numberOfParallelProcesses,
        int $pollInterval,
        int $processStartDelay,
    ): ProcessManager {
        $finishCallback = function (Process $process): void {
            $virtProcId = \spl_object_id($process);

            $exitCode    = $process->getExitCode();
            $errorOutput = \trim($process->getErrorOutput());
            $stdOutput   = \trim($process->getOutput());

            $this->procPool[$virtProcId]['time_end']  = \microtime(true);
            $this->procPool[$virtProcId]['exit_code'] = $exitCode;
            $this->procPool[$virtProcId]['std_out']   = $stdOutput;

            if ($exitCode > 0 || $errorOutput !== '') {
                $this->procPool[$virtProcId]['err_out'] = $errorOutput;
            }

            if ($this->progressBar !== null) {
                $this->progressBar->advance();
            }
        };

        return (new ProcessManager())
            ->setPollInterval($pollInterval)
            ->setNumberOfParallelProcesses($numberOfParallelProcesses)
            ->setProcessStartDelay($processStartDelay)
            ->setProcessStartCallback(function (Process $process): void {
                $virtProcId                                = \spl_object_id($process);
                $this->procPool[$virtProcId]['time_start'] = \microtime(true);
            })
            ->setProcessFinishCallback($finishCallback)
            ->setProcessTimeoutCallback(function (Process $process) use ($finishCallback): void {
                $finishCallback($process);

                $virtProcId                                     = \spl_object_id($process);
                $this->procPool[$virtProcId]['reached_timeout'] = true;
            });
    }

    private function createSubProcess(string $procId): Process
    {
        // Prepare option list from the parent process
        $options = \array_filter(
            $this->outputMode->getInput()->getOptions(),
            static fn ($optionValue): bool => $optionValue !== false && $optionValue !== '',
        );

        foreach (\array_keys($options) as $optionKey) {
            if (!$this->getDefinition()->getOption((string)$optionKey)->acceptValue()) {
                $options[$optionKey] = null;
            }
        }

        unset($options['ansi']);
        $options['no-ansi']        = null;
        $options['no-interaction'] = null;
        $options['pm-proc-id']     = $procId;

        // Prepare $argument list from the parent process
        $arguments     = $this->outputMode->getInput()->getArguments();
        $argumentsList = [];

        foreach ($arguments as $argKey => $argValue) {
            if (\is_array($argValue)) {
                continue;
            }

            /** @var string $argValue */
            if ($argKey !== 'command') {
                /** @phan-suppress-next-line PhanPartialTypeMismatchArgumentInternal */
                $argumentsList[] = '"' . \addcslashes($argValue, '"') . '"';
            }
        }

        // Build full command line
        $process = Process::fromShellCommandline(
            CliUtils::build(
                \implode(
                    ' ',
                    \array_filter([
                        Sys::getBinary(),
                        CliHelper::getBinPath(),
                        $this->getName(),
                        \implode(' ', $argumentsList),
                    ]),
                ),
                $options,
            ),
            CliHelper::getRootPath(),
            null,
            null,
            $this->getMaxTimeout(),
        );

        $this->procPool[\spl_object_id($process)] = [
            'command'         => $process->getCommandLine(),
            'proc_id'         => $procId,
            'exit_code'       => null,
            'std_out'         => null,
            'err_out'         => null,
            'reached_timeout' => false,
            'time_start'      => null,
            'time_end'        => null,
        ];

        return $process;
    }

    private function getErrorList(): array
    {
        return \array_reduce($this->procPool, function (array $acc, array $procInfo): array {
            if ($procInfo['reached_timeout']) {
                $acc[] = \implode("\n", [
                    "Command : {$procInfo['command']}",
                    "Error   : The process with ID \"{$procInfo['proc_id']}\""
                    . " exceeded the timeout of {$this->getMaxTimeout()} seconds.",
                ]);
            } elseif ($procInfo['err_out'] && $procInfo['exit_code'] > 0) {
                $acc[] = \implode("\n", [
                    "Command : {$procInfo['command']}",
                    "Code    : {$procInfo['exit_code']}",
                    "Error   : {$procInfo['err_out']}",
                    "StdOut  : {$procInfo['std_out']}",
                ]);
            }

            return $acc;
        }, []);
    }

    private function getWarningList(): array
    {
        return \array_reduce($this->procPool, static function (array $acc, array $procInfo): array {
            if ($procInfo['err_out'] && $procInfo['exit_code'] === 0) {
                $acc[] = \implode("\n", [
                    "Command : {$procInfo['command']}",
                    "Warning : {$procInfo['err_out']}",
                    "StdOut  : {$procInfo['std_out']}",
                ]);
            }

            return $acc;
        }, []);
    }

    private function getMaxTimeout(): int
    {
        $pmMaxTimeout = $this->getOptInt('pm-max-timeout');

        return $pmMaxTimeout > 0 ? $pmMaxTimeout : self::PM_DEFAULT_TIMEOUT;
    }

    private function gePmInterval(): int
    {
        $pmInterval = $this->getOptInt('pm-interval');

        return $pmInterval > 0 ? $pmInterval : self::PM_DEFAULT_INTERVAL;
    }

    private function getPmStartDelay(): int
    {
        $pmStartDelay = $this->getOptInt('pm-start-delay');

        return $pmStartDelay > 0 ? $pmStartDelay : self::PM_DEFAULT_START_DELAY;
    }

    private function getNumberOfProcesses(): int
    {
        $pmMax    = \strtolower($this->getOptString('pm-max'));
        $cpuCores = CliHelper::getNumberOfCpuCores();

        if ($pmMax === 'auto') {
            return $cpuCores;
        }

        $pmMaxInt = \abs(int($pmMax));

        return $pmMaxInt > 0 ? $pmMaxInt : $cpuCores;
    }
}

1	<?php
2
3	/**
4	* JBZoo Toolbox - Cli.
5	*
6	* This file is part of the JBZoo Toolbox project.
7	* For the full copyright and license information, please view the LICENSE
8	* file that was distributed with this source code.
9	*
10	* @license MIT
11	* @copyright Copyright (C) JBZoo.com, All rights reserved.
12	* @see https://github.com/JBZoo/Cli
13	*/
14
15	declare(strict_types=1);
16
17	namespace JBZoo\Cli;
18
19	use BluePsyduck\SymfonyProcessManager\ProcessManager;
20	use JBZoo\Cli\ProgressBars\ProgressBarProcessManager;
21	use JBZoo\Utils\Cli as CliUtils;
22	use JBZoo\Utils\Sys;
23	use Symfony\Component\Console\Input\InputOption;
24	use Symfony\Component\Process\Process;
25
26	use function JBZoo\Utils\int;
27
28	abstract class CliCommandMultiProc extends CliCommand
29	{
30	private const PM_DEFAULT_INTERVAL = 100;
31	private const PM_DEFAULT_START_DELAY = 1;
32	private const PM_DEFAULT_TIMEOUT = 7200;
33
34	private array $procPool = [];
35	private ?ProgressBarProcessManager $progressBar = null;
36
37	abstract protected function executeOneProcess(string $pmThreadId): int;
38
39	/**
40	* @return string[]
41	*/
42	abstract protected function getListOfChildIds(): array;
43
44	/**
45	* {@inheritDoc}
46	*/
47	protected function configure(): void
48	{
49	$this	292✔
50	->addOption(	584✔
51	'pm-max',	292✔
52	null,	292✔
53	InputOption::VALUE_REQUIRED,	292✔
54	'Process Manager. The number of processes to execute in parallel (os isolated processes)',	292✔
55	'auto',	292✔
56	)	292✔
57	->addOption(	584✔
58	'pm-interval',	292✔
59	null,	292✔
60	InputOption::VALUE_REQUIRED,	292✔
61	'Process Manager. The interval to use for polling the processes, in milliseconds',	292✔
62	self::PM_DEFAULT_INTERVAL,	292✔
63	)	292✔
64	->addOption(	584✔
65	'pm-start-delay',	292✔
66	null,	292✔
67	InputOption::VALUE_REQUIRED,	292✔
68	'Process Manager. The time to delay the start of processes to space them out, in milliseconds',	292✔
69	self::PM_DEFAULT_START_DELAY,	292✔
70	)	292✔
71	->addOption(	584✔
72	'pm-max-timeout',	292✔
73	null,	292✔
74	InputOption::VALUE_REQUIRED,	292✔
75	'Process Manager. The max timeout for each proccess, in seconds',	292✔
76	self::PM_DEFAULT_TIMEOUT,	292✔
77	)	292✔
78	->addOption(	584✔
79	'pm-proc-id',	292✔
80	null,	292✔
81	InputOption::VALUE_REQUIRED,	292✔
82	'Process Manager. Unique ID of process to execute one child proccess.',	292✔
83	'',	292✔
84	);	292✔
85
86	parent::configure();	584✔
87	}
88
89	/**
90	* @phan-suppress PhanPluginPossiblyStaticProtectedMethod
91	*/
92	protected function beforeStartAllProcesses(): void
93	{
94	// noop
95	}	8✔
96
97	/**
98	* @SuppressWarnings(PHPMD.UnusedFormalParameter)
99	* @phan-suppress PhanPluginPossiblyStaticProtectedMethod
100	* @phan-suppress PhanUnusedProtectedNoOverrideMethodParameter
101	*/
102	protected function afterFinishAllProcesses(array $procPool): void
103	{
104	// noop
UNCOV 105	}
106
107	protected function executeAction(): int
108	{
109	$pmProcId = $this->getOptString('pm-proc-id');	96✔
110	if ($pmProcId !== '') {	96✔
111	return $this->executeOneProcess($pmProcId);	80✔
112	}
113
114	return $this->executeMultiProcessAction();	16✔
115	}
116
117	protected function executeMultiProcessAction(): int
118	{
119	$procNum = $this->getNumberOfProcesses();	16✔
120	$cpuCores = CliHelper::getNumberOfCpuCores();	16✔
121	$this->_("Max number of sub-processes: {$procNum}", OutLvl::DEBUG);	16✔
122	if ($procNum > $cpuCores) {	16✔
123	$this->_(	16✔
124	"The specified number of processes (--pm-max={$procNum}) "	16✔
125	. "is more than the found number of CPU cores in the system ({$cpuCores}).",	16✔
126	OutLvl::WARNING,	8✔
127	);	8✔
128	}
129
130	$procManager = $this->initProcManager($procNum, $this->gePmInterval(), $this->getPmStartDelay());	16✔
131
132	$procListIds = $this->getListOfChildIds();	16✔
133
134	if (!$this->outputMode->isProgressBarDisabled()) {	16✔
135	$this->progressBar = new ProgressBarProcessManager($this->outputMode);	×
136	$this->progressBar->setMax(\count($procListIds));	×
137	$this->progressBar->start();	×
138	}
139
140	foreach ($procListIds as $procListId) {	16✔
141	$childProcess = $this->createSubProcess($procListId);	16✔
142	$procManager->addProcess($childProcess);	16✔
143	}
144
145	$this->beforeStartAllProcesses();	16✔
146	$procManager->waitForAllProcesses();	16✔
147	if ($this->progressBar !== null) {	16✔
148	$this->progressBar->finish();	×
149	$this->_('');	×
150	}
151
152	$this->afterFinishAllProcesses($this->procPool);	16✔
153
154	$errorList = $this->getErrorList();	16✔
155	if (\count($errorList) > 0) {	16✔
156	throw new Exception(\implode("\n" . \str_repeat('-', 60) . "\n", $errorList));	4✔
157	}
158
159	$warningList = $this->getWarningList();	12✔
160	if (\count($warningList) > 0) {	12✔
161	$this->_(\implode("\n" . \str_repeat('-', 60) . "\n", $warningList), OutLvl::WARNING);	×
162	}
163
164	return 0;	12✔
165	}
166
167	private function initProcManager(
168	int $numberOfParallelProcesses,
169	int $pollInterval,
170	int $processStartDelay,
171	): ProcessManager {
172	$finishCallback = function (Process $process): void {	16✔
173	$virtProcId = \spl_object_id($process);	16✔
174
175	$exitCode = $process->getExitCode();	16✔
176	$errorOutput = \trim($process->getErrorOutput());	16✔
177	$stdOutput = \trim($process->getOutput());	16✔
178
179	$this->procPool[$virtProcId]['time_end'] = \microtime(true);	16✔
180	$this->procPool[$virtProcId]['exit_code'] = $exitCode;	16✔
181	$this->procPool[$virtProcId]['std_out'] = $stdOutput;	16✔
182
183	if ($exitCode > 0 \|\| $errorOutput !== '') {	16✔
184	$this->procPool[$virtProcId]['err_out'] = $errorOutput;	4✔
185	}
186
187	if ($this->progressBar !== null) {	16✔
188	$this->progressBar->advance();	×
189	}
190	};	8✔
191
192	return (new ProcessManager())	16✔
193	->setPollInterval($pollInterval)	16✔
194	->setNumberOfParallelProcesses($numberOfParallelProcesses)	16✔
195	->setProcessStartDelay($processStartDelay)	16✔
196	->setProcessStartCallback(function (Process $process): void {	16✔
197	$virtProcId = \spl_object_id($process);	16✔
198	$this->procPool[$virtProcId]['time_start'] = \microtime(true);	16✔
199	})	8✔
200	->setProcessFinishCallback($finishCallback)	16✔
201	->setProcessTimeoutCallback(function (Process $process) use ($finishCallback): void {	16✔
202	$finishCallback($process);	×
203
204	$virtProcId = \spl_object_id($process);	×
205	$this->procPool[$virtProcId]['reached_timeout'] = true;	×
206	});	8✔
207	}
208
209	private function createSubProcess(string $procId): Process
210	{
211	// Prepare option list from the parent process
212	$options = \array_filter(	16✔
213	$this->outputMode->getInput()->getOptions(),	16✔
214	static fn ($optionValue): bool => $optionValue !== false && $optionValue !== '',	16✔
215	);	8✔
216
217	foreach (\array_keys($options) as $optionKey) {	16✔
218	if (!$this->getDefinition()->getOption((string)$optionKey)->acceptValue()) {	16✔
219	$options[$optionKey] = null;	16✔
220	}
221	}
222
223	unset($options['ansi']);	16✔
224	$options['no-ansi'] = null;	16✔
225	$options['no-interaction'] = null;	16✔
226	$options['pm-proc-id'] = $procId;	16✔
227
228	// Prepare $argument list from the parent process
229	$arguments = $this->outputMode->getInput()->getArguments();	16✔
230	$argumentsList = [];	16✔
231
232	foreach ($arguments as $argKey => $argValue) {	16✔
233	if (\is_array($argValue)) {	16✔
234	continue;	×
235	}
236
237	/** @var string $argValue */
238	if ($argKey !== 'command') {	16✔
239	/** @phan-suppress-next-line PhanPartialTypeMismatchArgumentInternal */
240	$argumentsList[] = '"' . \addcslashes($argValue, '"') . '"';	16✔
241	}
242	}
243
244	// Build full command line
245	$process = Process::fromShellCommandline(	16✔
246	CliUtils::build(	16✔
247	\implode(	16✔
248	' ',	8✔
249	\array_filter([	16✔
250	Sys::getBinary(),	16✔
251	CliHelper::getBinPath(),	16✔
252	$this->getName(),	16✔
253	\implode(' ', $argumentsList),	16✔
254	]),	8✔
255	),	8✔
256	$options,	8✔
257	),	8✔
258	CliHelper::getRootPath(),	16✔
259	null,	8✔
260	null,	8✔
261	$this->getMaxTimeout(),	16✔
262	);	8✔
263
264	$this->procPool[\spl_object_id($process)] = [	16✔
265	'command' => $process->getCommandLine(),	16✔
266	'proc_id' => $procId,	8✔
267	'exit_code' => null,	8✔
268	'std_out' => null,	8✔
269	'err_out' => null,	8✔
270	'reached_timeout' => false,	8✔
271	'time_start' => null,	8✔
272	'time_end' => null,	8✔
273	];	8✔
274
275	return $process;	16✔
276	}
277
278	private function getErrorList(): array
279	{
280	return \array_reduce($this->procPool, function (array $acc, array $procInfo): array {	16✔
281	if ($procInfo['reached_timeout']) {	16✔
282	$acc[] = \implode("\n", [	×
283	"Command : {$procInfo['command']}",	×
284	"Error : The process with ID \"{$procInfo['proc_id']}\""	×
285	. " exceeded the timeout of {$this->getMaxTimeout()} seconds.",	×
UNCOV 286	]);
287	} elseif ($procInfo['err_out'] && $procInfo['exit_code'] > 0) {	16✔
288	$acc[] = \implode("\n", [	4✔
289	"Command : {$procInfo['command']}",	4✔
290	"Code : {$procInfo['exit_code']}",	4✔
291	"Error : {$procInfo['err_out']}",	4✔
292	"StdOut : {$procInfo['std_out']}",	4✔
293	]);	2✔
294	}
295
296	return $acc;	16✔
297	}, []);	8✔
298	}
299
300	private function getWarningList(): array
301	{
302	return \array_reduce($this->procPool, static function (array $acc, array $procInfo): array {	12✔
303	if ($procInfo['err_out'] && $procInfo['exit_code'] === 0) {	12✔
304	$acc[] = \implode("\n", [	×
305	"Command : {$procInfo['command']}",	×
306	"Warning : {$procInfo['err_out']}",	×
307	"StdOut : {$procInfo['std_out']}",	×
UNCOV 308	]);
309	}
310
311	return $acc;	12✔
312	}, []);	6✔
313	}
314
315	private function getMaxTimeout(): int
316	{
317	$pmMaxTimeout = $this->getOptInt('pm-max-timeout');	16✔
318
319	return $pmMaxTimeout > 0 ? $pmMaxTimeout : self::PM_DEFAULT_TIMEOUT;	16✔
320	}
321
322	private function gePmInterval(): int
323	{
324	$pmInterval = $this->getOptInt('pm-interval');	16✔
325
326	return $pmInterval > 0 ? $pmInterval : self::PM_DEFAULT_INTERVAL;	16✔
327	}
328
329	private function getPmStartDelay(): int
330	{
331	$pmStartDelay = $this->getOptInt('pm-start-delay');	16✔
332
333	return $pmStartDelay > 0 ? $pmStartDelay : self::PM_DEFAULT_START_DELAY;	16✔
334	}
335
336	private function getNumberOfProcesses(): int
337	{
338	$pmMax = \strtolower($this->getOptString('pm-max'));	16✔
339	$cpuCores = CliHelper::getNumberOfCpuCores();	16✔
340
341	if ($pmMax === 'auto') {	16✔
342	return $cpuCores;	×
343	}
344
345	$pmMaxInt = \abs(int($pmMax));	16✔
346
347	return $pmMaxInt > 0 ? $pmMaxInt : $cpuCores;	16✔
348	}
349	}

JBZoo / Cli / 7392476149

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous