• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

Cecilapp / Cecil / 21911121390

11 Feb 2026 03:24PM UTC coverage: 77.135% (-5.2%) from 82.346%
21911121390

push

github

web-flow
Parallelize page conversion using pcntl (#2313)

Add optional parallel processing to the Convert step by forking worker processes with pcntl_fork. The change: detect pcntl availability and run processParallel(), otherwise fall back to the original sequential processing (processSequential()). Each child converts a chunk of pages and writes serializable results to a temp file; the parent waits for children, reads results, applies front-matter/HTML updates, and removes pages on errors. Concurrency is chosen based on CPU count (getConcurrency). Child processes silence logging with a NullLogger to avoid output conflicts. Also introduce helper methods convertChunk, applyChunkResults and getConcurrency, and add a NullLogger import.

---------

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: ArnaudLigny <80580+ArnaudLigny@users.noreply.github.com>

81 of 132 new or added lines in 4 files covered. (61.36%)

190 existing lines in 7 files now uncovered.

3198 of 4146 relevant lines covered (77.13%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

46.54
/src/Step/Pages/Convert.php
1
<?php
2

3
/**
4
 * This file is part of Cecil.
5
 *
6
 * (c) Arnaud Ligny <arnaud@ligny.fr>
7
 *
8
 * For the full copyright and license information, please view the LICENSE
9
 * file that was distributed with this source code.
10
 */
11

12
declare(strict_types=1);
13

14
namespace Cecil\Step\Pages;
15

16
use Cecil\Builder;
17
use Cecil\Collection\Page\Page;
18
use Cecil\Converter\Converter;
19
use Cecil\Converter\ConverterInterface;
20
use Cecil\Exception\RuntimeException;
21
use Cecil\Step\AbstractStep;
22
use Cecil\Util;
23
use Psr\Log\NullLogger;
24

25
/**
26
 * Convert step.
27
 *
28
 * This step is responsible for converting pages from their source format
29
 * (i.e. Markdown) to HTML, applying front matter processing,
30
 * and ensuring that the pages are ready for rendering. It handles both
31
 * published and draft pages, depending on the build options.
32
 *
33
 * When the `pcntl` extension is available, page conversions are parallelized
34
 * using pcntl_fork for better performance. Each child process inherits the
35
 * full Builder context (pages collection, config, etc.) via copy-on-write.
36
 */
37
class Convert extends AbstractStep
38
{
39
    /**
40
     * {@inheritdoc}
41
     */
42
    public function getName(): string
43
    {
44
        if ($this->builder->getBuildOptions()['drafts']) {
1✔
45
            return 'Converting pages (drafts included)';
1✔
46
        }
47

48
        return 'Converting pages';
×
49
    }
50

51
    /**
52
     * {@inheritdoc}
53
     */
54
    public function init(array $options): void
55
    {
56
        parent::init($options);
1✔
57

58
        if (\is_null($this->builder->getPages())) {
1✔
59
            $this->canProcess = false;
×
60
        }
61
    }
62

63
    /**
64
     * {@inheritdoc}
65
     */
66
    public function process(): void
67
    {
68
        if (!is_iterable($this->builder->getPages()) || \count($this->builder->getPages()) == 0) {
1✔
69
            return;
×
70
        }
71

72
        // Use parallel processing if pcntl is available, otherwise fall back to sequential
73
        if (\function_exists('pcntl_fork')) {
1✔
74
            $this->builder->getLogger()->debug('Starting page conversion using parallel processing');
1✔
75
            $this->processParallel();
1✔
76

77
            return;
1✔
78
        }
79

NEW
80
        $this->builder->getLogger()->debug('Starting page conversion using sequential processing (pcntl extension not available)');
×
NEW
81
        $this->processSequential();
×
82
    }
83

84
    /**
85
     * Processes pages sequentially (original behavior).
86
     */
87
    protected function processSequential(): void
88
    {
UNCOV
89
        $total = \count($this->builder->getPages());
×
UNCOV
90
        $count = 0;
×
91
        /** @var Page $page */
UNCOV
92
        foreach ($this->builder->getPages() as $page) {
×
UNCOV
93
            if (!$page->isVirtual()) {
×
UNCOV
94
                $count++;
×
95

96
                try {
UNCOV
97
                    $convertedPage = $this->convertPage($this->builder, $page);
×
98
                    // set default language (ex: "en") if necessary
UNCOV
99
                    if ($convertedPage->getVariable('language') === null) {
×
UNCOV
100
                        $convertedPage->setVariable('language', $this->config->getLanguageDefault());
×
101
                    }
UNCOV
102
                } catch (RuntimeException $e) {
×
UNCOV
103
                    $this->builder->getLogger()->error(\sprintf('Unable to convert "%s:%s": %s', $e->getFile(), $e->getLine(), $e->getMessage()));
×
UNCOV
104
                    $this->builder->getPages()->remove($page->getId());
×
UNCOV
105
                    continue;
×
106
                } catch (\Exception $e) {
×
107
                    $this->builder->getLogger()->error(\sprintf('Unable to convert "%s": %s', Util::joinPath(Util\File::getFS()->makePathRelative($page->getFilePath(), $this->config->getPagesPath())), $e->getMessage()));
×
108
                    $this->builder->getPages()->remove($page->getId());
×
109
                    continue;
×
110
                }
UNCOV
111
                $message = \sprintf('Page "%s" converted', $page->getId());
×
UNCOV
112
                $statusMessage = ' (not published)';
×
113
                // forces drafts convert?
UNCOV
114
                if ($this->builder->getBuildOptions()['drafts']) {
×
UNCOV
115
                    $page->setVariable('published', true);
×
116
                }
117
                // replaces page in collection
UNCOV
118
                if ($page->getVariable('published')) {
×
UNCOV
119
                    $this->builder->getPages()->replace($page->getId(), $convertedPage);
×
UNCOV
120
                    $statusMessage = '';
×
121
                }
UNCOV
122
                $this->builder->getLogger()->info($message . $statusMessage, ['progress' => [$count, $total]]);
×
123
            }
124
        }
125
    }
126

127
    /**
128
     * Processes pages in parallel using pcntl_fork.
129
     *
130
     * Each child process inherits the parent's memory (copy-on-write), so the full Builder context (pages collection, config, converters, etc.) is available.
131
     * Only the conversion results (variables array + HTML string) are serialized back to the parent via temporary files.
132
     */
133
    protected function processParallel(): void
134
    {
135
        // Collect non-virtual pages
136
        $pages = [];
1✔
137
        /** @var Page $page */
138
        foreach ($this->builder->getPages() as $page) {
1✔
139
            if (!$page->isVirtual()) {
1✔
140
                $pages[] = $page;
1✔
141
            }
142
        }
143

144
        $total = \count($pages);
1✔
145
        if ($total === 0) {
1✔
NEW
146
            return;
×
147
        }
148

149
        $concurrency = $this->getConcurrency($total);
1✔
150
        $this->builder->getLogger()->debug(\sprintf('Using concurrency level: %d (total pages: %d)', $concurrency, $total));
1✔
151
        $chunks = array_chunk($pages, max(1, (int) ceil($total / $concurrency)));
1✔
152
        $format = (string) $this->builder->getConfig()->get('pages.frontmatter');
1✔
153
        $drafts = (bool) $this->options['drafts'];
1✔
154

155
        $children = [];
1✔
156

157
        foreach ($chunks as $chunk) {
1✔
158
            $tmpFile = tempnam(sys_get_temp_dir(), 'cecil_convert_');
1✔
159
            if ($tmpFile === false) {
1✔
NEW
160
                throw new RuntimeException('Unable to create temporary file for pages conversion.');
×
161
            }
162

163
            $pid = pcntl_fork();
1✔
164

165
            if ($pid === -1) {
1✔
166
                // Fork failed: convert this chunk sequentially in the parent
NEW
167
                $this->convertChunk($chunk, $format, $drafts, $tmpFile);
×
NEW
168
                $children[] = ['pid' => null, 'tmpFile' => $tmpFile, 'pages' => $chunk];
×
NEW
169
                continue;
×
170
            }
171

172
            if ($pid === 0) {
1✔
173
                // Child process
174
                // Silence logger to avoid output conflicts with parent
NEW
175
                $this->builder->setLogger(new NullLogger());
×
176

NEW
177
                $this->convertChunk($chunk, $format, $drafts, $tmpFile);
×
178

179
                // Exit child without running parent's shutdown handlers
NEW
180
                exit(0);
×
181
            }
182

183
            // Parent process
184
            $children[] = ['pid' => $pid, 'tmpFile' => $tmpFile, 'pages' => $chunk];
1✔
185
        }
186

187
        // Wait for all child processes and apply results
188
        $count = 0;
1✔
189
        foreach ($children as $child) {
1✔
190
            if ($child['pid'] !== null) {
1✔
191
                pcntl_waitpid($child['pid'], $status);
1✔
192
            }
193
            $this->builder->getLogger()->debug(\sprintf('Applying conversion results from child process (PID: %s, status: %s)', $child['pid'] ?? 'N/A', $status ?? 'N/A'));
1✔
194
            $count = $this->applyChunkResults($child['pages'], $child['tmpFile'], $total, $count);
1✔
195
        }
196
    }
197

198
    /**
199
     * Converts a chunk of pages using the existing Builder and writes
200
     * results to a temporary file.
201
     *
202
     * Each page's conversion output (front matter variables + HTML body) is
203
     * stored as serializable primitives (arrays, strings).
204
     *
205
     * @param Page[] $pages
206
     */
207
    protected function convertChunk(array $pages, string $format, bool $drafts, string $tmpFile): void
208
    {
NEW
209
        $converter = new Converter($this->builder);
×
NEW
210
        $results = [];
×
211

NEW
212
        foreach ($pages as $page) {
×
NEW
213
            $pageId = $page->getId();
×
214

215
            try {
NEW
216
                $variables = null;
×
NEW
217
                $html = null;
×
218

219
                // Convert front matter
NEW
220
                if ($page->getFrontmatter()) {
×
NEW
221
                    $variables = $converter->convertFrontmatter($page->getFrontmatter(), $format);
×
222
                }
223

224
                // Determine effective published status by applying variables to a cloned page
225
                // This ensures side effects (e.g., schedule logic) are applied, matching convertPage() behavior
NEW
226
                $published = (bool) $page->getVariable('published');
×
NEW
227
                if ($variables !== null) {
×
NEW
228
                    $tempPage = clone $page;
×
NEW
229
                    $tempPage->setVariables($variables);
×
NEW
230
                    $published = (bool) $tempPage->getVariable('published');
×
231
                }
232

233
                // Convert body (only if page is published or drafts option is enabled)
NEW
234
                if ($published || $drafts) {
×
NEW
235
                    $html = $converter->convertBody($page->getBody() ?? '');
×
236
                }
237

NEW
238
                $results[$pageId] = [
×
NEW
239
                    'success' => true,
×
NEW
240
                    'variables' => $variables,
×
NEW
241
                    'html' => $html,
×
NEW
242
                ];
×
NEW
243
            } catch (\Throwable $e) {
×
NEW
244
                $results[$pageId] = [
×
NEW
245
                    'success' => false,
×
NEW
246
                    'error' => $e->getMessage(),
×
NEW
247
                ];
×
248
            }
249
        }
250

NEW
251
        file_put_contents($tmpFile, serialize($results));
×
252
    }
253

254
    /**
255
     * Reads conversion results from a temporary file and applies them
256
     * to the original Page objects in the parent process.
257
     *
258
     * @param Page[] $pages
259
     *
260
     * @return int Updated progress count
261
     */
262
    protected function applyChunkResults(array $pages, string $tmpFile, int $total, int $count): int
263
    {
264
        $results = [];
1✔
265

266
        if (file_exists($tmpFile)) {
1✔
267
            $data = file_get_contents($tmpFile);
1✔
268
            if ($data !== false && $data !== '') {
1✔
269
                $unserialized = @unserialize($data, ['allowed_classes' => [\DateTimeImmutable::class]]);
1✔
270
                if (\is_array($unserialized)) {
1✔
271
                    $results = $unserialized;
1✔
272
                } else {
NEW
273
                    $this->builder->getLogger()->warning(\sprintf('Invalid conversion results in temporary file "%s". Ignoring.', $tmpFile));
×
274
                }
275
            }
276
            @unlink($tmpFile);
1✔
277
        }
278

279
        foreach ($pages as $page) {
1✔
280
            $pageId = $page->getId();
1✔
281
            $count++;
1✔
282

283
            // If results are missing or conversion failed, remove the page
284
            if (!isset($results[$pageId]) || !$results[$pageId]['success']) {
1✔
285
                $error = $results[$pageId]['error'] ?? 'Unknown error';
1✔
286
                $this->builder->getLogger()->error(\sprintf('Unable to convert "%s": %s', $pageId, $error));
1✔
287
                $this->builder->getPages()->remove($pageId);
1✔
288
                continue;
1✔
289
            }
290

291
            $result = $results[$pageId];
1✔
292

293
            // Apply front matter variables
294
            if ($result['variables'] !== null) {
1✔
295
                $page->setFmVariables($result['variables']);
1✔
296
                $page->setVariables($result['variables']);
1✔
297
            }
298

299
            // Apply converted HTML body
300
            if ($result['html'] !== null) {
1✔
301
                $page->setBodyHtml($result['html']);
1✔
302
            }
303

304
            // Set default language if necessary
305
            if ($page->getVariable('language') === null) {
1✔
306
                $page->setVariable('language', $this->config->getLanguageDefault());
1✔
307
            }
308

309
            $message = \sprintf('Page "%s" converted', $pageId);
1✔
310
            $statusMessage = ' (not published)';
1✔
311

312
            // Forces drafts convert?
313
            if ($this->builder->getBuildOptions()['drafts']) {
1✔
314
                $page->setVariable('published', true);
1✔
315
            }
316

317
            // Replaces page in collection
318
            if ($page->getVariable('published')) {
1✔
319
                $this->builder->getPages()->replace($pageId, $page);
1✔
320
                $statusMessage = '';
1✔
321
            }
322

323
            $this->builder->getLogger()->info($message . $statusMessage, ['progress' => [$count, $total]]);
1✔
324
        }
325

326
        return $count;
1✔
327
    }
328

329
    /**
330
     * Determines the optimal concurrency level based on CPU count.
331
     */
332
    protected function getConcurrency(int $totalPages): int
333
    {
334
        $cpuCount = 4; // sensible default
1✔
335

336
        if (PHP_OS_FAMILY === 'Darwin') {
1✔
NEW
337
            $result = @shell_exec('sysctl -n hw.ncpu 2>/dev/null');
×
NEW
338
            if (\is_string($result)) {
×
NEW
339
                $cpuCount = max(1, (int) trim($result));
×
340
            }
341
        } elseif (is_file('/proc/cpuinfo')) {
1✔
342
            $content = @file_get_contents('/proc/cpuinfo');
1✔
343
            if ($content !== false) {
1✔
344
                $cpuCount = max(1, substr_count($content, 'processor'));
1✔
345
            }
NEW
346
        } elseif (PHP_OS_FAMILY === 'Windows') {
×
NEW
347
            $result = @shell_exec('echo %NUMBER_OF_PROCESSORS% 2>NUL');
×
NEW
348
            if (\is_string($result)) {
×
NEW
349
                $cpuCount = max(1, (int) trim($result));
×
350
            }
351
        }
352

353
        return min($cpuCount, $totalPages);
1✔
354
    }
355

356
    /**
357
     * Converts page content:
358
     *  - front matter to PHP array
359
     *  - body to HTML.
360
     *
361
     * @throws RuntimeException
362
     */
363
    public function convertPage(Builder $builder, Page $page, ?string $format = null, ?ConverterInterface $converter = null): Page
364
    {
UNCOV
365
        $format = $format ?? (string) $builder->getConfig()->get('pages.frontmatter');
×
UNCOV
366
        $converter = $converter ?? new Converter($builder);
×
367

368
        // converts front matter
UNCOV
369
        if ($page->getFrontmatter()) {
×
370
            try {
UNCOV
371
                $variables = $converter->convertFrontmatter($page->getFrontmatter(), $format);
×
UNCOV
372
            } catch (RuntimeException $e) {
×
UNCOV
373
                throw new RuntimeException($e->getMessage(), file: $page->getFilePath(), line: $e->getLine());
×
374
            }
UNCOV
375
            $page->setFmVariables($variables);
×
UNCOV
376
            $page->setVariables($variables);
×
377
        }
378

379
        // converts body (only if page is published or drafts option is enabled)
UNCOV
380
        if ($page->getVariable('published') || $this->options['drafts']) {
×
381
            try {
UNCOV
382
                $html = $converter->convertBody($page->getBody());
×
383
            } catch (RuntimeException $e) {
×
NEW
384
                throw new RuntimeException($e->getMessage(), file: $page->getFilePath(), line: $e->getLine());
×
385
            }
UNCOV
386
            $page->setBodyHtml($html);
×
387
        }
388

UNCOV
389
        return $page;
×
390
    }
391
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc