• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

Cecilapp / Cecil / 21900210283

11 Feb 2026 09:48AM UTC coverage: 77.124%. First build
21900210283

Pull #2313

github

web-flow
Merge 8a1111676 into ef8bfe522
Pull Request #2313: Parallelize page conversion using pcntl

79 of 129 new or added lines in 4 files covered. (61.24%)

3196 of 4144 relevant lines covered (77.12%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.86
/src/Step/Pages/Convert.php
1
<?php
2

3
/**
4
 * This file is part of Cecil.
5
 *
6
 * (c) Arnaud Ligny <arnaud@ligny.fr>
7
 *
8
 * For the full copyright and license information, please view the LICENSE
9
 * file that was distributed with this source code.
10
 */
11

12
declare(strict_types=1);
13

14
namespace Cecil\Step\Pages;
15

16
use Cecil\Builder;
17
use Cecil\Collection\Page\Page;
18
use Cecil\Converter\Converter;
19
use Cecil\Converter\ConverterInterface;
20
use Cecil\Exception\RuntimeException;
21
use Cecil\Step\AbstractStep;
22
use Cecil\Util;
23
use Psr\Log\NullLogger;
24

25
/**
26
 * Convert step.
27
 *
28
 * This step is responsible for converting pages from their source format
29
 * (i.e. Markdown) to HTML, applying front matter processing,
30
 * and ensuring that the pages are ready for rendering. It handles both
31
 * published and draft pages, depending on the build options.
32
 *
33
 * When the `pcntl` extension is available, page conversions are parallelized
34
 * using pcntl_fork for better performance. Each child process inherits the
35
 * full Builder context (pages collection, config, etc.) via copy-on-write.
36
 */
37
class Convert extends AbstractStep
38
{
39
    /**
40
     * {@inheritdoc}
41
     */
42
    public function getName(): string
43
    {
44
        if ($this->builder->getBuildOptions()['drafts']) {
1✔
45
            return 'Converting pages (drafts included)';
1✔
46
        }
47

48
        return 'Converting pages';
×
49
    }
50

51
    /**
52
     * {@inheritdoc}
53
     */
54
    public function init(array $options): void
55
    {
56
        parent::init($options);
1✔
57

58
        if (\is_null($this->builder->getPages())) {
1✔
59
            $this->canProcess = false;
×
60
        }
61
    }
62

63
    /**
64
     * {@inheritdoc}
65
     */
66
    public function process(): void
67
    {
68
        if (!is_iterable($this->builder->getPages()) || \count($this->builder->getPages()) == 0) {
1✔
69
            return;
×
70
        }
71

72
        // Use parallel processing if pcntl is available, otherwise fall back to sequential
73
        if (\function_exists('pcntl_fork')) {
1✔
74
            $this->builder->getLogger()->debug('Starting page conversion using parallel processing');
1✔
75
            $this->processParallel();
1✔
76
        } else {
NEW
77
            $this->builder->getLogger()->debug('Starting page conversion using sequential processing (pcntl extension not available)');
×
NEW
78
            $this->processSequential();
×
79
        }
80
    }
81

82
    /**
83
     * Processes pages sequentially (original behavior).
84
     */
85
    protected function processSequential(): void
86
    {
87
        $total = \count($this->builder->getPages());
×
88
        $count = 0;
×
89
        /** @var Page $page */
90
        foreach ($this->builder->getPages() as $page) {
×
91
            if (!$page->isVirtual()) {
×
92
                $count++;
×
93

94
                try {
95
                    $convertedPage = $this->convertPage($this->builder, $page);
×
96
                    // set default language (ex: "en") if necessary
97
                    if ($convertedPage->getVariable('language') === null) {
×
98
                        $convertedPage->setVariable('language', $this->config->getLanguageDefault());
×
99
                    }
100
                } catch (RuntimeException $e) {
×
101
                    $this->builder->getLogger()->error(\sprintf('Unable to convert "%s:%s": %s', $e->getFile(), $e->getLine(), $e->getMessage()));
×
102
                    $this->builder->getPages()->remove($page->getId());
×
103
                    continue;
×
104
                } catch (\Exception $e) {
×
105
                    $this->builder->getLogger()->error(\sprintf('Unable to convert "%s": %s', Util::joinPath(Util\File::getFS()->makePathRelative($page->getFilePath(), $this->config->getPagesPath())), $e->getMessage()));
×
106
                    $this->builder->getPages()->remove($page->getId());
×
107
                    continue;
×
108
                }
109
                $message = \sprintf('Page "%s" converted', $page->getId());
×
110
                $statusMessage = ' (not published)';
×
111
                // forces drafts convert?
112
                if ($this->builder->getBuildOptions()['drafts']) {
×
113
                    $page->setVariable('published', true);
×
114
                }
115
                // replaces page in collection
116
                if ($page->getVariable('published')) {
×
117
                    $this->builder->getPages()->replace($page->getId(), $convertedPage);
×
118
                    $statusMessage = '';
×
119
                }
120
                $this->builder->getLogger()->info($message . $statusMessage, ['progress' => [$count, $total]]);
×
121
            }
122
        }
123
    }
124

125
    /**
126
     * Processes pages in parallel using pcntl_fork.
127
     *
128
     * Each child process inherits the parent's memory (copy-on-write), so the full Builder context (pages collection, config, converters, etc.) is available.
129
     * Only the conversion results (variables array + HTML string) are serialized back to the parent via temporary files.
130
     */
131
    protected function processParallel(): void
132
    {
133
        // Collect non-virtual pages
134
        $pages = [];
1✔
135
        /** @var Page $page */
136
        foreach ($this->builder->getPages() as $page) {
1✔
137
            if (!$page->isVirtual()) {
1✔
138
                $pages[] = $page;
1✔
139
            }
140
        }
141

142
        $total = \count($pages);
1✔
143
        if ($total === 0) {
1✔
NEW
144
            return;
×
145
        }
146

147
        $concurrency = $this->getConcurrency($total);
1✔
148
        $this->builder->getLogger()->debug(\sprintf('Using concurrency level: %d (total pages: %d)', $concurrency, $total));
1✔
149
        $chunks = \array_chunk($pages, max(1, (int) ceil($total / $concurrency)));
1✔
150
        $format = (string) $this->builder->getConfig()->get('pages.frontmatter');
1✔
151
        $drafts = (bool) $this->options['drafts'];
1✔
152

153
        $children = [];
1✔
154

155
        foreach ($chunks as $chunk) {
1✔
156
            $tmpFile = \tempnam(\sys_get_temp_dir(), 'cecil_convert_');
1✔
157
            if ($tmpFile === false) {
1✔
NEW
158
                throw new RuntimeException('Unable to create temporary file for pages conversion.');
×
159
            }
160

161
            $pid = \pcntl_fork();
1✔
162

163
            if ($pid === -1) {
1✔
164
                // Fork failed: convert this chunk sequentially in the parent
NEW
165
                $this->convertChunk($chunk, $format, $drafts, $tmpFile);
×
NEW
166
                $children[] = ['pid' => null, 'tmpFile' => $tmpFile, 'pages' => $chunk];
×
NEW
167
                continue;
×
168
            }
169

170
            if ($pid === 0) {
1✔
171
                // Child process
172
                // Silence logger to avoid output conflicts with parent
NEW
173
                $this->builder->setLogger(new NullLogger());
×
174

NEW
175
                $this->convertChunk($chunk, $format, $drafts, $tmpFile);
×
176

177
                // Exit child without running parent's shutdown handlers
NEW
178
                exit(0);
×
179
            }
180

181
            // Parent process
182
            $children[] = ['pid' => $pid, 'tmpFile' => $tmpFile, 'pages' => $chunk];
1✔
183
        }
184

185
        // Wait for all child processes and apply results
186
        $count = 0;
1✔
187
        foreach ($children as $child) {
1✔
188
            if ($child['pid'] !== null) {
1✔
189
                \pcntl_waitpid($child['pid'], $status);
1✔
190
            }
191
            $count = $this->applyChunkResults($child['pages'], $child['tmpFile'], $total, $count);
1✔
192
        }
193
    }
194

195
    /**
196
     * Converts a chunk of pages using the existing Builder and writes
197
     * results to a temporary file.
198
     *
199
     * Each page's conversion output (front matter variables + HTML body) is
200
     * stored as serializable primitives (arrays, strings).
201
     *
202
     * @param Page[] $pages
203
     */
204
    protected function convertChunk(array $pages, string $format, bool $drafts, string $tmpFile): void
205
    {
NEW
206
        $converter = new Converter($this->builder);
×
NEW
207
        $results = [];
×
208

NEW
209
        foreach ($pages as $page) {
×
NEW
210
            $pageId = $page->getId();
×
211

212
            try {
NEW
213
                $variables = null;
×
NEW
214
                $html = null;
×
215

216
                // Convert front matter
NEW
217
                if ($page->getFrontmatter()) {
×
NEW
218
                    $variables = $converter->convertFrontmatter($page->getFrontmatter(), $format);
×
219
                }
220

221
                // Determine effective published status by applying variables to a cloned page
222
                // This ensures side effects (e.g., schedule logic) are applied, matching convertPage() behavior
NEW
223
                $published = (bool) $page->getVariable('published');
×
NEW
224
                if ($variables !== null) {
×
NEW
225
                    $tempPage = clone $page;
×
NEW
226
                    $tempPage->setVariables($variables);
×
NEW
227
                    $published = (bool) $tempPage->getVariable('published');
×
228
                }
229

230
                // Convert body (only if page is published or drafts option is enabled)
NEW
231
                if ($published || $drafts) {
×
NEW
232
                    $html = $converter->convertBody($page->getBody() ?? '');
×
233
                }
234

NEW
235
                $results[$pageId] = [
×
NEW
236
                    'success' => true,
×
NEW
237
                    'variables' => $variables,
×
NEW
238
                    'html' => $html,
×
NEW
239
                ];
×
NEW
240
            } catch (\Throwable $e) {
×
NEW
241
                $results[$pageId] = [
×
NEW
242
                    'success' => false,
×
NEW
243
                    'error' => $e->getMessage(),
×
NEW
244
                ];
×
245
            }
246
        }
247

NEW
248
        \file_put_contents($tmpFile, \serialize($results));
×
249
    }
250

251
    /**
252
     * Reads conversion results from a temporary file and applies them
253
     * to the original Page objects in the parent process.
254
     *
255
     * @param Page[] $pages
256
     *
257
     * @return int Updated progress count
258
     */
259
    protected function applyChunkResults(array $pages, string $tmpFile, int $total, int $count): int
260
    {
261
        $results = [];
1✔
262

263
        if (\file_exists($tmpFile)) {
1✔
264
            $data = \file_get_contents($tmpFile);
1✔
265
            if ($data !== false && $data !== '') {
1✔
266
                $unserialized = @\unserialize($data, ['allowed_classes' => [\DateTimeImmutable::class]]);
1✔
267
                if (\is_array($unserialized)) {
1✔
268
                    $results = $unserialized;
1✔
269
                } else {
NEW
270
                    $this->builder->getLogger()->warning(\sprintf('Invalid conversion results in temporary file "%s". Ignoring.', $tmpFile));
×
271
                }
272
            }
273
            @\unlink($tmpFile);
1✔
274
        }
275

276
        foreach ($pages as $page) {
1✔
277
            $pageId = $page->getId();
1✔
278
            $count++;
1✔
279

280
            // If results are missing or conversion failed, remove the page
281
            if (!isset($results[$pageId]) || !$results[$pageId]['success']) {
1✔
282
                $error = $results[$pageId]['error'] ?? 'Unknown error';
1✔
283
                $this->builder->getLogger()->error(\sprintf('Unable to convert "%s": %s', $pageId, $error));
1✔
284
                $this->builder->getPages()->remove($pageId);
1✔
285
                continue;
1✔
286
            }
287

288
            $result = $results[$pageId];
1✔
289

290
            // Apply front matter variables
291
            if ($result['variables'] !== null) {
1✔
292
                $page->setFmVariables($result['variables']);
1✔
293
                $page->setVariables($result['variables']);
1✔
294
            }
295

296
            // Apply converted HTML body
297
            if ($result['html'] !== null) {
1✔
298
                $page->setBodyHtml($result['html']);
1✔
299
            }
300

301
            // Set default language if necessary
302
            if ($page->getVariable('language') === null) {
1✔
303
                $page->setVariable('language', $this->config->getLanguageDefault());
1✔
304
            }
305

306
            $message = \sprintf('Page "%s" converted (%s)', $pageId, $tmpFile);
1✔
307
            $statusMessage = ' (not published)';
1✔
308

309
            // Forces drafts convert?
310
            if ($this->builder->getBuildOptions()['drafts']) {
1✔
311
                $page->setVariable('published', true);
1✔
312
            }
313

314
            // Replaces page in collection
315
            if ($page->getVariable('published')) {
1✔
316
                $this->builder->getPages()->replace($pageId, $page);
1✔
317
                $statusMessage = '';
1✔
318
            }
319

320
            $this->builder->getLogger()->info($message . $statusMessage, ['progress' => [$count, $total]]);
1✔
321
        }
322

323
        return $count;
1✔
324
    }
325

326
    /**
327
     * Determines the optimal concurrency level based on CPU count.
328
     */
329
    protected function getConcurrency(int $totalPages): int
330
    {
331
        $cpuCount = 4; // sensible default
1✔
332

333
        if (PHP_OS_FAMILY === 'Darwin') {
1✔
NEW
334
            $result = @\shell_exec('sysctl -n hw.ncpu 2>/dev/null');
×
NEW
335
            if ($result !== null) {
×
NEW
336
                $cpuCount = max(1, (int) trim($result));
×
337
            }
338
        } elseif (\is_file('/proc/cpuinfo')) {
1✔
339
            $content = @\file_get_contents('/proc/cpuinfo');
1✔
340
            if ($content !== false) {
1✔
341
                $cpuCount = max(1, \substr_count($content, 'processor'));
1✔
342
            }
NEW
343
        } elseif (PHP_OS_FAMILY === 'Windows') {
×
NEW
344
            $result = @\shell_exec('echo %NUMBER_OF_PROCESSORS% 2>NUL');
×
NEW
345
            if ($result !== null) {
×
NEW
346
                $cpuCount = max(1, (int) trim($result));
×
347
            }
348
        }
349

350
        return min($cpuCount, $totalPages);
1✔
351
    }
352

353
    /**
354
     * Converts page content:
355
     *  - front matter to PHP array
356
     *  - body to HTML.
357
     *
358
     * @throws RuntimeException
359
     */
360
    public function convertPage(Builder $builder, Page $page, ?string $format = null, ?ConverterInterface $converter = null): Page
361
    {
362
        $format = $format ?? (string) $builder->getConfig()->get('pages.frontmatter');
×
363
        $converter = $converter ?? new Converter($builder);
×
364

365
        // converts front matter
366
        if ($page->getFrontmatter()) {
×
367
            try {
368
                $variables = $converter->convertFrontmatter($page->getFrontmatter(), $format);
×
369
            } catch (RuntimeException $e) {
×
370
                throw new RuntimeException($e->getMessage(), file: $page->getFilePath(), line: $e->getLine());
×
371
            }
372
            $page->setFmVariables($variables);
×
373
            $page->setVariables($variables);
×
374
        }
375

376
        // converts body (only if page is published or drafts option is enabled)
377
        if ($page->getVariable('published') || $this->options['drafts']) {
×
378
            try {
379
                $html = $converter->convertBody($page->getBody());
×
380
            } catch (RuntimeException $e) {
×
381
                throw new \Exception($e->getMessage());
×
382
            }
383
            $page->setBodyHtml($html);
×
384
        }
385

386
        return $page;
×
387
    }
388
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc