• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

Cecilapp / Cecil / 21908668128

11 Feb 2026 02:17PM UTC coverage: 77.129%. First build
21908668128

Pull #2313

github

web-flow
Merge 27cf6eed8 into ef8bfe522
Pull Request #2313: Parallelize page conversion using pcntl

80 of 130 new or added lines in 4 files covered. (61.54%)

3197 of 4145 relevant lines covered (77.13%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

46.2
/src/Step/Pages/Convert.php
1
<?php
2

3
/**
4
 * This file is part of Cecil.
5
 *
6
 * (c) Arnaud Ligny <arnaud@ligny.fr>
7
 *
8
 * For the full copyright and license information, please view the LICENSE
9
 * file that was distributed with this source code.
10
 */
11

12
declare(strict_types=1);
13

14
namespace Cecil\Step\Pages;
15

16
use Cecil\Builder;
17
use Cecil\Collection\Page\Page;
18
use Cecil\Converter\Converter;
19
use Cecil\Converter\ConverterInterface;
20
use Cecil\Exception\RuntimeException;
21
use Cecil\Step\AbstractStep;
22
use Cecil\Util;
23
use Psr\Log\NullLogger;
24

25
/**
26
 * Convert step.
27
 *
28
 * This step is responsible for converting pages from their source format
29
 * (i.e. Markdown) to HTML, applying front matter processing,
30
 * and ensuring that the pages are ready for rendering. It handles both
31
 * published and draft pages, depending on the build options.
32
 *
33
 * When the `pcntl` extension is available, page conversions are parallelized
34
 * using pcntl_fork for better performance. Each child process inherits the
35
 * full Builder context (pages collection, config, etc.) via copy-on-write.
36
 */
37
class Convert extends AbstractStep
38
{
39
    /**
40
     * {@inheritdoc}
41
     */
42
    public function getName(): string
43
    {
44
        if ($this->builder->getBuildOptions()['drafts']) {
1✔
45
            return 'Converting pages (drafts included)';
1✔
46
        }
47

48
        return 'Converting pages';
×
49
    }
50

51
    /**
52
     * {@inheritdoc}
53
     */
54
    public function init(array $options): void
55
    {
56
        parent::init($options);
1✔
57

58
        if (\is_null($this->builder->getPages())) {
1✔
59
            $this->canProcess = false;
×
60
        }
61
    }
62

63
    /**
64
     * {@inheritdoc}
65
     */
66
    public function process(): void
67
    {
68
        if (!is_iterable($this->builder->getPages()) || \count($this->builder->getPages()) == 0) {
1✔
69
            return;
×
70
        }
71

72
        // Use parallel processing if pcntl is available, otherwise fall back to sequential
73
        if (\function_exists('pcntl_fork')) {
1✔
74
            $this->builder->getLogger()->debug('Starting page conversion using parallel processing');
1✔
75
            $this->processParallel();
1✔
76

77
            return;
1✔
78
        }
79

NEW
80
        $this->builder->getLogger()->debug('Starting page conversion using sequential processing (pcntl extension not available)');
×
NEW
81
        $this->processSequential();
×
82
    }
83

84
    /**
85
     * Processes pages sequentially (original behavior).
86
     */
87
    protected function processSequential(): void
88
    {
89
        $total = \count($this->builder->getPages());
×
90
        $count = 0;
×
91
        /** @var Page $page */
92
        foreach ($this->builder->getPages() as $page) {
×
93
            if (!$page->isVirtual()) {
×
94
                $count++;
×
95

96
                try {
97
                    $convertedPage = $this->convertPage($this->builder, $page);
×
98
                    // set default language (ex: "en") if necessary
99
                    if ($convertedPage->getVariable('language') === null) {
×
100
                        $convertedPage->setVariable('language', $this->config->getLanguageDefault());
×
101
                    }
102
                } catch (RuntimeException $e) {
×
103
                    $this->builder->getLogger()->error(\sprintf('Unable to convert "%s:%s": %s', $e->getFile(), $e->getLine(), $e->getMessage()));
×
104
                    $this->builder->getPages()->remove($page->getId());
×
105
                    continue;
×
106
                } catch (\Exception $e) {
×
107
                    $this->builder->getLogger()->error(\sprintf('Unable to convert "%s": %s', Util::joinPath(Util\File::getFS()->makePathRelative($page->getFilePath(), $this->config->getPagesPath())), $e->getMessage()));
×
108
                    $this->builder->getPages()->remove($page->getId());
×
109
                    continue;
×
110
                }
111
                $message = \sprintf('Page "%s" converted', $page->getId());
×
112
                $statusMessage = ' (not published)';
×
113
                // forces drafts convert?
114
                if ($this->builder->getBuildOptions()['drafts']) {
×
115
                    $page->setVariable('published', true);
×
116
                }
117
                // replaces page in collection
118
                if ($page->getVariable('published')) {
×
119
                    $this->builder->getPages()->replace($page->getId(), $convertedPage);
×
120
                    $statusMessage = '';
×
121
                }
122
                $this->builder->getLogger()->info($message . $statusMessage, ['progress' => [$count, $total]]);
×
123
            }
124
        }
125
    }
126

127
    /**
128
     * Processes pages in parallel using pcntl_fork.
129
     *
130
     * Each child process inherits the parent's memory (copy-on-write), so the full Builder context (pages collection, config, converters, etc.) is available.
131
     * Only the conversion results (variables array + HTML string) are serialized back to the parent via temporary files.
132
     */
133
    protected function processParallel(): void
134
    {
135
        // Collect non-virtual pages
136
        $pages = [];
1✔
137
        /** @var Page $page */
138
        foreach ($this->builder->getPages() as $page) {
1✔
139
            if (!$page->isVirtual()) {
1✔
140
                $pages[] = $page;
1✔
141
            }
142
        }
143

144
        $total = \count($pages);
1✔
145
        if ($total === 0) {
1✔
NEW
146
            return;
×
147
        }
148

149
        $concurrency = $this->getConcurrency($total);
1✔
150
        $this->builder->getLogger()->debug(\sprintf('Using concurrency level: %d (total pages: %d)', $concurrency, $total));
1✔
151
        $chunks = \array_chunk($pages, max(1, (int) ceil($total / $concurrency)));
1✔
152
        $format = (string) $this->builder->getConfig()->get('pages.frontmatter');
1✔
153
        $drafts = (bool) $this->options['drafts'];
1✔
154

155
        $children = [];
1✔
156

157
        foreach ($chunks as $chunk) {
1✔
158
            $tmpFile = \tempnam(\sys_get_temp_dir(), 'cecil_convert_');
1✔
159
            if ($tmpFile === false) {
1✔
NEW
160
                throw new RuntimeException('Unable to create temporary file for pages conversion.');
×
161
            }
162

163
            $pid = \pcntl_fork();
1✔
164

165
            if ($pid === -1) {
1✔
166
                // Fork failed: convert this chunk sequentially in the parent
NEW
167
                $this->convertChunk($chunk, $format, $drafts, $tmpFile);
×
NEW
168
                $children[] = ['pid' => null, 'tmpFile' => $tmpFile, 'pages' => $chunk];
×
NEW
169
                continue;
×
170
            }
171

172
            if ($pid === 0) {
1✔
173
                // Child process
174
                // Silence logger to avoid output conflicts with parent
NEW
175
                $this->builder->setLogger(new NullLogger());
×
176

NEW
177
                $this->convertChunk($chunk, $format, $drafts, $tmpFile);
×
178

179
                // Exit child without running parent's shutdown handlers
NEW
180
                exit(0);
×
181
            }
182

183
            // Parent process
184
            $children[] = ['pid' => $pid, 'tmpFile' => $tmpFile, 'pages' => $chunk];
1✔
185
        }
186

187
        // Wait for all child processes and apply results
188
        $count = 0;
1✔
189
        foreach ($children as $child) {
1✔
190
            if ($child['pid'] !== null) {
1✔
191
                \pcntl_waitpid($child['pid'], $status);
1✔
192
            }
193
            $count = $this->applyChunkResults($child['pages'], $child['tmpFile'], $total, $count);
1✔
194
        }
195
    }
196

197
    /**
198
     * Converts a chunk of pages using the existing Builder and writes
199
     * results to a temporary file.
200
     *
201
     * Each page's conversion output (front matter variables + HTML body) is
202
     * stored as serializable primitives (arrays, strings).
203
     *
204
     * @param Page[] $pages
205
     */
206
    protected function convertChunk(array $pages, string $format, bool $drafts, string $tmpFile): void
207
    {
NEW
208
        $converter = new Converter($this->builder);
×
NEW
209
        $results = [];
×
210

NEW
211
        foreach ($pages as $page) {
×
NEW
212
            $pageId = $page->getId();
×
213

214
            try {
NEW
215
                $variables = null;
×
NEW
216
                $html = null;
×
217

218
                // Convert front matter
NEW
219
                if ($page->getFrontmatter()) {
×
NEW
220
                    $variables = $converter->convertFrontmatter($page->getFrontmatter(), $format);
×
221
                }
222

223
                // Determine effective published status by applying variables to a cloned page
224
                // This ensures side effects (e.g., schedule logic) are applied, matching convertPage() behavior
NEW
225
                $published = (bool) $page->getVariable('published');
×
NEW
226
                if ($variables !== null) {
×
NEW
227
                    $tempPage = clone $page;
×
NEW
228
                    $tempPage->setVariables($variables);
×
NEW
229
                    $published = (bool) $tempPage->getVariable('published');
×
230
                }
231

232
                // Convert body (only if page is published or drafts option is enabled)
NEW
233
                if ($published || $drafts) {
×
NEW
234
                    $html = $converter->convertBody($page->getBody() ?? '');
×
235
                }
236

NEW
237
                $results[$pageId] = [
×
NEW
238
                    'success' => true,
×
NEW
239
                    'variables' => $variables,
×
NEW
240
                    'html' => $html,
×
NEW
241
                ];
×
NEW
242
            } catch (\Throwable $e) {
×
NEW
243
                $results[$pageId] = [
×
NEW
244
                    'success' => false,
×
NEW
245
                    'error' => $e->getMessage(),
×
NEW
246
                ];
×
247
            }
248
        }
249

NEW
250
        \file_put_contents($tmpFile, \serialize($results));
×
251
    }
252

253
    /**
254
     * Reads conversion results from a temporary file and applies them
255
     * to the original Page objects in the parent process.
256
     *
257
     * @param Page[] $pages
258
     *
259
     * @return int Updated progress count
260
     */
261
    protected function applyChunkResults(array $pages, string $tmpFile, int $total, int $count): int
262
    {
263
        $results = [];
1✔
264

265
        if (\file_exists($tmpFile)) {
1✔
266
            $data = \file_get_contents($tmpFile);
1✔
267
            if ($data !== false && $data !== '') {
1✔
268
                $unserialized = @\unserialize($data, ['allowed_classes' => [\DateTimeImmutable::class]]);
1✔
269
                if (\is_array($unserialized)) {
1✔
270
                    $results = $unserialized;
1✔
271
                } else {
NEW
272
                    $this->builder->getLogger()->warning(\sprintf('Invalid conversion results in temporary file "%s". Ignoring.', $tmpFile));
×
273
                }
274
            }
275
            @\unlink($tmpFile);
1✔
276
        }
277

278
        foreach ($pages as $page) {
1✔
279
            $pageId = $page->getId();
1✔
280
            $count++;
1✔
281

282
            // If results are missing or conversion failed, remove the page
283
            if (!isset($results[$pageId]) || !$results[$pageId]['success']) {
1✔
284
                $error = $results[$pageId]['error'] ?? 'Unknown error';
1✔
285
                $this->builder->getLogger()->error(\sprintf('Unable to convert "%s": %s', $pageId, $error));
1✔
286
                $this->builder->getPages()->remove($pageId);
1✔
287
                continue;
1✔
288
            }
289

290
            $result = $results[$pageId];
1✔
291

292
            // Apply front matter variables
293
            if ($result['variables'] !== null) {
1✔
294
                $page->setFmVariables($result['variables']);
1✔
295
                $page->setVariables($result['variables']);
1✔
296
            }
297

298
            // Apply converted HTML body
299
            if ($result['html'] !== null) {
1✔
300
                $page->setBodyHtml($result['html']);
1✔
301
            }
302

303
            // Set default language if necessary
304
            if ($page->getVariable('language') === null) {
1✔
305
                $page->setVariable('language', $this->config->getLanguageDefault());
1✔
306
            }
307

308
            $message = \sprintf('Page "%s" converted (%s)', $pageId, $tmpFile);
1✔
309
            $statusMessage = ' (not published)';
1✔
310

311
            // Forces drafts convert?
312
            if ($this->builder->getBuildOptions()['drafts']) {
1✔
313
                $page->setVariable('published', true);
1✔
314
            }
315

316
            // Replaces page in collection
317
            if ($page->getVariable('published')) {
1✔
318
                $this->builder->getPages()->replace($pageId, $page);
1✔
319
                $statusMessage = '';
1✔
320
            }
321

322
            $this->builder->getLogger()->info($message . $statusMessage, ['progress' => [$count, $total]]);
1✔
323
        }
324

325
        return $count;
1✔
326
    }
327

328
    /**
329
     * Determines the optimal concurrency level based on CPU count.
330
     */
331
    protected function getConcurrency(int $totalPages): int
332
    {
333
        $cpuCount = 4; // sensible default
1✔
334

335
        if (PHP_OS_FAMILY === 'Darwin') {
1✔
NEW
336
            $result = @\shell_exec('sysctl -n hw.ncpu 2>/dev/null');
×
NEW
337
            if ($result !== null) {
×
NEW
338
                $cpuCount = max(1, (int) trim($result));
×
339
            }
340
        } elseif (\is_file('/proc/cpuinfo')) {
1✔
341
            $content = @\file_get_contents('/proc/cpuinfo');
1✔
342
            if ($content !== false) {
1✔
343
                $cpuCount = max(1, \substr_count($content, 'processor'));
1✔
344
            }
NEW
345
        } elseif (PHP_OS_FAMILY === 'Windows') {
×
NEW
346
            $result = @\shell_exec('echo %NUMBER_OF_PROCESSORS% 2>NUL');
×
NEW
347
            if ($result !== null) {
×
NEW
348
                $cpuCount = max(1, (int) trim($result));
×
349
            }
350
        }
351

352
        return min($cpuCount, $totalPages);
1✔
353
    }
354

355
    /**
356
     * Converts page content:
357
     *  - front matter to PHP array
358
     *  - body to HTML.
359
     *
360
     * @throws RuntimeException
361
     */
362
    public function convertPage(Builder $builder, Page $page, ?string $format = null, ?ConverterInterface $converter = null): Page
363
    {
364
        $format = $format ?? (string) $builder->getConfig()->get('pages.frontmatter');
×
365
        $converter = $converter ?? new Converter($builder);
×
366

367
        // converts front matter
368
        if ($page->getFrontmatter()) {
×
369
            try {
370
                $variables = $converter->convertFrontmatter($page->getFrontmatter(), $format);
×
371
            } catch (RuntimeException $e) {
×
372
                throw new RuntimeException($e->getMessage(), file: $page->getFilePath(), line: $e->getLine());
×
373
            }
374
            $page->setFmVariables($variables);
×
375
            $page->setVariables($variables);
×
376
        }
377

378
        // converts body (only if page is published or drafts option is enabled)
379
        if ($page->getVariable('published') || $this->options['drafts']) {
×
380
            try {
381
                $html = $converter->convertBody($page->getBody());
×
382
            } catch (RuntimeException $e) {
×
383
                throw new \Exception($e->getMessage());
×
384
            }
385
            $page->setBodyHtml($html);
×
386
        }
387

388
        return $page;
×
389
    }
390
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc