3936

Committed 15 Apr 2026 06:18AM UTC coverage: 66.884% (-4.1%) from 70.939%

Build # 3936

Build Type

Pull #155

circleci

Committed by

edwh

Commit Message

fix: adversarial review fixes for vector search

- Fix sort order: results were unsorted when count <= limit (selection
  sort only ran when truncation needed). Now always sorts top-K.
- Add DB nil check in embedding store Load()
- Add request size limit (100KB) to sidecar to prevent OOM
- Pin @huggingface/transformers to ^3.3.0 in both batch and sidecar
  to ensure identical embeddings
- Add TestStoreSearchSortOrder test verifying descending score order

Pull Request Pull Request #155: feat: Vector search using nomic-embed-text-v1.5 embeddings

Coverage Stats

8597 of 9952 branches covered (86.38%)

Branch coverage included in aggregate %.

56 of 354 new or added lines in 7 files covered. (15.82%)

11803 existing lines in 139 files now uncovered.

91748 of 140077 relevant lines covered (65.5%)

14.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.0

/iznik-batch/app/Console/Commands/Message/GenerateEmbeddingsCommand.php

<?php

namespace App\Console\Commands\Message;

use App\Traits\GracefulShutdown;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Symfony\Component\Process\Process;

class GenerateEmbeddingsCommand extends Command
{
    use GracefulShutdown;

    protected $signature = 'embeddings:generate
                            {--backfill : Process all messages without embeddings}
                            {--limit=500 : Maximum messages to process per run}';

    protected $description = 'Generate vector embeddings for messages in messages_spatial';

    public function handle(): int
    {
        $this->registerShutdownHandlers();

        $limit = (int) $this->option('limit');

        if ($this->option('backfill')) {
            $limit = 50000;
        }

        // Find messages in messages_spatial that don't have embeddings yet
        $messages = DB::select("
            SELECT ms.msgid, m.subject, LEFT(m.textbody, 500) as body
            FROM messages_spatial ms
            JOIN messages m ON m.id = ms.msgid
            LEFT JOIN messages_embeddings me ON me.msgid = ms.msgid
            WHERE me.msgid IS NULL
            AND ms.successful = 0
            AND ms.promised = 0
            ORDER BY ms.arrival DESC
            LIMIT ?
        ", [$limit]);

        if (empty($messages)) {
            $this->info('No messages need embedding.');

            return Command::SUCCESS;
        }

        $this->info(sprintf('Generating embeddings for %d messages...', count($messages)));

        // Prepare NDJSON input for the Node script
        $input = '';
        foreach ($messages as $msg) {
            $subject = $msg->subject ?? '';
            // Strip OFFER:/WANTED: prefix
            $subject = preg_replace('/^(OFFER|WANTED|Offered|Requested):\s*/i', '', $subject);
            $body = $msg->body ?? '';
            $text = trim($subject.'. '.$body);

            $input .= json_encode([
                'msgid' => $msg->msgid,
                'text' => $text,
            ])."\n";
        }

        // Shell out to Node.js embed script
        $scriptPath = base_path('resources/js/embed.mjs');
        $process = new Process(['node', $scriptPath]);
        $process->setInput($input);
        $process->setTimeout(300); // 5 min timeout

        $process->run();

        if (! $process->isSuccessful()) {
            $this->error('Embedding script failed: '.$process->getErrorOutput());
            Log::error('Embedding generation failed', [
                'stderr' => $process->getErrorOutput(),
            ]);

            return Command::FAILURE;
        }

        // Parse output and insert into DB
        $count = 0;
        $lines = explode("\n", trim($process->getOutput()));

        foreach ($lines as $line) {
            if ($this->shouldAbort()) {
                $this->warn('Aborting due to shutdown signal.');
                break;
            }

            $result = json_decode($line, true);
            if (! $result || ! isset($result['msgid'], $result['embedding'])) {
                continue;
            }

            // Pack float32 array into little-endian binary blob
            $binary = pack('g*', ...$result['embedding']);

            DB::statement(
                'INSERT IGNORE INTO messages_embeddings (msgid, embedding, model_version) VALUES (?, ?, ?)',
                [$result['msgid'], $binary, 'nomic-embed-text-v1.5-dim256']
            );

            $count++;
        }

        $this->info("Generated {$count} embeddings.");
        Log::info('Embedding generation complete', ['count' => $count]);

        return Command::SUCCESS;
    }
}

1	<?php
2
3	namespace App\Console\Commands\Message;
4
5	use App\Traits\GracefulShutdown;
6	use Illuminate\Console\Command;
7	use Illuminate\Support\Facades\DB;
8	use Illuminate\Support\Facades\Log;
9	use Symfony\Component\Process\Process;
10
11	class GenerateEmbeddingsCommand extends Command
12	{
13	use GracefulShutdown;
14
15	protected $signature = 'embeddings:generate
16	{--backfill : Process all messages without embeddings}
17	{--limit=500 : Maximum messages to process per run}';
18
19	protected $description = 'Generate vector embeddings for messages in messages_spatial';
20
21	public function handle(): int	2✔
22	{
23	$this->registerShutdownHandlers();	2✔
24
25	$limit = (int) $this->option('limit');	2✔
26
27	if ($this->option('backfill')) {	2✔
28	$limit = 50000;	1✔
29	}
30
31	// Find messages in messages_spatial that don't have embeddings yet
32	$messages = DB::select("	2✔
33	SELECT ms.msgid, m.subject, LEFT(m.textbody, 500) as body
34	FROM messages_spatial ms
35	JOIN messages m ON m.id = ms.msgid
36	LEFT JOIN messages_embeddings me ON me.msgid = ms.msgid
37	WHERE me.msgid IS NULL
38	AND ms.successful = 0
39	AND ms.promised = 0
40	ORDER BY ms.arrival DESC
41	LIMIT ?
42	", [$limit]);	2✔
43
44	if (empty($messages)) {	2✔
45	$this->info('No messages need embedding.');	2✔
46
47	return Command::SUCCESS;	2✔
48	}
49
NEW 50	$this->info(sprintf('Generating embeddings for %d messages...', count($messages)));	×
51
52	// Prepare NDJSON input for the Node script
NEW 53	$input = '';	×
NEW 54	foreach ($messages as $msg) {	×
NEW 55	$subject = $msg->subject ?? '';	×
56	// Strip OFFER:/WANTED: prefix
NEW 57	$subject = preg_replace('/^(OFFER\|WANTED\|Offered\|Requested):\s*/i', '', $subject);	×
NEW 58	$body = $msg->body ?? '';	×
NEW 59	$text = trim($subject.'. '.$body);	×
60
NEW 61	$input .= json_encode([	×
NEW 62	'msgid' => $msg->msgid,	×
NEW 63	'text' => $text,	×
NEW 64	])."\n";	×
65	}
66
67	// Shell out to Node.js embed script
NEW 68	$scriptPath = base_path('resources/js/embed.mjs');	×
NEW 69	$process = new Process(['node', $scriptPath]);	×
NEW 70	$process->setInput($input);	×
NEW 71	$process->setTimeout(300); // 5 min timeout	×
72
NEW 73	$process->run();	×
74
NEW 75	if (! $process->isSuccessful()) {	×
NEW 76	$this->error('Embedding script failed: '.$process->getErrorOutput());	×
NEW 77	Log::error('Embedding generation failed', [	×
NEW 78	'stderr' => $process->getErrorOutput(),	×
NEW 79	]);	×
80
NEW 81	return Command::FAILURE;	×
82	}
83
84	// Parse output and insert into DB
NEW 85	$count = 0;	×
NEW 86	$lines = explode("\n", trim($process->getOutput()));	×
87
NEW 88	foreach ($lines as $line) {	×
NEW 89	if ($this->shouldAbort()) {	×
NEW 90	$this->warn('Aborting due to shutdown signal.');	×
NEW 91	break;	×
92	}
93
NEW 94	$result = json_decode($line, true);	×
NEW 95	if (! $result \|\| ! isset($result['msgid'], $result['embedding'])) {	×
NEW 96	continue;	×
97	}
98
99	// Pack float32 array into little-endian binary blob
NEW 100	$binary = pack('g*', ...$result['embedding']);	×
101
NEW 102	DB::statement(	×
NEW 103	'INSERT IGNORE INTO messages_embeddings (msgid, embedding, model_version) VALUES (?, ?, ?)',	×
NEW 104	[$result['msgid'], $binary, 'nomic-embed-text-v1.5-dim256']	×
NEW 105	);	×
106
NEW 107	$count++;	×
108	}
109
NEW 110	$this->info("Generated {$count} embeddings.");	×
NEW 111	Log::info('Embedding generation complete', ['count' => $count]);	×
112
NEW 113	return Command::SUCCESS;	×
114	}
115	}

Freegle / Iznik / 3936

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous