• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

Freegle / Iznik / 4025

15 Apr 2026 09:46AM UTC coverage: 70.796% (+0.04%) from 70.755%
4025

Pull #155

circleci

edwh
Merge remote-tracking branch 'origin/master' into feature/vector-search
Pull Request #155: feat: Vector search using nomic-embed-text-v1.5 embeddings

13083 of 20082 branches covered (65.15%)

Branch coverage included in aggregate %.

241 of 369 new or added lines in 7 files covered. (65.31%)

7 existing lines in 2 files now uncovered.

93001 of 129762 relevant lines covered (71.67%)

17.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.0
/iznik-batch/app/Console/Commands/Message/GenerateEmbeddingsCommand.php
1
<?php
2

3
namespace App\Console\Commands\Message;
4

5
use App\Traits\GracefulShutdown;
6
use Illuminate\Console\Command;
7
use Illuminate\Support\Facades\DB;
8
use Illuminate\Support\Facades\Log;
9
use Symfony\Component\Process\Process;
10

11
class GenerateEmbeddingsCommand extends Command
12
{
13
    use GracefulShutdown;
14

15
    protected $signature = 'embeddings:generate
16
                            {--backfill : Process all messages without embeddings}
17
                            {--limit=500 : Maximum messages to process per run}';
18

19
    protected $description = 'Generate vector embeddings for messages in messages_spatial';
20

21
    public function handle(): int
2✔
22
    {
23
        $this->registerShutdownHandlers();
2✔
24

25
        $limit = (int) $this->option('limit');
2✔
26

27
        if ($this->option('backfill')) {
2✔
28
            $limit = 50000;
1✔
29
        }
30

31
        // Find messages in messages_spatial that don't have embeddings yet
32
        $messages = DB::select("
2✔
33
            SELECT ms.msgid, m.subject, LEFT(m.textbody, 500) as body
34
            FROM messages_spatial ms
35
            JOIN messages m ON m.id = ms.msgid
36
            LEFT JOIN messages_embeddings me ON me.msgid = ms.msgid
37
            WHERE me.msgid IS NULL
38
            AND ms.successful = 0
39
            AND ms.promised = 0
40
            ORDER BY ms.arrival DESC
41
            LIMIT ?
42
        ", [$limit]);
2✔
43

44
        if (empty($messages)) {
2✔
45
            $this->info('No messages need embedding.');
2✔
46

47
            return Command::SUCCESS;
2✔
48
        }
49

NEW
50
        $this->info(sprintf('Generating embeddings for %d messages...', count($messages)));
×
51

52
        // Prepare NDJSON input for the Node script
NEW
53
        $input = '';
×
NEW
54
        foreach ($messages as $msg) {
×
NEW
55
            $subject = $msg->subject ?? '';
×
56
            // Strip OFFER:/WANTED: prefix
NEW
57
            $subject = preg_replace('/^(OFFER|WANTED|Offered|Requested):\s*/i', '', $subject);
×
NEW
58
            $body = $msg->body ?? '';
×
NEW
59
            $text = trim($subject.'. '.$body);
×
60

NEW
61
            $input .= json_encode([
×
NEW
62
                'msgid' => $msg->msgid,
×
NEW
63
                'text' => $text,
×
NEW
64
            ])."\n";
×
65
        }
66

67
        // Shell out to Node.js embed script
NEW
68
        $scriptPath = base_path('resources/js/embed.mjs');
×
NEW
69
        $process = new Process(['node', $scriptPath]);
×
NEW
70
        $process->setInput($input);
×
NEW
71
        $process->setTimeout(300); // 5 min timeout
×
72

NEW
73
        $process->run();
×
74

NEW
75
        if (! $process->isSuccessful()) {
×
NEW
76
            $this->error('Embedding script failed: '.$process->getErrorOutput());
×
NEW
77
            Log::error('Embedding generation failed', [
×
NEW
78
                'stderr' => $process->getErrorOutput(),
×
NEW
79
            ]);
×
80

NEW
81
            return Command::FAILURE;
×
82
        }
83

84
        // Parse output and insert into DB
NEW
85
        $count = 0;
×
NEW
86
        $lines = explode("\n", trim($process->getOutput()));
×
87

NEW
88
        foreach ($lines as $line) {
×
NEW
89
            if ($this->shouldAbort()) {
×
NEW
90
                $this->warn('Aborting due to shutdown signal.');
×
NEW
91
                break;
×
92
            }
93

NEW
94
            $result = json_decode($line, true);
×
NEW
95
            if (! $result || ! isset($result['msgid'], $result['embedding'])) {
×
NEW
96
                continue;
×
97
            }
98

99
            // Pack float32 array into little-endian binary blob
NEW
100
            $binary = pack('g*', ...$result['embedding']);
×
101

NEW
102
            DB::statement(
×
NEW
103
                'INSERT IGNORE INTO messages_embeddings (msgid, embedding, model_version) VALUES (?, ?, ?)',
×
NEW
104
                [$result['msgid'], $binary, 'nomic-embed-text-v1.5-dim256']
×
NEW
105
            );
×
106

NEW
107
            $count++;
×
108
        }
109

NEW
110
        $this->info("Generated {$count} embeddings.");
×
NEW
111
        Log::info('Embedding generation complete', ['count' => $count]);
×
112

NEW
113
        return Command::SUCCESS;
×
114
    }
115
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc