#2566

Committed 12 Jan 2026 06:33AM UTC coverage: 88.008%. First build

Build # #2566

Build Type

push

Committed by

Commit Message

Fix illustration scripts getting stuck in infinite loop on failing items

Changes:
- fetchBatch() now returns partial results instead of FALSE when individual
  items fail to return data (only true rate-limiting fails the batch)
- Add file-based tracking for items that repeatedly fail (/tmp/pollinations_failed.json)
- Items that fail 3 times are skipped for 1 day before retrying
- Both messages_illustrations.php and jobs_illustrations.php updated to:
  - Skip items that have exceeded failure threshold
  - Record failures for items that don't return data
  - Process successful results from partial batches

This prevents a single problematic item (like "2 toilet seat trainers") from
blocking all illustration generation indefinitely.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Run Details

0 of 47 new or added lines in 1 file covered. (0.0%)

26303 of 29887 relevant lines covered (88.01%)

31.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/include/misc/Pollinations.php

<?php

namespace Freegle\Iznik;

/**
 * Helper class for fetching AI-generated images from Pollinations.ai.
 * Tracks image hashes to detect rate-limiting (same image returned for different prompts).
 * Uses both in-memory and file-based caching for persistence across processes.
 */
class Pollinations {
    # Track hashes we've seen in this process, keyed by hash => prompt.
    private static $seenHashes = [];

    # File-based cache location.
    const CACHE_FILE = '/tmp/pollinations_hashes.json';

    # Cache expiry in seconds (24 hours).
    const CACHE_EXPIRY = 86400;

    # File-based cache for failed items.
    const FAILED_CACHE_FILE = '/tmp/pollinations_failed.json';

    # Max failures before permanently skipping an item.
    const MAX_FAILURES = 3;

    # Failed items cache expiry (1 day - give items a chance to work later).
    const FAILED_CACHE_EXPIRY = 86400;

    /**
     * Load the file-based hash cache.
     * @return array Hash => [prompt, timestamp] mapping.
     */
    private static function loadFileCache() {
        if (!file_exists(self::CACHE_FILE)) {
            return [];
        }

        $data = @file_get_contents(self::CACHE_FILE);
        if (!$data) {
            return [];
        }

        $cache = @json_decode($data, TRUE);
        if (!is_array($cache)) {
            return [];
        }

        # Remove expired entries.
        $now = time();
        $cache = array_filter($cache, function($entry) use ($now) {
            return isset($entry['timestamp']) && ($now - $entry['timestamp']) < self::CACHE_EXPIRY;
        });

        return $cache;
    }

    /**
     * Save the file-based hash cache.
     * @param array $cache Hash => [prompt, timestamp] mapping.
     */
    private static function saveFileCache($cache) {
        # Use file locking to prevent race conditions.
        $fp = @fopen(self::CACHE_FILE, 'c');
        if (!$fp) {
            return;
        }

        if (flock($fp, LOCK_EX)) {
            ftruncate($fp, 0);
            fwrite($fp, json_encode($cache));
            fflush($fp);
            flock($fp, LOCK_UN);
        }

        fclose($fp);
    }

    /**
     * Check if a hash exists in the file cache for a different prompt.
     * @param string $hash Image hash.
     * @param string $prompt Current prompt.
     * @return string|false The existing prompt if duplicate found, FALSE otherwise.
     */
    private static function checkFileCache($hash, $prompt) {
        $cache = self::loadFileCache();

        if (isset($cache[$hash]) && $cache[$hash]['prompt'] !== $prompt) {
            return $cache[$hash]['prompt'];
        }

        return FALSE;
    }

    /**
     * Add a hash to the file cache.
     * @param string $hash Image hash.
     * @param string $prompt The prompt that generated this image.
     */
    private static function addToFileCache($hash, $prompt) {
        $cache = self::loadFileCache();

        $cache[$hash] = [
            'prompt' => $prompt,
            'timestamp' => time()
        ];

        self::saveFileCache($cache);
    }

    /**
     * Load the failed items cache.
     * @return array itemName => ['count' => int, 'timestamp' => int]
     */
    private static function loadFailedCache() {
        if (!file_exists(self::FAILED_CACHE_FILE)) {
            return [];
        }

        $data = @file_get_contents(self::FAILED_CACHE_FILE);
        if (!$data) {
            return [];
        }

        $cache = @json_decode($data, TRUE);
        if (!is_array($cache)) {
            return [];
        }

        # Remove expired entries.
        $now = time();
        $cache = array_filter($cache, function($entry) use ($now) {
            return isset($entry['timestamp']) && ($now - $entry['timestamp']) < self::FAILED_CACHE_EXPIRY;
        });

        return $cache;
    }

    /**
     * Save the failed items cache.
     * @param array $cache itemName => ['count' => int, 'timestamp' => int]
     */
    private static function saveFailedCache($cache) {
        $fp = @fopen(self::FAILED_CACHE_FILE, 'c');
        if (!$fp) {
            return;
        }

        if (flock($fp, LOCK_EX)) {
            ftruncate($fp, 0);
            fwrite($fp, json_encode($cache));
            fflush($fp);
            flock($fp, LOCK_UN);
        }

        fclose($fp);
    }

    /**
     * Record a failure for an item. Returns TRUE if item should be skipped (too many failures).
     * @param string $itemName The item name that failed.
     * @return bool TRUE if item has exceeded max failures and should be skipped.
     */
    public static function recordFailure($itemName) {
        $cache = self::loadFailedCache();

        if (!isset($cache[$itemName])) {
            $cache[$itemName] = ['count' => 0, 'timestamp' => time()];
        }

        $cache[$itemName]['count']++;
        $cache[$itemName]['timestamp'] = time();

        self::saveFailedCache($cache);

        $shouldSkip = $cache[$itemName]['count'] >= self::MAX_FAILURES;
        if ($shouldSkip) {
            error_log("Item '$itemName' has failed " . $cache[$itemName]['count'] . " times, will skip for 1 day");
        }

        return $shouldSkip;
    }

    /**
     * Check if an item should be skipped due to previous failures.
     * @param string $itemName The item name to check.
     * @return bool TRUE if item should be skipped.
     */
    public static function shouldSkipItem($itemName) {
        $cache = self::loadFailedCache();

        if (!isset($cache[$itemName])) {
            return FALSE;
        }

        return $cache[$itemName]['count'] >= self::MAX_FAILURES;
    }

    /**
     * Clear the failed items cache (mainly for testing/maintenance).
     */
    public static function clearFailedCache() {
        if (file_exists(self::FAILED_CACHE_FILE)) {
            @unlink(self::FAILED_CACHE_FILE);
        }
    }

    /**
     * Fetch an image from Pollinations.ai for the given prompt.
     * Returns image data on success, or FALSE if rate-limited/failed.
     *
     * @param string $prompt The item name/description to generate an image for.
     * @param string $fullPrompt The full prompt string to send to Pollinations.
     * @param int $width Image width.
     * @param int $height Image height.
     * @param int $timeout Timeout in seconds.
     * @return string|false Image data on success, FALSE on failure or rate-limiting.
     */
    public static function fetchImage($prompt, $fullPrompt, $width = 640, $height = 480, $timeout = 120) {
        global $dbhr;

        $url = "https://image.pollinations.ai/prompt/" . urlencode($fullPrompt) .
               "?width={$width}&height={$height}&nologo=true&seed=1";

        $ctx = stream_context_create([
            'http' => [
                'timeout' => $timeout
            ]
        ]);

        $data = @file_get_contents($url, FALSE, $ctx);

        # Check for HTTP 429 rate limiting.
        if (isset($http_response_header)) {
            foreach ($http_response_header as $header) {
                if (preg_match('/^HTTP\/\d+\.?\d*\s+429/', $header)) {
                    error_log("Pollinations rate limited (HTTP 429) for: " . $prompt);
                    return FALSE;
                }
            }
        }

        if (!$data || strlen($data) == 0) {
            error_log("Pollinations failed to return data for: " . $prompt);
            return FALSE;
        }

        # Compute hash of received image.
        $hash = md5($data);

        # Check 1: In-memory cache (same process).
        if (isset(self::$seenHashes[$hash]) && self::$seenHashes[$hash] !== $prompt) {
            error_log("Pollinations rate limited (in-memory duplicate) for: " . $prompt .
                      " (same image as: " . self::$seenHashes[$hash] . ")");
            # Clean up the recently added entry that we now know is rate-limited.
            self::cleanupRateLimitedHash($hash);
            return FALSE;
        }

        # Check 2: File-based cache (across processes).
        $existingPrompt = self::checkFileCache($hash, $prompt);
        if ($existingPrompt !== FALSE) {
            error_log("Pollinations rate limited (file cache duplicate) for: " . $prompt .
                      " (same image as: " . $existingPrompt . ")");
            # Clean up recently added entries with this hash.
            self::cleanupRateLimitedHash($hash);
            return FALSE;
        }

        # Check 3: Database (historical data).
        if ($dbhr) {
            $existing = $dbhr->preQuery(
                "SELECT name FROM ai_images WHERE imagehash = ? AND name != ? LIMIT 1",
                [$hash, $prompt]
            );

            if (count($existing) > 0) {
                error_log("Pollinations rate limited (DB duplicate) for: " . $prompt .
                          " (same image as: " . $existing[0]['name'] . ")");
                # Also add to file cache to speed up future checks.
                self::addToFileCache($hash, $existing[0]['name']);
                return FALSE;
            }
        }

        # All checks passed - track this hash.
        self::$seenHashes[$hash] = $prompt;
        self::addToFileCache($hash, $prompt);

        return $data;
    }

    /**
     * Clean up recently added ai_images entries with a rate-limited hash.
     * Only removes entries added in the last hour to avoid removing legitimate old entries.
     * @param string $hash The rate-limited image hash.
     */
    private static function cleanupRateLimitedHash($hash) {
        global $dbhm;

        if (!$dbhm) {
            return;
        }

        # Only clean up entries added in the last hour - these are likely from this rate-limiting event.
        $deleted = $dbhm->preExec(
            "DELETE FROM ai_images WHERE imagehash = ? AND created > DATE_SUB(NOW(), INTERVAL 1 HOUR)",
            [$hash]
        );

        if ($deleted) {
            $count = $dbhm->rowsAffected();
            if ($count > 0) {
                error_log("Cleaned up $count recent ai_images entries with rate-limited hash: $hash");
            }
        }

        # Also clean up messages_attachments that were recently added with externaluids
        # that are now known to be rate-limited (i.e., the ai_images entry was just deleted).
        # We look for recent AI attachments that no longer have a matching ai_images entry.
        $minId = $dbhm->preQuery("SELECT COALESCE(MAX(id), 0) - 10000 as minid FROM messages_attachments");
        $minIdVal = $minId[0]['minid'] ?? 0;

        $orphaned = $dbhm->preExec(
            "DELETE ma FROM messages_attachments ma
             LEFT JOIN ai_images ai ON ma.externaluid = ai.externaluid
             WHERE ma.externaluid LIKE 'freegletusd-%'
             AND JSON_EXTRACT(ma.externalmods, '$.ai') = TRUE
             AND ai.id IS NULL
             AND ma.id > ?",
            [$minIdVal]
        );

        if ($orphaned) {
            $count = $dbhm->rowsAffected();
            if ($count > 0) {
                error_log("Cleaned up $count orphaned message attachments");
            }
        }
    }

    /**
     * Build a prompt for a message illustration.
     * @param string $itemName The item name.
     * @return string The full prompt.
     */
    public static function buildMessagePrompt($itemName) {
        # Prompt injection defense.
        $cleanName = str_replace('CRITICAL:', '', $itemName);
        $cleanName = str_replace('Draw only', '', $cleanName);

        return "Draw a single friendly cartoon white line drawing on dark green background, moderate shading, " .
               "cute and quirky style, UK audience, centered, gender-neutral, " .
               "if showing people use abstract non-gendered figures. " .
               "CRITICAL: Do not include any text, words, letters, numbers or labels anywhere in the image. " .
               "Draw only a picture of: " . $cleanName;
    }

    /**
     * Build a prompt for a job illustration.
     * @param string $jobTitle The job title.
     * @return string The full prompt.
     */
    public static function buildJobPrompt($jobTitle) {
        # Prompt injection defense.
        $cleanName = str_replace('CRITICAL:', '', $jobTitle);
        $cleanName = str_replace('Draw only', '', $cleanName);

        return "simple cute cartoon " . $cleanName . " white line drawing on solid dark forest green background, " .
               "minimalist icon style, gender-neutral, if showing people use abstract non-gendered figures, " .
               "absolutely no text, no words, no letters, no numbers, no labels, " .
               "no writing, no captions, no signs, no speech bubbles, no border, filling the entire frame";
    }

    /**
     * Fetch a batch of images, returning successful results and tracking failures.
     * Individual item failures (no data returned) do NOT fail the batch - only actual
     * rate-limiting (HTTP 429 or duplicate images) fails the entire batch.
     *
     * @param array $items Array of ['name' => string, 'prompt' => string, 'width' => int, 'height' => int]
     * @param int $timeout Timeout per request in seconds.
     * @return array|false Array with 'results' and 'failed' keys on success, FALSE if rate-limited.
     *                     'results' => [['name' => string, 'data' => string, 'hash' => string], ...]
     *                     'failed' => [name => TRUE, ...] items that failed to fetch (not rate-limited)
     */
    public static function fetchBatch($items, $timeout = 120) {
        if (empty($items)) {
            return ['results' => [], 'failed' => []];
        }

        $results = [];
        $failed = [];
        $batchHashes = [];

        foreach ($items as $item) {
            $name = $item['name'];
            $prompt = $item['prompt'];
            $width = $item['width'] ?? 640;
            $height = $item['height'] ?? 480;

            $url = "https://image.pollinations.ai/prompt/" . urlencode($prompt) .
                   "?width={$width}&height={$height}&nologo=true&seed=1";

            $ctx = stream_context_create([
                'http' => [
                    'timeout' => $timeout
                ]
            ]);

            error_log("Batch fetching image for: $name");
            $data = @file_get_contents($url, FALSE, $ctx);

            # Check for HTTP 429 - this is real rate-limiting, fail entire batch.
            if (isset($http_response_header)) {
                foreach ($http_response_header as $header) {
                    if (preg_match('/^HTTP\/\d+\.?\d*\s+429/', $header)) {
                        error_log("Pollinations rate limited (HTTP 429) for batch item: $name");
                        return FALSE;
                    }
                }
            }

            # No data returned - this is an individual failure, NOT rate-limiting.
            # Skip this item but continue with others.
            if (!$data || strlen($data) == 0) {
                error_log("Pollinations failed to return data for batch item: $name (skipping)");
                $failed[$name] = TRUE;
                continue;
            }

            $hash = md5($data);

            # Check if this hash already appeared in this batch for a different item.
            # This IS rate-limiting - fail entire batch.
            if (isset($batchHashes[$hash]) && $batchHashes[$hash] !== $name) {
                error_log("Pollinations rate limited (batch duplicate) for: $name (same as: " . $batchHashes[$hash] . ")");
                # Clean up any recently added entries with this hash.
                self::cleanupRateLimitedHash($hash);
                return FALSE;
            }

            # Check file cache - this IS rate-limiting.
            $existingPrompt = self::checkFileCache($hash, $name);
            if ($existingPrompt !== FALSE) {
                error_log("Pollinations rate limited (file cache) for batch item: $name (same as: $existingPrompt)");
                self::cleanupRateLimitedHash($hash);
                return FALSE;
            }

            # Check database for historical duplicates - this IS rate-limiting.
            global $dbhr;
            if ($dbhr) {
                $existing = $dbhr->preQuery(
                    "SELECT name FROM ai_images WHERE imagehash = ? AND name != ? LIMIT 1",
                    [$hash, $name]
                );

                if (count($existing) > 0) {
                    error_log("Pollinations rate limited (DB) for batch item: $name (same as: " . $existing[0]['name'] . ")");
                    self::addToFileCache($hash, $existing[0]['name']);
                    self::cleanupRateLimitedHash($hash);
                    return FALSE;
                }
            }

            $batchHashes[$hash] = $name;
            $results[] = [
                'name' => $name,
                'data' => $data,
                'hash' => $hash,
                'msgid' => $item['msgid'] ?? NULL,
                'jobid' => $item['jobid'] ?? NULL
            ];

            # Small delay between requests.
            sleep(1);
        }

        # Add successful images to caches.
        foreach ($results as $result) {
            self::$seenHashes[$result['hash']] = $result['name'];
            self::addToFileCache($result['hash'], $result['name']);
        }

        return ['results' => $results, 'failed' => $failed];
    }

    /**
     * Cache an image in ai_images table.
     * @param string $name The item/job name.
     * @param string $uid The externaluid.
     * @param string $hash Image hash.
     */
    public static function cacheImage($name, $uid, $hash) {
        global $dbhm;

        if ($dbhm) {
            $dbhm->preExec(
                "INSERT INTO ai_images (name, externaluid, imagehash) VALUES (?, ?, ?)
                 ON DUPLICATE KEY UPDATE externaluid = VALUES(externaluid), imagehash = VALUES(imagehash), created = NOW()",
                [$name, $uid, $hash]
            );
        }
    }

    /**
     * Upload image to TUS and cache in ai_images table.
     * @param string $name The item/job name.
     * @param string $data Image data.
     * @param string $hash Image hash.
     * @return string|false The externaluid on success, FALSE on failure.
     */
    public static function uploadAndCache($name, $data, $hash) {
        $t = new Tus();
        $tusUrl = $t->upload(NULL, 'image/jpeg', $data);

        if (!$tusUrl) {
            error_log("Failed to upload image to TUS for: $name");
            return FALSE;
        }

        $uid = 'freegletusd-' . basename($tusUrl);
        self::cacheImage($name, $uid, $hash);

        return $uid;
    }

    /**
     * Get the hash of image data.
     * @param string $data Image data.
     * @return string MD5 hash.
     */
    public static function getImageHash($data) {
        return md5($data);
    }

    /**
     * Clear the in-memory hash cache (mainly for testing).
     */
    public static function clearCache() {
        self::$seenHashes = [];
    }

    /**
     * Clear the file-based hash cache (mainly for testing/maintenance).
     */
    public static function clearFileCache() {
        if (file_exists(self::CACHE_FILE)) {
            @unlink(self::CACHE_FILE);
        }
    }
}

1	<?php
2
3	namespace Freegle\Iznik;
4
5	/**
6	* Helper class for fetching AI-generated images from Pollinations.ai.
7	* Tracks image hashes to detect rate-limiting (same image returned for different prompts).
8	* Uses both in-memory and file-based caching for persistence across processes.
9	*/
10	class Pollinations {
11	# Track hashes we've seen in this process, keyed by hash => prompt.
12	private static $seenHashes = [];
13
14	# File-based cache location.
15	const CACHE_FILE = '/tmp/pollinations_hashes.json';
16
17	# Cache expiry in seconds (24 hours).
18	const CACHE_EXPIRY = 86400;
19
20	# File-based cache for failed items.
21	const FAILED_CACHE_FILE = '/tmp/pollinations_failed.json';
22
23	# Max failures before permanently skipping an item.
24	const MAX_FAILURES = 3;
25
26	# Failed items cache expiry (1 day - give items a chance to work later).
27	const FAILED_CACHE_EXPIRY = 86400;
28
29	/**
30	* Load the file-based hash cache.
31	* @return array Hash => [prompt, timestamp] mapping.
32	*/
33	private static function loadFileCache() {
34	if (!file_exists(self::CACHE_FILE)) {	×
35	return [];	×
36	}
37
38	$data = @file_get_contents(self::CACHE_FILE);	×
39	if (!$data) {	×
40	return [];	×
41	}
42
43	$cache = @json_decode($data, TRUE);	×
44	if (!is_array($cache)) {	×
45	return [];	×
46	}
47
48	# Remove expired entries.
49	$now = time();	×
50	$cache = array_filter($cache, function($entry) use ($now) {	×
51	return isset($entry['timestamp']) && ($now - $entry['timestamp']) < self::CACHE_EXPIRY;	×
52	});	×
53
54	return $cache;	×
55	}
56
57	/**
58	* Save the file-based hash cache.
59	* @param array $cache Hash => [prompt, timestamp] mapping.
60	*/
61	private static function saveFileCache($cache) {
62	# Use file locking to prevent race conditions.
63	$fp = @fopen(self::CACHE_FILE, 'c');	×
64	if (!$fp) {	×
65	return;	×
66	}
67
68	if (flock($fp, LOCK_EX)) {	×
69	ftruncate($fp, 0);	×
70	fwrite($fp, json_encode($cache));	×
71	fflush($fp);	×
72	flock($fp, LOCK_UN);	×
73	}
74
75	fclose($fp);	×
76	}
77
78	/**
79	* Check if a hash exists in the file cache for a different prompt.
80	* @param string $hash Image hash.
81	* @param string $prompt Current prompt.
82	* @return string\|false The existing prompt if duplicate found, FALSE otherwise.
83	*/
84	private static function checkFileCache($hash, $prompt) {
85	$cache = self::loadFileCache();	×
86
87	if (isset($cache[$hash]) && $cache[$hash]['prompt'] !== $prompt) {	×
88	return $cache[$hash]['prompt'];	×
89	}
90
91	return FALSE;	×
92	}
93
94	/**
95	* Add a hash to the file cache.
96	* @param string $hash Image hash.
97	* @param string $prompt The prompt that generated this image.
98	*/
99	private static function addToFileCache($hash, $prompt) {
100	$cache = self::loadFileCache();	×
101
102	$cache[$hash] = [	×
103	'prompt' => $prompt,	×
104	'timestamp' => time()	×
105	];	×
106
107	self::saveFileCache($cache);	×
108	}
109
110	/**
111	* Load the failed items cache.
112	* @return array itemName => ['count' => int, 'timestamp' => int]
113	*/
114	private static function loadFailedCache() {
NEW 115	if (!file_exists(self::FAILED_CACHE_FILE)) {	×
NEW 116	return [];	×
117	}
118
NEW 119	$data = @file_get_contents(self::FAILED_CACHE_FILE);	×
NEW 120	if (!$data) {	×
NEW 121	return [];	×
122	}
123
NEW 124	$cache = @json_decode($data, TRUE);	×
NEW 125	if (!is_array($cache)) {	×
NEW 126	return [];	×
127	}
128
129	# Remove expired entries.
NEW 130	$now = time();	×
NEW 131	$cache = array_filter($cache, function($entry) use ($now) {	×
NEW 132	return isset($entry['timestamp']) && ($now - $entry['timestamp']) < self::FAILED_CACHE_EXPIRY;	×
NEW 133	});	×
134
NEW 135	return $cache;	×
136	}
137
138	/**
139	* Save the failed items cache.
140	* @param array $cache itemName => ['count' => int, 'timestamp' => int]
141	*/
142	private static function saveFailedCache($cache) {
NEW 143	$fp = @fopen(self::FAILED_CACHE_FILE, 'c');	×
NEW 144	if (!$fp) {	×
NEW 145	return;	×
146	}
147
NEW 148	if (flock($fp, LOCK_EX)) {	×
NEW 149	ftruncate($fp, 0);	×
NEW 150	fwrite($fp, json_encode($cache));	×
NEW 151	fflush($fp);	×
NEW 152	flock($fp, LOCK_UN);	×
153	}
154
NEW 155	fclose($fp);	×
156	}
157
158	/**
159	* Record a failure for an item. Returns TRUE if item should be skipped (too many failures).
160	* @param string $itemName The item name that failed.
161	* @return bool TRUE if item has exceeded max failures and should be skipped.
162	*/
163	public static function recordFailure($itemName) {
NEW 164	$cache = self::loadFailedCache();	×
165
NEW 166	if (!isset($cache[$itemName])) {	×
NEW 167	$cache[$itemName] = ['count' => 0, 'timestamp' => time()];	×
168	}
169
NEW 170	$cache[$itemName]['count']++;	×
NEW 171	$cache[$itemName]['timestamp'] = time();	×
172
NEW 173	self::saveFailedCache($cache);	×
174
NEW 175	$shouldSkip = $cache[$itemName]['count'] >= self::MAX_FAILURES;	×
NEW 176	if ($shouldSkip) {	×
NEW 177	error_log("Item '$itemName' has failed " . $cache[$itemName]['count'] . " times, will skip for 1 day");	×
178	}
179
NEW 180	return $shouldSkip;	×
181	}
182
183	/**
184	* Check if an item should be skipped due to previous failures.
185	* @param string $itemName The item name to check.
186	* @return bool TRUE if item should be skipped.
187	*/
188	public static function shouldSkipItem($itemName) {
NEW 189	$cache = self::loadFailedCache();	×
190
NEW 191	if (!isset($cache[$itemName])) {	×
NEW 192	return FALSE;	×
193	}
194
NEW 195	return $cache[$itemName]['count'] >= self::MAX_FAILURES;	×
196	}
197
198	/**
199	* Clear the failed items cache (mainly for testing/maintenance).
200	*/
201	public static function clearFailedCache() {
NEW 202	if (file_exists(self::FAILED_CACHE_FILE)) {	×
NEW 203	@unlink(self::FAILED_CACHE_FILE);	×
204	}
205	}
206
207	/**
208	* Fetch an image from Pollinations.ai for the given prompt.
209	* Returns image data on success, or FALSE if rate-limited/failed.
210	*
211	* @param string $prompt The item name/description to generate an image for.
212	* @param string $fullPrompt The full prompt string to send to Pollinations.
213	* @param int $width Image width.
214	* @param int $height Image height.
215	* @param int $timeout Timeout in seconds.
216	* @return string\|false Image data on success, FALSE on failure or rate-limiting.
217	*/
218	public static function fetchImage($prompt, $fullPrompt, $width = 640, $height = 480, $timeout = 120) {
219	global $dbhr;	×
220
221	$url = "https://image.pollinations.ai/prompt/" . urlencode($fullPrompt) .	×
222	"?width={$width}&height={$height}&nologo=true&seed=1";	×
223
224	$ctx = stream_context_create([	×
225	'http' => [	×
226	'timeout' => $timeout	×
227	]	×
228	]);	×
229
230	$data = @file_get_contents($url, FALSE, $ctx);	×
231
232	# Check for HTTP 429 rate limiting.
233	if (isset($http_response_header)) {	×
234	foreach ($http_response_header as $header) {	×
235	if (preg_match('/^HTTP\/\d+\.?\d*\s+429/', $header)) {	×
236	error_log("Pollinations rate limited (HTTP 429) for: " . $prompt);	×
237	return FALSE;	×
238	}
239	}
240	}
241
242	if (!$data \|\| strlen($data) == 0) {	×
243	error_log("Pollinations failed to return data for: " . $prompt);	×
244	return FALSE;	×
245	}
246
247	# Compute hash of received image.
248	$hash = md5($data);	×
249
250	# Check 1: In-memory cache (same process).
251	if (isset(self::$seenHashes[$hash]) && self::$seenHashes[$hash] !== $prompt) {	×
252	error_log("Pollinations rate limited (in-memory duplicate) for: " . $prompt .	×
253	" (same image as: " . self::$seenHashes[$hash] . ")");	×
254	# Clean up the recently added entry that we now know is rate-limited.
255	self::cleanupRateLimitedHash($hash);	×
256	return FALSE;	×
257	}
258
259	# Check 2: File-based cache (across processes).
260	$existingPrompt = self::checkFileCache($hash, $prompt);	×
261	if ($existingPrompt !== FALSE) {	×
262	error_log("Pollinations rate limited (file cache duplicate) for: " . $prompt .	×
263	" (same image as: " . $existingPrompt . ")");	×
264	# Clean up recently added entries with this hash.
265	self::cleanupRateLimitedHash($hash);	×
266	return FALSE;	×
267	}
268
269	# Check 3: Database (historical data).
270	if ($dbhr) {	×
271	$existing = $dbhr->preQuery(	×
272	"SELECT name FROM ai_images WHERE imagehash = ? AND name != ? LIMIT 1",	×
273	[$hash, $prompt]	×
274	);	×
275
276	if (count($existing) > 0) {	×
277	error_log("Pollinations rate limited (DB duplicate) for: " . $prompt .	×
278	" (same image as: " . $existing[0]['name'] . ")");	×
279	# Also add to file cache to speed up future checks.
280	self::addToFileCache($hash, $existing[0]['name']);	×
281	return FALSE;	×
282	}
283	}
284
285	# All checks passed - track this hash.
286	self::$seenHashes[$hash] = $prompt;	×
287	self::addToFileCache($hash, $prompt);	×
288
289	return $data;	×
290	}
291
292	/**
293	* Clean up recently added ai_images entries with a rate-limited hash.
294	* Only removes entries added in the last hour to avoid removing legitimate old entries.
295	* @param string $hash The rate-limited image hash.
296	*/
297	private static function cleanupRateLimitedHash($hash) {
298	global $dbhm;	×
299
300	if (!$dbhm) {	×
301	return;	×
302	}
303
304	# Only clean up entries added in the last hour - these are likely from this rate-limiting event.
305	$deleted = $dbhm->preExec(	×
306	"DELETE FROM ai_images WHERE imagehash = ? AND created > DATE_SUB(NOW(), INTERVAL 1 HOUR)",	×
307	[$hash]	×
308	);	×
309
310	if ($deleted) {	×
311	$count = $dbhm->rowsAffected();	×
312	if ($count > 0) {	×
313	error_log("Cleaned up $count recent ai_images entries with rate-limited hash: $hash");	×
314	}
315	}
316
317	# Also clean up messages_attachments that were recently added with externaluids
318	# that are now known to be rate-limited (i.e., the ai_images entry was just deleted).
319	# We look for recent AI attachments that no longer have a matching ai_images entry.
320	$minId = $dbhm->preQuery("SELECT COALESCE(MAX(id), 0) - 10000 as minid FROM messages_attachments");	×
321	$minIdVal = $minId[0]['minid'] ?? 0;	×
322
323	$orphaned = $dbhm->preExec(	×
324	"DELETE ma FROM messages_attachments ma	×
325	LEFT JOIN ai_images ai ON ma.externaluid = ai.externaluid
326	WHERE ma.externaluid LIKE 'freegletusd-%'
327	AND JSON_EXTRACT(ma.externalmods, '$.ai') = TRUE
328	AND ai.id IS NULL
329	AND ma.id > ?",	×
330	[$minIdVal]	×
331	);	×
332
333	if ($orphaned) {	×
334	$count = $dbhm->rowsAffected();	×
335	if ($count > 0) {	×
336	error_log("Cleaned up $count orphaned message attachments");	×
337	}
338	}
339	}
340
341	/**
342	* Build a prompt for a message illustration.
343	* @param string $itemName The item name.
344	* @return string The full prompt.
345	*/
346	public static function buildMessagePrompt($itemName) {
347	# Prompt injection defense.
348	$cleanName = str_replace('CRITICAL:', '', $itemName);	×
349	$cleanName = str_replace('Draw only', '', $cleanName);	×
350
351	return "Draw a single friendly cartoon white line drawing on dark green background, moderate shading, " .	×
352	"cute and quirky style, UK audience, centered, gender-neutral, " .	×
353	"if showing people use abstract non-gendered figures. " .	×
354	"CRITICAL: Do not include any text, words, letters, numbers or labels anywhere in the image. " .	×
355	"Draw only a picture of: " . $cleanName;	×
356	}
357
358	/**
359	* Build a prompt for a job illustration.
360	* @param string $jobTitle The job title.
361	* @return string The full prompt.
362	*/
363	public static function buildJobPrompt($jobTitle) {
364	# Prompt injection defense.
365	$cleanName = str_replace('CRITICAL:', '', $jobTitle);	×
366	$cleanName = str_replace('Draw only', '', $cleanName);	×
367
368	return "simple cute cartoon " . $cleanName . " white line drawing on solid dark forest green background, " .	×
369	"minimalist icon style, gender-neutral, if showing people use abstract non-gendered figures, " .	×
370	"absolutely no text, no words, no letters, no numbers, no labels, " .	×
371	"no writing, no captions, no signs, no speech bubbles, no border, filling the entire frame";	×
372	}
373
374	/**
375	* Fetch a batch of images, returning successful results and tracking failures.
376	* Individual item failures (no data returned) do NOT fail the batch - only actual
377	* rate-limiting (HTTP 429 or duplicate images) fails the entire batch.
378	*
379	* @param array $items Array of ['name' => string, 'prompt' => string, 'width' => int, 'height' => int]
380	* @param int $timeout Timeout per request in seconds.
381	* @return array\|false Array with 'results' and 'failed' keys on success, FALSE if rate-limited.
382	* 'results' => [['name' => string, 'data' => string, 'hash' => string], ...]
383	* 'failed' => [name => TRUE, ...] items that failed to fetch (not rate-limited)
384	*/
385	public static function fetchBatch($items, $timeout = 120) {
386	if (empty($items)) {	×
NEW 387	return ['results' => [], 'failed' => []];	×
388	}
389
390	$results = [];	×
NEW 391	$failed = [];	×
392	$batchHashes = [];	×
393
394	foreach ($items as $item) {	×
395	$name = $item['name'];	×
396	$prompt = $item['prompt'];	×
397	$width = $item['width'] ?? 640;	×
398	$height = $item['height'] ?? 480;	×
399
400	$url = "https://image.pollinations.ai/prompt/" . urlencode($prompt) .	×
401	"?width={$width}&height={$height}&nologo=true&seed=1";	×
402
403	$ctx = stream_context_create([	×
404	'http' => [	×
405	'timeout' => $timeout	×
406	]	×
407	]);	×
408
409	error_log("Batch fetching image for: $name");	×
410	$data = @file_get_contents($url, FALSE, $ctx);	×
411
412	# Check for HTTP 429 - this is real rate-limiting, fail entire batch.
413	if (isset($http_response_header)) {	×
414	foreach ($http_response_header as $header) {	×
415	if (preg_match('/^HTTP\/\d+\.?\d*\s+429/', $header)) {	×
416	error_log("Pollinations rate limited (HTTP 429) for batch item: $name");	×
417	return FALSE;	×
418	}
419	}
420	}
421
422	# No data returned - this is an individual failure, NOT rate-limiting.
423	# Skip this item but continue with others.
424	if (!$data \|\| strlen($data) == 0) {	×
NEW 425	error_log("Pollinations failed to return data for batch item: $name (skipping)");	×
NEW 426	$failed[$name] = TRUE;	×
NEW 427	continue;	×
428	}
429
430	$hash = md5($data);	×
431
432	# Check if this hash already appeared in this batch for a different item.
433	# This IS rate-limiting - fail entire batch.
434	if (isset($batchHashes[$hash]) && $batchHashes[$hash] !== $name) {	×
435	error_log("Pollinations rate limited (batch duplicate) for: $name (same as: " . $batchHashes[$hash] . ")");	×
436	# Clean up any recently added entries with this hash.
437	self::cleanupRateLimitedHash($hash);	×
438	return FALSE;	×
439	}
440
441	# Check file cache - this IS rate-limiting.
442	$existingPrompt = self::checkFileCache($hash, $name);	×
443	if ($existingPrompt !== FALSE) {	×
444	error_log("Pollinations rate limited (file cache) for batch item: $name (same as: $existingPrompt)");	×
445	self::cleanupRateLimitedHash($hash);	×
446	return FALSE;	×
447	}
448
449	# Check database for historical duplicates - this IS rate-limiting.
450	global $dbhr;	×
451	if ($dbhr) {	×
452	$existing = $dbhr->preQuery(	×
453	"SELECT name FROM ai_images WHERE imagehash = ? AND name != ? LIMIT 1",	×
454	[$hash, $name]	×
455	);	×
456
457	if (count($existing) > 0) {	×
458	error_log("Pollinations rate limited (DB) for batch item: $name (same as: " . $existing[0]['name'] . ")");	×
459	self::addToFileCache($hash, $existing[0]['name']);	×
460	self::cleanupRateLimitedHash($hash);	×
461	return FALSE;	×
462	}
463	}
464
465	$batchHashes[$hash] = $name;	×
466	$results[] = [	×
467	'name' => $name,	×
468	'data' => $data,	×
NEW 469	'hash' => $hash,	×
NEW 470	'msgid' => $item['msgid'] ?? NULL,	×
NEW 471	'jobid' => $item['jobid'] ?? NULL	×
472	];	×
473
474	# Small delay between requests.
475	sleep(1);	×
476	}
477
478	# Add successful images to caches.
479	foreach ($results as $result) {	×
480	self::$seenHashes[$result['hash']] = $result['name'];	×
481	self::addToFileCache($result['hash'], $result['name']);	×
482	}
483
NEW 484	return ['results' => $results, 'failed' => $failed];	×
485	}
486
487	/**
488	* Cache an image in ai_images table.
489	* @param string $name The item/job name.
490	* @param string $uid The externaluid.
491	* @param string $hash Image hash.
492	*/
493	public static function cacheImage($name, $uid, $hash) {
494	global $dbhm;	×
495
496	if ($dbhm) {	×
497	$dbhm->preExec(	×
498	"INSERT INTO ai_images (name, externaluid, imagehash) VALUES (?, ?, ?)	×
499	ON DUPLICATE KEY UPDATE externaluid = VALUES(externaluid), imagehash = VALUES(imagehash), created = NOW()",	×
500	[$name, $uid, $hash]	×
501	);	×
502	}
503	}
504
505	/**
506	* Upload image to TUS and cache in ai_images table.
507	* @param string $name The item/job name.
508	* @param string $data Image data.
509	* @param string $hash Image hash.
510	* @return string\|false The externaluid on success, FALSE on failure.
511	*/
512	public static function uploadAndCache($name, $data, $hash) {
513	$t = new Tus();	×
514	$tusUrl = $t->upload(NULL, 'image/jpeg', $data);	×
515
516	if (!$tusUrl) {	×
517	error_log("Failed to upload image to TUS for: $name");	×
518	return FALSE;	×
519	}
520
521	$uid = 'freegletusd-' . basename($tusUrl);	×
522	self::cacheImage($name, $uid, $hash);	×
523
524	return $uid;	×
525	}
526
527	/**
528	* Get the hash of image data.
529	* @param string $data Image data.
530	* @return string MD5 hash.
531	*/
532	public static function getImageHash($data) {
533	return md5($data);	×
534	}
535
536	/**
537	* Clear the in-memory hash cache (mainly for testing).
538	*/
539	public static function clearCache() {
540	self::$seenHashes = [];	×
541	}
542
543	/**
544	* Clear the file-based hash cache (mainly for testing/maintenance).
545	*/
546	public static function clearFileCache() {
547	if (file_exists(self::CACHE_FILE)) {	×
548	@unlink(self::CACHE_FILE);	×
549	}
550	}
551	}

Freegle / iznik-server / #2566

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous