#2567

Committed 12 Jan 2026 06:33AM UTC coverage: 87.991% (-0.02%) from 88.008%

Build # #2567

Build Type

push

Committed by

Commit Message

Fix illustration scripts getting stuck in infinite loop on failing items

Changes:
- fetchBatch() now returns partial results instead of FALSE when individual
  items fail to return data (only true rate-limiting fails the batch)
- Add file-based tracking for items that repeatedly fail (/tmp/pollinations_failed.json)
- Items that fail 3 times are skipped for 1 day before retrying
- Both messages_illustrations.php and jobs_illustrations.php updated to:
  - Skip items that have exceeded failure threshold
  - Record failures for items that don't return data
  - Process successful results from partial batches

This prevents a single problematic item (like "2 toilet seat trainers") from
blocking all illustration generation indefinitely.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Run Details

0 of 47 new or added lines in 1 file covered. (0.0%)

114 existing lines in 3 files now uncovered.

26298 of 29887 relevant lines covered (87.99%)

31.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.41

/include/misc/Log.php

<?php
namespace Freegle\Iznik;

require_once(dirname(__FILE__) . '/Loki.php');

# Logging.  This is not guaranteed against loss in the event of serious failure.
class Log
{
    /** @var  $dbhr LoggedPDO */
    var $dbhr;
    /** @var  $dbhm LoggedPDO */
    var $dbhm;

    # Log types must match the enumeration in the logs table.
    const TYPE_GROUP = 'Group';
    const TYPE_USER = 'User';
    const TYPE_MESSAGE = 'Message';
    const TYPE_CONFIG = 'Config';
    const TYPE_STDMSG = 'StdMsg';
    const TYPE_BULKOP = 'BulkOp';
    const TYPE_LOCATION = 'Location';
    const TYPE_CHAT = 'Chat';

    const SUBTYPE_CREATED = 'Created';
    const SUBTYPE_DELETED = 'Deleted';
    const SUBTYPE_EDIT = 'Edit';
    const SUBTYPE_APPROVED = 'Approved';
    const SUBTYPE_REJECTED = 'Rejected';
    const SUBTYPE_RECEIVED = 'Received';
    const SUBTYPE_NOTSPAM = 'NotSpam';
    const SUBTYPE_HOLD = 'Hold';
    const SUBTYPE_RELEASE = 'Release';
    const SUBTYPE_FAILURE = 'Failure';
    const SUBTYPE_JOINED = 'Joined';
    const SUBTYPE_APPLIED = 'Applied';
    const SUBTYPE_LEFT = 'Left';
    const SUBTYPE_REPLIED = 'Replied';
    const SUBTYPE_MAILED = 'Mailed';
    const SUBTYPE_LOGIN = 'Login';
    const SUBTYPE_LOGOUT = 'Logout';
    const SUBTYPE_CLASSIFIED_SPAM = 'ClassifiedSpam';
    const SUBTYPE_SUSPECT = 'Suspect';
    const SUBTYPE_SENT = 'Sent';
    const SUBTYPE_OUR_POSTING_STATUS = 'OurPostingStatus';
    const SUBTYPE_OUR_EMAIL_FREQUENCY = 'OurEmailFrequency';
    const SUBTYPE_ROLE_CHANGE = 'RoleChange';
    const SUBTYPE_MERGED = 'Merged';
    const SUBTYPE_SPLIT = 'Split';
    const SUBTYPE_MAILOFF = 'MailOff';
    const SUBTYPE_EVENTSOFF = 'EventsOff';
    const SUBTYPE_NEWSLETTERSOFF = 'NewslettersOff';
    const SUBTYPE_RELEVANTOFF = 'RelevantOff';
    const SUBTYPE_VOLUNTEERSOFF = 'VolunteersOff';
    const SUBTYPE_BOUNCE = 'Bounce';
    const SUBTYPE_SUSPEND_MAIL = 'SuspendMail';
    const SUBTYPE_AUTO_REPOSTED = 'Autoreposted';
    const SUBTYPE_OUTCOME = 'Outcome';
    const SUBTYPE_NOTIFICATIONOFF = 'NotificationOff';
    const SUBTYPE_AUTO_APPROVED = 'Autoapproved';
    const SUBTYPE_UNBOUNCE = 'Unbounce';
    const SUBTYPE_WORRYWORDS = 'WorryWords';
    const SUBTYPE_POSTCODECHANGE = 'PostcodeChange';
    const SUBTYPE_REPOST = 'Repost';
    
    const LOG_USER_CACHE_SIZE = 1000;

    function __construct($dbhr, $dbhm)
    {
        $this->dbhr = $dbhr;
        $this->dbhm = $dbhm;
    }

    public function log($params) {
        # Insert the current timestamp, so that the time in the log is when the log was made, rather than when
        # it percolates through the background processing.
        $params['timestamp'] = date("Y-m-d H:i:s", time());

        # We assume that the parameters passed match fields in the logs table.
        # If they don't, the caller is at fault and should be taken out and shot.
        $q = [];
        foreach ($params as $key => $val) {
            $q[] = !is_null($val) ? $this->dbhm->quote($val) : 'NULL';
        }

        $atts = implode('`,`', array_keys($params));
        $vals = implode(',', $q);

        $sql = "INSERT INTO logs (`$atts`) VALUES ($vals);";

        # No need to check return code - if it doesn't work, nobody dies.
        $this->dbhm->background($sql);

        # Also log to Loki (fire-and-forget, async).
        $loki = Loki::getInstance();
        if ($loki->isEnabled()) {
            $loki->logFromLogsTable($params);
        }
    }

    public function get($types, $subtypes, $groupid, $userid, $date, $search, $limit, &$ctx, $uid = NULL) {
        $limit = intval($limit);

        $groupq = $groupid ? " groupid = $groupid " : '1 = 1 ';
        $userq = $userid ? " groupid = $groupid " : '1 = 1 ';
        $typeq = $types ? (" AND logs.type IN ('" . implode("','", $types) . "') ") : '';
        $subtypeq = $subtypes ? (" AND `subtype` IN ('" . implode("','", $subtypes) . "') ") : '';
        $mysqltime = date("Y-m-d", strtotime("midnight $date days ago"));
        $dateq = $date ? " AND timestamp >= '$mysqltime' " : '';

        $searchq = $this->dbhr->quote("%$search%");

        $idq = Utils::pres('id', $ctx) ? (" AND logs.id < " . intval($ctx['id']) . " ") : '';
        
        # We might have consecutive logs for the same messages/users, so try to speed that up.
        if ($uid) {
            $sql = "SELECT logs.* FROM logs 
                LEFT JOIN users ON users.id = logs.user 
                LEFT JOIN messages ON messages.id = logs.msgid
                WHERE $groupq $idq $typeq $subtypeq $dateq AND 
                (logs.user = $uid OR logs.byuser = $uid)
                ORDER BY logs.id DESC LIMIT $limit";
        } else if (!$search) {
            # This is simple.
            $sql = "SELECT * FROM logs WHERE $groupq $idq $typeq $subtypeq $dateq ORDER BY id DESC LIMIT $limit";
        } else  {
            # This is complex.  We want to search in the various user names, and the message
            # subject.  And the email - and people might search using an email belonging to a member but which
            # isn't the fromaddr of any messages.  So first expand the email.
            $sql = "SELECT users_emails.userid FROM users_emails INNER JOIN memberships ON groupid = $groupid AND memberships.userid = users_emails.userid AND email LIKE $searchq;";
            $emails = $this->dbhr->preQuery($sql);
            $uids = [];

            foreach ($emails as $email) {
                $uids[] = $email['userid'];
            }

            $uidq = count($uids) > 0 ? (" OR logs.user IN (" . implode(',', $uids) . ") OR logs.byuser IN (" . implode(',', $uids) . ")") : '';

            $sql = "SELECT logs.* FROM logs 
                LEFT JOIN users ON users.id = logs.user 
                LEFT JOIN messages ON messages.id = logs.msgid
                WHERE $groupq $idq $typeq $subtypeq $dateq AND 
                ((users.firstname LIKE $searchq OR users.lastname LIKE $searchq OR users.fullname LIKE $searchq OR CONCAT(users.firstname, ' ', users.lastname) LIKE $searchq) OR
                 (messages.subject LIKE $searchq $uidq))
                ORDER BY logs.id DESC LIMIT $limit";
        }

        $logs = $this->dbhr->preQuery($sql);
        $total = count($logs);
        #error_log("...total logs $total");
        $count = 0;

        $uids = array_filter(array_unique(array_merge(array_column($logs, 'user'), array_column($logs, 'byuser'))));
        $u = new User($this->dbhr, $this->dbhm);
        $users = [];
        if (count($uids)) {
            $users = $u->getPublicsById($uids, NULL, NULL, FALSE, TRUE, TRUE);
        }

        $mids = array_filter(array_column($logs, 'msgid'));
        $msgs = [];

        if (count($mids)) {
            $ms = $this->dbhr->preQuery("SELECT id, subject, sourceheader, envelopeto FROM messages WHERE id IN (" . implode(',', $mids) .");");
            foreach ($ms as $m) {
                $msgs[$m['id']] = $m;
            }
        }

        foreach ($logs as &$log) {
            $count++;

            if ($count % 1000 == 0) {
                #error_log("...$count / $total");
            }

            $log['timestamp'] = Utils::ISODate($log['timestamp']);

            if (Utils::pres('user', $log)) {
                $id = $log['user'];
                $log['user'] = Utils::presdef($log['user'], $users, NULL);

                if (!$log['user']) {
                    $log['user'] = User::purgedUser($id);
                }
            }

            if (Utils::pres('byuser', $log)) {
                $id = $log['byuser'];
                $log['byuser'] = Utils::presdef($log['byuser'], $users, NULL);

                if (!$log['byuser']) {
                    $log['byuser'] = User::purgedUser($id);
                }
            }

            if (Utils::pres('msgid', $log)) {
                $log['message'] = Utils::presdef($log['msgid'], $msgs, NULL);
            }

            if ($log['subtype'] == Log::SUBTYPE_OUTCOME && $log['text']) {
                # Trim the long text leaving just the outcome.
                $p = strpos($log['text'], ' ');

                if ($p) {
                    $log['text'] = substr($log['text'], 0, $p);
                    $log['text'] = str_replace(':', '', $log['text']);
                }
            }

            $ctx['id'] = $log['id'];
        }

        return($logs);
    }

    public function deleteLogsForMessage($msgid) {
        # Need to background as the original log request might be backgrounded.
        $this->dbhm->background("DELETE FROM logs WHERE msgid = $msgid");
    }
}

1	<?php
2	namespace Freegle\Iznik;
3
4	require_once(dirname(__FILE__) . '/Loki.php');	93✔
5
6	# Logging. This is not guaranteed against loss in the event of serious failure.
7	class Log
8	{
9	/** @var $dbhr LoggedPDO */
10	var $dbhr;
11	/** @var $dbhm LoggedPDO */
12	var $dbhm;
13
14	# Log types must match the enumeration in the logs table.
15	const TYPE_GROUP = 'Group';
16	const TYPE_USER = 'User';
17	const TYPE_MESSAGE = 'Message';
18	const TYPE_CONFIG = 'Config';
19	const TYPE_STDMSG = 'StdMsg';
20	const TYPE_BULKOP = 'BulkOp';
21	const TYPE_LOCATION = 'Location';
22	const TYPE_CHAT = 'Chat';
23
24	const SUBTYPE_CREATED = 'Created';
25	const SUBTYPE_DELETED = 'Deleted';
26	const SUBTYPE_EDIT = 'Edit';
27	const SUBTYPE_APPROVED = 'Approved';
28	const SUBTYPE_REJECTED = 'Rejected';
29	const SUBTYPE_RECEIVED = 'Received';
30	const SUBTYPE_NOTSPAM = 'NotSpam';
31	const SUBTYPE_HOLD = 'Hold';
32	const SUBTYPE_RELEASE = 'Release';
33	const SUBTYPE_FAILURE = 'Failure';
34	const SUBTYPE_JOINED = 'Joined';
35	const SUBTYPE_APPLIED = 'Applied';
36	const SUBTYPE_LEFT = 'Left';
37	const SUBTYPE_REPLIED = 'Replied';
38	const SUBTYPE_MAILED = 'Mailed';
39	const SUBTYPE_LOGIN = 'Login';
40	const SUBTYPE_LOGOUT = 'Logout';
41	const SUBTYPE_CLASSIFIED_SPAM = 'ClassifiedSpam';
42	const SUBTYPE_SUSPECT = 'Suspect';
43	const SUBTYPE_SENT = 'Sent';
44	const SUBTYPE_OUR_POSTING_STATUS = 'OurPostingStatus';
45	const SUBTYPE_OUR_EMAIL_FREQUENCY = 'OurEmailFrequency';
46	const SUBTYPE_ROLE_CHANGE = 'RoleChange';
47	const SUBTYPE_MERGED = 'Merged';
48	const SUBTYPE_SPLIT = 'Split';
49	const SUBTYPE_MAILOFF = 'MailOff';
50	const SUBTYPE_EVENTSOFF = 'EventsOff';
51	const SUBTYPE_NEWSLETTERSOFF = 'NewslettersOff';
52	const SUBTYPE_RELEVANTOFF = 'RelevantOff';
53	const SUBTYPE_VOLUNTEERSOFF = 'VolunteersOff';
54	const SUBTYPE_BOUNCE = 'Bounce';
55	const SUBTYPE_SUSPEND_MAIL = 'SuspendMail';
56	const SUBTYPE_AUTO_REPOSTED = 'Autoreposted';
57	const SUBTYPE_OUTCOME = 'Outcome';
58	const SUBTYPE_NOTIFICATIONOFF = 'NotificationOff';
59	const SUBTYPE_AUTO_APPROVED = 'Autoapproved';
60	const SUBTYPE_UNBOUNCE = 'Unbounce';
61	const SUBTYPE_WORRYWORDS = 'WorryWords';
62	const SUBTYPE_POSTCODECHANGE = 'PostcodeChange';
63	const SUBTYPE_REPOST = 'Repost';
64
65	const LOG_USER_CACHE_SIZE = 1000;
66
67	function __construct($dbhr, $dbhm)
68	{
69	$this->dbhr = $dbhr;	678✔
70	$this->dbhm = $dbhm;	678✔
71	}
72
73	public function log($params) {
74	# Insert the current timestamp, so that the time in the log is when the log was made, rather than when
75	# it percolates through the background processing.
76	$params['timestamp'] = date("Y-m-d H:i:s", time());	642✔
77
78	# We assume that the parameters passed match fields in the logs table.
79	# If they don't, the caller is at fault and should be taken out and shot.
80	$q = [];	642✔
81	foreach ($params as $key => $val) {	642✔
82	$q[] = !is_null($val) ? $this->dbhm->quote($val) : 'NULL';	642✔
83	}
84
85	$atts = implode('`,`', array_keys($params));	642✔
86	$vals = implode(',', $q);	642✔
87
88	$sql = "INSERT INTO logs (`$atts`) VALUES ($vals);";	642✔
89
90	# No need to check return code - if it doesn't work, nobody dies.
91	$this->dbhm->background($sql);	642✔
92
93	# Also log to Loki (fire-and-forget, async).
94	$loki = Loki::getInstance();	642✔
95	if ($loki->isEnabled()) {	642✔
96	$loki->logFromLogsTable($params);	×
97	}
98	}
99
100	public function get($types, $subtypes, $groupid, $userid, $date, $search, $limit, &$ctx, $uid = NULL) {
101	$limit = intval($limit);	8✔
102
103	$groupq = $groupid ? " groupid = $groupid " : '1 = 1 ';	8✔
104	$userq = $userid ? " groupid = $groupid " : '1 = 1 ';	8✔
105	$typeq = $types ? (" AND logs.type IN ('" . implode("','", $types) . "') ") : '';	8✔
106	$subtypeq = $subtypes ? (" AND `subtype` IN ('" . implode("','", $subtypes) . "') ") : '';	8✔
107	$mysqltime = date("Y-m-d", strtotime("midnight $date days ago"));	8✔
108	$dateq = $date ? " AND timestamp >= '$mysqltime' " : '';	8✔
109
110	$searchq = $this->dbhr->quote("%$search%");	8✔
111
112	$idq = Utils::pres('id', $ctx) ? (" AND logs.id < " . intval($ctx['id']) . " ") : '';	8✔
113
114	# We might have consecutive logs for the same messages/users, so try to speed that up.
115	if ($uid) {	8✔
116	$sql = "SELECT logs.* FROM logs	7✔
117	LEFT JOIN users ON users.id = logs.user
118	LEFT JOIN messages ON messages.id = logs.msgid
119	WHERE $groupq $idq $typeq $subtypeq $dateq AND	7✔
120	(logs.user = $uid OR logs.byuser = $uid)	7✔
121	ORDER BY logs.id DESC LIMIT $limit";	7✔
122	} else if (!$search) {	1✔
123	# This is simple.
124	$sql = "SELECT * FROM logs WHERE $groupq $idq $typeq $subtypeq $dateq ORDER BY id DESC LIMIT $limit";	1✔
125	} else {
126	# This is complex. We want to search in the various user names, and the message
127	# subject. And the email - and people might search using an email belonging to a member but which
128	# isn't the fromaddr of any messages. So first expand the email.
129	$sql = "SELECT users_emails.userid FROM users_emails INNER JOIN memberships ON groupid = $groupid AND memberships.userid = users_emails.userid AND email LIKE $searchq;";	1✔
130	$emails = $this->dbhr->preQuery($sql);	1✔
131	$uids = [];	1✔
132
133	foreach ($emails as $email) {	1✔
134	$uids[] = $email['userid'];	1✔
135	}
136
137	$uidq = count($uids) > 0 ? (" OR logs.user IN (" . implode(',', $uids) . ") OR logs.byuser IN (" . implode(',', $uids) . ")") : '';	1✔
138
139	$sql = "SELECT logs.* FROM logs	1✔
140	LEFT JOIN users ON users.id = logs.user
141	LEFT JOIN messages ON messages.id = logs.msgid
142	WHERE $groupq $idq $typeq $subtypeq $dateq AND	1✔
143	((users.firstname LIKE $searchq OR users.lastname LIKE $searchq OR users.fullname LIKE $searchq OR CONCAT(users.firstname, ' ', users.lastname) LIKE $searchq) OR	1✔
144	(messages.subject LIKE $searchq $uidq))	1✔
145	ORDER BY logs.id DESC LIMIT $limit";	1✔
146	}
147
148	$logs = $this->dbhr->preQuery($sql);	8✔
149	$total = count($logs);	8✔
150	#error_log("...total logs $total");
151	$count = 0;	8✔
152
153	$uids = array_filter(array_unique(array_merge(array_column($logs, 'user'), array_column($logs, 'byuser'))));	8✔
154	$u = new User($this->dbhr, $this->dbhm);	8✔
155	$users = [];	8✔
156	if (count($uids)) {	8✔
157	$users = $u->getPublicsById($uids, NULL, NULL, FALSE, TRUE, TRUE);	8✔
158	}
159
160	$mids = array_filter(array_column($logs, 'msgid'));	8✔
161	$msgs = [];	8✔
162
163	if (count($mids)) {	8✔
164	$ms = $this->dbhr->preQuery("SELECT id, subject, sourceheader, envelopeto FROM messages WHERE id IN (" . implode(',', $mids) .");");	1✔
165	foreach ($ms as $m) {	1✔
166	$msgs[$m['id']] = $m;	1✔
167	}
168	}
169
170	foreach ($logs as &$log) {	8✔
171	$count++;	8✔
172
173	if ($count % 1000 == 0) {	8✔
174	#error_log("...$count / $total");
175	}
176
177	$log['timestamp'] = Utils::ISODate($log['timestamp']);	8✔
178
179	if (Utils::pres('user', $log)) {	8✔
180	$id = $log['user'];	8✔
181	$log['user'] = Utils::presdef($log['user'], $users, NULL);	8✔
182
183	if (!$log['user']) {	8✔
184	$log['user'] = User::purgedUser($id);	1✔
185	}
186	}
187
188	if (Utils::pres('byuser', $log)) {	8✔
189	$id = $log['byuser'];	7✔
190	$log['byuser'] = Utils::presdef($log['byuser'], $users, NULL);	7✔
191
192	if (!$log['byuser']) {	7✔
193	$log['byuser'] = User::purgedUser($id);	×
194	}
195	}
196
197	if (Utils::pres('msgid', $log)) {	8✔
198	$log['message'] = Utils::presdef($log['msgid'], $msgs, NULL);	1✔
199	}
200
201	if ($log['subtype'] == Log::SUBTYPE_OUTCOME && $log['text']) {	8✔
202	# Trim the long text leaving just the outcome.
UNCOV 203	$p = strpos($log['text'], ' ');	×
204
UNCOV 205	if ($p) {	×
UNCOV 206	$log['text'] = substr($log['text'], 0, $p);	×
UNCOV 207	$log['text'] = str_replace(':', '', $log['text']);	×
208	}
209	}
210
211	$ctx['id'] = $log['id'];	8✔
212	}
213
214	return($logs);	8✔
215	}
216
217	public function deleteLogsForMessage($msgid) {
218	# Need to background as the original log request might be backgrounded.
219	$this->dbhm->background("DELETE FROM logs WHERE msgid = $msgid");	3✔
220	}
221	}

Freegle / iznik-server / #2567

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous