#2585

Committed 02 Feb 2026 12:27PM UTC coverage: 85.462% (-0.1%) from 85.583%

Build # #2585

Build Type

push

Committed by

Commit Message

Merge remote-tracking branch 'origin/master' into feature/incoming-email-migration

Run Details

25583 of 29935 relevant lines covered (85.46%)

30.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.35

/include/integrations/ReachVolunteering.php

<?php

namespace Freegle\Iznik;

use PhpMimeMailParser\Exception;

class ReachVolunteering {
    private $dbhr;
    private $dbhm;
    private $useNewFieldNames;

    public function __construct(LoggedPDO $dbhr, LoggedPDO $dbhm, $useNewFieldNames = TRUE) {
        # Reach switched to new field names on 06/10/2025
        $this->dbhr = $dbhr;
        $this->dbhm = $dbhm;
        $this->useNewFieldNames = $useNewFieldNames;
    }

    private function getFieldMapping() {
        if ($this->useNewFieldNames) {
            return [
                'title' => 'title',
                'date_posted' => 'date_posted',
                'job_id' => 'job_id',
                'summary' => 'summary',
                'description' => 'description',
                'person_description' => 'person_description',
                'person_impact' => 'person_impact',
                'other_details' => 'other_details',
                'town' => 'location',  // Combined location field contains both town and postcode
                'skills' => 'skills',
                'organisation' => 'organisation',
                'causes' => 'causes',
                'activities' => 'activities',
                'objectives' => 'objectives',
                'url' => 'url'
            ];
        } else {
            return [
                'title' => 'title',
                'date_posted' => 'Posting date',
                'job_id' => 'Job id',
                'summary' => 'summary',
                'description' => 'Job description',
                'person_description' => 'Person specification',
                'person_impact' => 'What impact the opportunity will have',
                'other_details' => 'Other details',
                'town' => 'Location',
                'postcode' => 'Location',
                'skills' => 'Required skills',
                'organisation' => 'Organisation',
                'causes' => 'Charity sector',
                'activities' => 'Organisation activities',
                'objectives' => 'Organisation objective',
                'url' => 'Apply url'
            ];
        }
    }

    private function processOpportunity($opp, $fieldMap, &$externalsSeen, &$urlsSeen, &$added, &$updated) {
        // Track this opportunity as seen FIRST, before any early returns
        // This prevents deletion of opportunities that are skipped for processing reasons
        $externalid = "reach-" . $opp[$fieldMap['job_id']];
        $externalsSeen[$externalid] = TRUE;

        // Also track URL to handle migration from old to new format
        $url = $opp[$fieldMap['url']];
        $urlsSeen[$url] = TRUE;

        $postingDate = $opp[$fieldMap['date_posted']];
        $postingAgeInDays = (time() - strtotime($postingDate)) / (60 * 60 * 24);

        if ($postingAgeInDays > Volunteering::EXPIRE_AGE) {
            error_log("...skipping as too old $postingDate");
            return;
        }

        if ($this->useNewFieldNames) {
            // New format: location field contains combined "Town, Postcode, Country" format
            $locationField = $opp[$fieldMap['town']] ?? ''; // 'town' maps to 'location' in new format
            if (preg_match(Utils::POSTCODE_PATTERN, $locationField, $matches)) {
                $pc = strtoupper($matches[0]);
                $loc = $locationField; // Use the full location string
            } else {
                error_log("No postcode in $locationField");
                return;
            }
        } else {
            // Old format: extract postcode from Location field using regex
            $loc = $opp[$fieldMap['town']]; // This is 'Location' field in old format
            if (preg_match(Utils::POSTCODE_PATTERN, $loc, $matches)) {
                $pc = strtoupper($matches[0]);
            } else {
                error_log("No postcode in $loc");
                return;
            }
        }

        error_log("...postcode $pc");

        $l = new Location($this->dbhr, $this->dbhm);
        $pc = $l->findByName($pc);

        if ($pc) {
            $l = new Location($this->dbhr, $this->dbhm, $pc);

            if ($l->getPrivate('type') == 'Postcode') {
                $groups = $l->groupsNear(Location::QUITENEARBY);

                if (count($groups)) {
                    $g = Group::get($this->dbhr, $this->dbhm, $groups[0]);
                    error_log("...on #{$groups[0]} " . $g->getName());

                    if ($g->getSetting('volunteering', 1)) {
                        $url = $opp[$fieldMap['url']];

                        // Check for existing opportunity by contacturl (for migration from old to new format)
                        // or by externalid (for regular updates)
                        // Match by exact URL or base URL (without query params) or externalid
                        $existing = $this->dbhr->preQuery("SELECT id, externalid, contacturl FROM volunteering WHERE contacturl = ? OR externalid = ?", [ $url, $externalid ]);

                        $title = $opp[$fieldMap['title']];

                        $descriptionRaw = $opp[$fieldMap['description']];
                        $html = new \Html2Text\Html2Text($descriptionRaw);
                        $description = $html->getText();

                        $organisation = Utils::presdef($fieldMap['organisation'], $opp, NULL);
                        if ($organisation) {
                            $description = "Posted by $organisation.\n\n" . $description;
                        }

                        # Strip country suffix from location (e.g. ", United Kingdom")
                        $location = preg_replace('/,\s*(United Kingdom|UK|England|Scotland|Wales|Northern Ireland)\s*$/i', '', $loc);
                        $commitment = Utils::presdef($fieldMap['other_details'], $opp, NULL);

                        if (count($existing)) {
                            # Make sure the info is up to date.
                            #
                            # We don't update the photo as there is no good way to
                            # check it hasn't changed.
                            error_log("...updated existing " . $existing[0]['id']);
                            $v = new Volunteering($this->dbhr, $this->dbhm, $existing[0]['id']);
                            $v->setPrivate('title', $title);
                            $v->setPrivate('location', $location);
                            $v->setPrivate('description', $description);
                            $v->setPrivate('contacturl', $url);
                            $v->setPrivate('externalid', $externalid);
                            $v->setPrivate('timecommitment', $commitment);
                            $updated++;
                        } else {
                            $added++;

                            # We don't - create it.
                            $v = new Volunteering($this->dbhr, $this->dbhm);
                            $vid = $v->create(
                                null,
                                $title,
                                FALSE,
                                $location,
                                NULL,
                                NULL,
                                NULL,
                                $url,
                                $description,
                                $commitment,
                                $externalid
                            );

                            error_log("...created as $vid");
                            $added++;

                            $v->addGroup($g->getId());

                            # Get an image if we can.
                            $image = Utils::presdef('Logo url', $opp, NULL);

                            if ($image) {
                                $t = new Tus($this->dbhr, $this->dbhm);
                                $url = $t->upload($image);

                                if ($url) {
                                    $uid = 'freegletusd-' . basename($url);
                                    $this->dbhm->preExec("INSERT INTO volunteering_images (opportunityid, externaluid) VALUES (?,?);", [
                                        $vid,
                                        $uid
                                    ]);
                                }
                            }
                        }
                    } else {
                        error_log("Volunteering not allowed on " . $g->getName());
                    }
                } else {
                    error_log("No groups near $pc");
                }
            } else {
                error_log("Not a postcode $pc");
            }
        } else {
            error_log("Can't find postcode $pc");
        }
    }

    protected function fetchFeedData($feedUrl) {
        $auth = base64_encode(REACH_USER . ":" . REACH_PASSWORD);
        $ctx = stream_context_create(array('http'=> [
            'timeout' => 120,
            "method" => "GET",
            "header" => "Authorization: Basic $auth"
        ]));

        $return = file_get_contents($feedUrl, FALSE, $ctx);

        error_log("Got feed len " . strlen($return));
        error_log(substr($return, 0, 100));

        return $return;
    }

    public function processFeed($feedUrl) {
        $added = 0;
        $updated = 0;
        $deleted = 0;

        $fieldMap = $this->getFieldMapping();

        $return = $this->fetchFeedData($feedUrl);

        if ($return) {
            $data = json_decode($return, TRUE, 512, JSON_INVALID_UTF8_IGNORE);

            if ($data === NULL) {
                throw new Exception("Failed to decode JSON: " . json_last_error_msg());
            }

            $externalsSeen = [];
            $urlsSeen = [];

            if ($this->useNewFieldNames) {
                // New format: top level is array of opportunities
                if (!is_array($data)) {
                    throw new Exception("Expected JSON array for new format, got: " . gettype($data));
                }

                $opps = $data;
                error_log("Found " . count($opps) . " opportunities");

                foreach ($opps as $opp) {
                    // Each item is directly an opportunity object
                    $this->processOpportunity($opp, $fieldMap, $externalsSeen, $urlsSeen, $added, $updated);
                }
            } else {
                // Old format: { "Opportunities": [ { "Opportunity": {} } ] }
                if (Utils::pres('Opportunities', $data)) {
                    $opps = $data['Opportunities'];
                    error_log("Found " . count($opps) . " opportunities");

                    foreach ($opps as $oppWrapper) {
                        $opp = $oppWrapper['Opportunity'];
                        $this->processOpportunity($opp, $fieldMap, $externalsSeen, $urlsSeen, $added, $updated);
                    }
                } else {
                    throw new Exception("JSON is unexpected " . json_last_error_msg());
                }
            }

        } else {
            throw new Exception("Failed to get " . $feedUrl . " with $http_response_header");
        }

        error_log("Added $added");

        # Look for ops which need removing because they aren't on Reach any more.
        # Check both externalid and contacturl to handle migration from old to new format
        $existings = $this->dbhr->preQuery("SELECT id, externalid, contacturl FROM volunteering WHERE externalid LIKE 'reach-%';");

        foreach ($existings as $e) {
            // Don't delete if we've seen either the externalid OR the URL (for migration compatibility)
            $seenByExternalId = array_key_exists($e['externalid'], $externalsSeen);
            $seenByUrl = $e['contacturl'] && array_key_exists($e['contacturl'], $urlsSeen);

            if (!$seenByExternalId && !$seenByUrl) {
                error_log("...deleting old {$e['id']}, externalid={$e['externalid']}, url={$e['contacturl']}");
                $cv = new Volunteering($this->dbhr, $this->dbhm, $e['id']);
                $cv->setPrivate('deleted', 1);
                $deleted++;
            } else {
                $matchedBy = $seenByExternalId ? 'externalid' : 'url';
                error_log("...keeping {$e['id']} (matched by $matchedBy)");
            }
        }

        error_log("Added $added, updated $updated, deleted $deleted");
    }
}

1	<?php
2
3	namespace Freegle\Iznik;
4
5	use PhpMimeMailParser\Exception;
6
7	class ReachVolunteering {
8	private $dbhr;
9	private $dbhm;
10	private $useNewFieldNames;
11
12	public function __construct(LoggedPDO $dbhr, LoggedPDO $dbhm, $useNewFieldNames = TRUE) {
13	# Reach switched to new field names on 06/10/2025
14	$this->dbhr = $dbhr;	7✔
15	$this->dbhm = $dbhm;	7✔
16	$this->useNewFieldNames = $useNewFieldNames;	7✔
17	}
18
19	private function getFieldMapping() {
20	if ($this->useNewFieldNames) {	7✔
21	return [	4✔
22	'title' => 'title',	4✔
23	'date_posted' => 'date_posted',	4✔
24	'job_id' => 'job_id',	4✔
25	'summary' => 'summary',	4✔
26	'description' => 'description',	4✔
27	'person_description' => 'person_description',	4✔
28	'person_impact' => 'person_impact',	4✔
29	'other_details' => 'other_details',	4✔
30	'town' => 'location', // Combined location field contains both town and postcode	4✔
31	'skills' => 'skills',	4✔
32	'organisation' => 'organisation',	4✔
33	'causes' => 'causes',	4✔
34	'activities' => 'activities',	4✔
35	'objectives' => 'objectives',	4✔
36	'url' => 'url'	4✔
37	];	4✔
38	} else {
39	return [	4✔
40	'title' => 'title',	4✔
41	'date_posted' => 'Posting date',	4✔
42	'job_id' => 'Job id',	4✔
43	'summary' => 'summary',	4✔
44	'description' => 'Job description',	4✔
45	'person_description' => 'Person specification',	4✔
46	'person_impact' => 'What impact the opportunity will have',	4✔
47	'other_details' => 'Other details',	4✔
48	'town' => 'Location',	4✔
49	'postcode' => 'Location',	4✔
50	'skills' => 'Required skills',	4✔
51	'organisation' => 'Organisation',	4✔
52	'causes' => 'Charity sector',	4✔
53	'activities' => 'Organisation activities',	4✔
54	'objectives' => 'Organisation objective',	4✔
55	'url' => 'Apply url'	4✔
56	];	4✔
57	}
58	}
59
60	private function processOpportunity($opp, $fieldMap, &$externalsSeen, &$urlsSeen, &$added, &$updated) {
61	// Track this opportunity as seen FIRST, before any early returns
62	// This prevents deletion of opportunities that are skipped for processing reasons
63	$externalid = "reach-" . $opp[$fieldMap['job_id']];	6✔
64	$externalsSeen[$externalid] = TRUE;	6✔
65
66	// Also track URL to handle migration from old to new format
67	$url = $opp[$fieldMap['url']];	6✔
68	$urlsSeen[$url] = TRUE;	6✔
69
70	$postingDate = $opp[$fieldMap['date_posted']];	6✔
71	$postingAgeInDays = (time() - strtotime($postingDate)) / (60 * 60 * 24);	6✔
72
73	if ($postingAgeInDays > Volunteering::EXPIRE_AGE) {	6✔
74	error_log("...skipping as too old $postingDate");	1✔
75	return;	1✔
76	}
77
78	if ($this->useNewFieldNames) {	5✔
79	// New format: location field contains combined "Town, Postcode, Country" format
80	$locationField = $opp[$fieldMap['town']] ?? ''; // 'town' maps to 'location' in new format	3✔
81	if (preg_match(Utils::POSTCODE_PATTERN, $locationField, $matches)) {	3✔
82	$pc = strtoupper($matches[0]);	3✔
83	$loc = $locationField; // Use the full location string	3✔
84	} else {
85	error_log("No postcode in $locationField");	×
86	return;	3✔
87	}
88	} else {
89	// Old format: extract postcode from Location field using regex
90	$loc = $opp[$fieldMap['town']]; // This is 'Location' field in old format	2✔
91	if (preg_match(Utils::POSTCODE_PATTERN, $loc, $matches)) {	2✔
92	$pc = strtoupper($matches[0]);	2✔
93	} else {
94	error_log("No postcode in $loc");	×
95	return;	×
96	}
97	}
98
99	error_log("...postcode $pc");	5✔
100
101	$l = new Location($this->dbhr, $this->dbhm);	5✔
102	$pc = $l->findByName($pc);	5✔
103
104	if ($pc) {	5✔
105	$l = new Location($this->dbhr, $this->dbhm, $pc);	5✔
106
107	if ($l->getPrivate('type') == 'Postcode') {	5✔
108	$groups = $l->groupsNear(Location::QUITENEARBY);	5✔
109
110	if (count($groups)) {	5✔
111	$g = Group::get($this->dbhr, $this->dbhm, $groups[0]);	5✔
112	error_log("...on #{$groups[0]} " . $g->getName());	5✔
113
114	if ($g->getSetting('volunteering', 1)) {	5✔
115	$url = $opp[$fieldMap['url']];	5✔
116
117	// Check for existing opportunity by contacturl (for migration from old to new format)
118	// or by externalid (for regular updates)
119	// Match by exact URL or base URL (without query params) or externalid
120	$existing = $this->dbhr->preQuery("SELECT id, externalid, contacturl FROM volunteering WHERE contacturl = ? OR externalid = ?", [ $url, $externalid ]);	5✔
121
122	$title = $opp[$fieldMap['title']];	5✔
123
124	$descriptionRaw = $opp[$fieldMap['description']];	5✔
125	$html = new \Html2Text\Html2Text($descriptionRaw);	5✔
126	$description = $html->getText();	5✔
127
128	$organisation = Utils::presdef($fieldMap['organisation'], $opp, NULL);	5✔
129	if ($organisation) {	5✔
130	$description = "Posted by $organisation.\n\n" . $description;	3✔
131	}
132
133	# Strip country suffix from location (e.g. ", United Kingdom")
134	$location = preg_replace('/,\s(United Kingdom\|UK\|England\|Scotland\|Wales\|Northern Ireland)\s$/i', '', $loc);	5✔
135	$commitment = Utils::presdef($fieldMap['other_details'], $opp, NULL);	5✔
136
137	if (count($existing)) {	5✔
138	# Make sure the info is up to date.
139	#
140	# We don't update the photo as there is no good way to
141	# check it hasn't changed.
142	error_log("...updated existing " . $existing[0]['id']);	×
143	$v = new Volunteering($this->dbhr, $this->dbhm, $existing[0]['id']);	×
144	$v->setPrivate('title', $title);	×
145	$v->setPrivate('location', $location);	×
146	$v->setPrivate('description', $description);	×
147	$v->setPrivate('contacturl', $url);	×
148	$v->setPrivate('externalid', $externalid);	×
149	$v->setPrivate('timecommitment', $commitment);	×
150	$updated++;	×
151	} else {
152	$added++;	5✔
153
154	# We don't - create it.
155	$v = new Volunteering($this->dbhr, $this->dbhm);	5✔
156	$vid = $v->create(	5✔
157	null,	5✔
158	$title,	5✔
159	FALSE,	5✔
160	$location,	5✔
161	NULL,	5✔
162	NULL,	5✔
163	NULL,	5✔
164	$url,	5✔
165	$description,	5✔
166	$commitment,	5✔
167	$externalid	5✔
168	);	5✔
169
170	error_log("...created as $vid");	5✔
171	$added++;	5✔
172
173	$v->addGroup($g->getId());	5✔
174
175	# Get an image if we can.
176	$image = Utils::presdef('Logo url', $opp, NULL);	5✔
177
178	if ($image) {	5✔
179	$t = new Tus($this->dbhr, $this->dbhm);	×
180	$url = $t->upload($image);	×
181
182	if ($url) {	×
183	$uid = 'freegletusd-' . basename($url);	×
184	$this->dbhm->preExec("INSERT INTO volunteering_images (opportunityid, externaluid) VALUES (?,?);", [	5✔
185	$vid,	5✔
186	$uid	5✔
187	]);	5✔
188	}
189	}
190	}
191	} else {
192	error_log("Volunteering not allowed on " . $g->getName());	5✔
193	}
194	} else {
195	error_log("No groups near $pc");	5✔
196	}
197	} else {
198	error_log("Not a postcode $pc");	5✔
199	}
200	} else {
201	error_log("Can't find postcode $pc");	×
202	}
203	}
204
205	protected function fetchFeedData($feedUrl) {
206	$auth = base64_encode(REACH_USER . ":" . REACH_PASSWORD);	×
207	$ctx = stream_context_create(array('http'=> [	×
208	'timeout' => 120,	×
209	"method" => "GET",	×
210	"header" => "Authorization: Basic $auth"	×
211	]));	×
212
213	$return = file_get_contents($feedUrl, FALSE, $ctx);	×
214
215	error_log("Got feed len " . strlen($return));	×
216	error_log(substr($return, 0, 100));	×
217
218	return $return;	×
219	}
220
221	public function processFeed($feedUrl) {
222	$added = 0;	6✔
223	$updated = 0;	6✔
224	$deleted = 0;	6✔
225
226	$fieldMap = $this->getFieldMapping();	6✔
227
228	$return = $this->fetchFeedData($feedUrl);	6✔
229
230	if ($return) {	6✔
231	$data = json_decode($return, TRUE, 512, JSON_INVALID_UTF8_IGNORE);	6✔
232
233	if ($data === NULL) {	6✔
234	throw new Exception("Failed to decode JSON: " . json_last_error_msg());	×
235	}
236
237	$externalsSeen = [];	6✔
238	$urlsSeen = [];	6✔
239
240	if ($this->useNewFieldNames) {	6✔
241	// New format: top level is array of opportunities
242	if (!is_array($data)) {	3✔
243	throw new Exception("Expected JSON array for new format, got: " . gettype($data));	×
244	}
245
246	$opps = $data;	3✔
247	error_log("Found " . count($opps) . " opportunities");	3✔
248
249	foreach ($opps as $opp) {	3✔
250	// Each item is directly an opportunity object
251	$this->processOpportunity($opp, $fieldMap, $externalsSeen, $urlsSeen, $added, $updated);	3✔
252	}
253	} else {
254	// Old format: { "Opportunities": [ { "Opportunity": {} } ] }
255	if (Utils::pres('Opportunities', $data)) {	3✔
256	$opps = $data['Opportunities'];	3✔
257	error_log("Found " . count($opps) . " opportunities");	3✔
258
259	foreach ($opps as $oppWrapper) {	3✔
260	$opp = $oppWrapper['Opportunity'];	3✔
261	$this->processOpportunity($opp, $fieldMap, $externalsSeen, $urlsSeen, $added, $updated);	3✔
262	}
263	} else {
264	throw new Exception("JSON is unexpected " . json_last_error_msg());	6✔
265	}
266	}
267
268	} else {
269	throw new Exception("Failed to get " . $feedUrl . " with $http_response_header");	×
270	}
271
272	error_log("Added $added");	6✔
273
274	# Look for ops which need removing because they aren't on Reach any more.
275	# Check both externalid and contacturl to handle migration from old to new format
276	$existings = $this->dbhr->preQuery("SELECT id, externalid, contacturl FROM volunteering WHERE externalid LIKE 'reach-%';");	6✔
277
278	foreach ($existings as $e) {	6✔
279	// Don't delete if we've seen either the externalid OR the URL (for migration compatibility)
280	$seenByExternalId = array_key_exists($e['externalid'], $externalsSeen);	5✔
281	$seenByUrl = $e['contacturl'] && array_key_exists($e['contacturl'], $urlsSeen);	5✔
282
283	if (!$seenByExternalId && !$seenByUrl) {	5✔
284	error_log("...deleting old {$e['id']}, externalid={$e['externalid']}, url={$e['contacturl']}");	×
285	$cv = new Volunteering($this->dbhr, $this->dbhm, $e['id']);	×
286	$cv->setPrivate('deleted', 1);	×
287	$deleted++;	×
288	} else {
289	$matchedBy = $seenByExternalId ? 'externalid' : 'url';	5✔
290	error_log("...keeping {$e['id']} (matched by $matchedBy)");	5✔
291	}
292	}
293
294	error_log("Added $added, updated $updated, deleted $deleted");	6✔
295	}
296	}

Freegle / iznik-server / #2585

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous