17108989360

Committed 20 Aug 2025 08:00PM UTC coverage: 68.598% (-1.6%) from 70.247%

Build # 17108989360

Build Type

push

github

Committed by

Michael-Schaer

Commit Message

[FEAT] Sync from mailchimp to crm in specific cases

Run Details

197 of 333 new or added lines in 10 files covered. (59.16%)

4 existing lines in 2 files now uncovered.

959 of 1398 relevant lines covered (68.6%)

11.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.59

/app/Synchronizer/MailchimpToCrmCronSynchronizer.php

<?php

namespace App\Synchronizer;

use App\Http\MailChimpClient;
use App\Exceptions\ConfigException;
use App\Synchronizer\LogTrait;
use Psr\Http\Message\ResponseInterface;

// Example member data from getListMembers():
// "{"email_address":"test@example.com",
// "id":"55502f40dc8b7c769880b10874abc9d0",
// "status":"subscribed",
// "merge_fields":{
//     "FNAME":"Test",
//     "LNAME":"User",
//     "GENDER":"m",
//     "WEBLINGID":"11195",
//     "NOTES_CH":""},
// "last_changed":"2025-04-11T15:39:31+00:00",
// "tags":[
//     {"id":6289607,"name":"member"},
//     {"id":6289608,"name":"donor"},
//     {"id":6289612,"name":"Region Olten"},
//     {"id":6289618,"name":"climate"},
//     {"id":6289625,"name":"Thal-G\u00e4u"}
// ]}"

/**
 * Batch synchronizer for pulling changed Mailchimp subscribers and upserting them into the CRM
 * when executed via cron or a CLI command. Designed to reduce CRM load and licensing overhead by
 * only processing relevant members.
 *
 * Applies filters and processes matching members in batches.
 */
class MailchimpToCrmCronSynchronizer extends MailchimpToCrmSynchronizer
{
    use LogTrait;

    private array $languageTags;
    private string $mailchimpKeyOfCrmId;
    private int $groupForNewMembers;
    private array $interestsToSync;

    public function __construct(string $configFileName)
    {
        parent::__construct($configFileName);

        if (!$this->config->isUpsertToCrmEnabled()) {
            throw new ConfigException('Upsert to CRM is disabled. Please enable it in the config file.');
        }

        $this->languageTags = $this->config->getLanguageTagsFromConfig();
        $this->mailchimpKeyOfCrmId = $this->config->getMailchimpKeyOfCrmId();
        $this->groupForNewMembers = $this->config->getGroupForNewMembers();

        $this->interestsToSync = $this->config->getInterestsToSync();
        if (empty($this->interestsToSync)) {
            throw new ConfigException('No interests configured for sync. Please configure it in the config file.');
        }
    }

    /**
     * Synchronize all changed Mailchimp members to CRM
     *
     * @param int $batchSize Number of members to process per batch
     * @param int $limit Maximum number of members to process (0 for no limit)
     * @return array Statistics about the synchronization
     * @throws \Exception
     */
    public function syncAll(int $batchSize = 100, int $limit = 0): array
    {
        $this->log('info', 'Starting Mailchimp to CRM synchronization');

        $offset = 0;
        $totalProcessed = 0;
        $totalSuccess = 0;
        $totalFailed = 0;
        $hasMore = true;
        $requestFilterParams = $this->getRequestFilterParams();

        while ($hasMore && ($limit === 0 || $totalProcessed < $limit)) {
            $fetchCount = $limit > 0 ? min($batchSize, $limit - $totalProcessed) : $batchSize;

            try {
                $members = $this->mcClient->getListMembers($fetchCount, $offset, $requestFilterParams);

                if (empty($members)) {
                    $hasMore = false;
                    break;
                }

                $this->log('info', "Processing batch of " . count($members) . " members (offset: $offset)");

                foreach ($members as $member) {
                    $totalProcessed++;

                    $emailForLog = $member['email_address'] ?? '(no email)';
                    $this->log('info', "Processing member " . $emailForLog);
                    try {
                        $filterResult = $this->filterSingle($member);
                        if (!$filterResult) {
                            $totalFailed++;
                            continue;
                        }

                        $syncResult = $this->syncSingle($member);
                        if ($syncResult) {
                            $this->log('info', "Member {$member['email_address']} sync to Crm was successful.");
                            $totalSuccess++;
                        } else {
                            $this->log('info', "Member {$member['email_address']} sync to Crm has failed.");
                            $totalFailed++;
                        }
                    } catch (\Exception $e) {
                        $this->log('error', "Error processing member: " . $e->getMessage());
                        $totalFailed++;
                    }

                    if ($limit > 0 && $totalProcessed >= $limit) {
                        break;
                    }
                }

                $offset += count($members);

                if (count($members) < $fetchCount) {
                    $hasMore = false;
                }
            } catch (\Exception $e) {
                $this->log('error', "Error in syncAll batch: " . $e->getMessage());
                $totalFailed++;
                break;
            }
        }

        $this->log('info', "Completed Mailchimp to CRM synchronization: $totalProcessed processed, $totalSuccess successful, $totalFailed failed");

        return [
            'processed' => $totalProcessed,
            'success' => $totalSuccess,
            'failed' => $totalFailed
        ];
    }

    /**
     * Filter a single Mailchimp member for potential CRM update
     *
     * @param array $member The member data to filter
     * @return bool True if the member should be processed, false otherwise
     */
    private function filterSingle(array $member): bool
    {
        $email = $member['email_address'] ?? null;

        if (!$email) {
            $this->log('error', "Missing email in member data");
            return false;
        }

        if (!empty($member['merge_fields'][$this->mailchimpKeyOfCrmId])) {
            $this->log('debug', "Member " . $member['email_address'] . " has a CRM ID. Skipping.");
            return false;
        }

        $hasNewsletter = false;
        $memberInterests = $member['interests'] ?? [];
        foreach ($this->interestsToSync as $interestId) {
            if (isset($memberInterests[$interestId]) && $memberInterests[$interestId] === true) {
                $hasNewsletter = true;
                break;
            }
        }
        if (!$hasNewsletter) {
            $this->log('debug', "Member " . $member['email_address'] . " has no newsletter interests set. Skipping.");
            return false;
        }

        $configuredNewTag = $this->config->getNewTag();
        $newTagFound = false;
        if (isset($member['tags']) && is_array($member['tags'])) {
            foreach ($member['tags'] as $tag) {
                if ($tag['name'] === $configuredNewTag) {
                    $newTagFound = true;
                    break;
                }
            }
        }
        if (!$newTagFound) {
            $this->log('debug', "Member {$member['email_address']} does not have the configured new tag. Skipping.");
            return false;
        }

        return true;
    }

    /**
     * Process a single Mailchimp member for potential CRM update
     *
     * @param array $member The member data to sync
     * @return bool True if the member was processed successfully, false otherwise
     * @throws \Exception
     */
    public function syncSingle(array $member): bool
    {
        try {
            $crmData = $this->mapper->mailchimpToCrm($member, true);
            $crmData = $this->addCustomMapping($crmData, $member);

            $mailchimpId = MailChimpClient::calculateSubscriberId($member['email_address']);
            $crmId = $this->upsertToCrm($crmData, $member, 'daily_sync', $mailchimpId);

            return $crmId !== false;
        } catch (\Exception $e) {
            $this->log('error', "Error processing member {$member['email_address']}: " . $e->getMessage());
            return false;
        }
    }

    /**
     * Add custom CRM data to the mapped data
     *
     * @param array $crmData The mapped CRM data
     * @param array $member The member data from Mailchimp
     *
     * @return array The CRM data with custom fields added
     *
     * @throws \Exception
     */
    private function addCustomMapping(array $crmData, array $member): array
    {
        $crmData['entryChannel'] = [
            'value' => 'Mailchimp import ' . date('Y-m-d H:i:s'),
            'mode' => 'replaceEmpty'
        ];
        $crmData['groups'] = [
            'value' => (string) $this->groupForNewMembers,
            'mode' => 'append'
        ];

        $languageTag = $this->determineLanguageFromTags($member);
        if ($languageTag) {
            $crmData['language'] = [
                'value' => $languageTag,
                'mode' => 'replaceEmpty'
            ];
        }

        // remove empty WEBLINGID from CRM data
        if (isset($crmData['id'])) {
            unset($crmData['id']);
        }
        return $crmData;
    }

    /**
     * Return the interest filter parameters for the list members request
     */
    private function getRequestFilterParams(): array
    {
        $nowUtc = new \DateTimeImmutable('now', new \DateTimeZone('UTC'));
        $params = [
            'status' => 'subscribed',
            'since_last_changed' => $nowUtc->modify('-' . $this->config->getChangedWithinMonths() . ' months')->format(DATE_ATOM),
            // Only include subscribers whose opt-in is older than the specified months
            'before_timestamp_opt' => $nowUtc->modify('-' . $this->config->getOptInOlderThanMonths() . ' months')->format(DATE_ATOM),
        ];

        $defaultFields = [
            'members.email_address',
            'members.merge_fields',
            'members.status',
            'members.tags',
            'members.id',
            'members.interests'
        ];
        $params['fields'] = implode(',', $defaultFields);

        return $params;
    }

    /**
     * Try to upsert a contact to CRM and update Mailchimp with the CRM ID
     *
     * @param array $crmData The CRM data to upsert
     * @param array $member The member data from Mailchimp
     * @param string $callType The webhook event type
     * @param string $mailchimpId The Mailchimp ID of the contact
     *
     * @return string|false The CRM ID if successful, false otherwise
     * @throws Exception
     */
    private function upsertToCrm(array $crmData, array $member, string $callType, string $mailchimpId)
    {
        try {
            $response = $this->crmClient->post('/api/v1/member', $crmData);
            $crmId = (string) json_decode((string) $response->getBody(), true);

            if (!empty($crmId)) {
                $this->updateMailchimpWithCrmId($crmId, $mailchimpId);
                return $crmId;
            } else {
                $this->logWebhook('error', $callType, $mailchimpId, "Failed to get CRM ID from upsert response.");
                return false;
            }
        } catch (\Exception $e) {
            $this->logWebhook('error', $callType, $mailchimpId, "Error upserting to CRM: " . $e->getMessage());
            return false;
        }
    }

    /**
     * Update Mailchimp with the CRM ID
     *
     * @param string $crmId The CRM ID to update
     */
    private function updateMailchimpWithCrmId(string $crmId, string $mailchimpId): void
    {
        $this->mcClient->updateMergeFields(
            $mailchimpId,
            [$this->config->getMailchimpKeyOfCrmId() => $crmId]
        );

        $this->mcClient->removeTagFromSubscriber($mailchimpId, $this->config->getNewTag());
    }

    /**
     * Determine the language from subscriber tags
     *
     * @param array $subscriberTags The tags of the subscriber
     *
     * @return string|null The language tag or null if not found
     */
    private function determineLanguageFromTags(array $subscriberTags): ?string
    {
        if (!isset($subscriberTags['tags']) || !is_array($subscriberTags['tags'])) {
            return null;
        }

        foreach ($subscriberTags['tags'] as $tag) {
            if (!isset($tag['name'])) {
                continue;
            }

            $tagName = $tag['name'];
            if (in_array($tagName, $this->languageTags)) {
                // e.g. Deutsch -> "d", Francais -> "f", Italiano -> "i"
                return strtolower(substr($tagName, 0, 1));
            }
        }

        return null;
    }
}

1	<?php
2
3	namespace App\Synchronizer;
4
5	use App\Http\MailChimpClient;
6	use App\Exceptions\ConfigException;
7	use App\Synchronizer\LogTrait;
8	use Psr\Http\Message\ResponseInterface;
9
10	// Example member data from getListMembers():
11	// "{"email_address":"test@example.com",
12	// "id":"55502f40dc8b7c769880b10874abc9d0",
13	// "status":"subscribed",
14	// "merge_fields":{
15	// "FNAME":"Test",
16	// "LNAME":"User",
17	// "GENDER":"m",
18	// "WEBLINGID":"11195",
19	// "NOTES_CH":""},
20	// "last_changed":"2025-04-11T15:39:31+00:00",
21	// "tags":[
22	// {"id":6289607,"name":"member"},
23	// {"id":6289608,"name":"donor"},
24	// {"id":6289612,"name":"Region Olten"},
25	// {"id":6289618,"name":"climate"},
26	// {"id":6289625,"name":"Thal-G\u00e4u"}
27	// ]}"
28
29	/**
30	* Batch synchronizer for pulling changed Mailchimp subscribers and upserting them into the CRM
31	* when executed via cron or a CLI command. Designed to reduce CRM load and licensing overhead by
32	* only processing relevant members.
33	*
34	* Applies filters and processes matching members in batches.
35	*/
36	class MailchimpToCrmCronSynchronizer extends MailchimpToCrmSynchronizer
37	{
38	use LogTrait;
39
40	private array $languageTags;
41	private string $mailchimpKeyOfCrmId;
42	private int $groupForNewMembers;
43	private array $interestsToSync;
44
45	public function __construct(string $configFileName)
46	{
47	parent::__construct($configFileName);	1✔
48
NEW 49	if (!$this->config->isUpsertToCrmEnabled()) {	×
NEW 50	throw new ConfigException('Upsert to CRM is disabled. Please enable it in the config file.');	×
51	}
52
NEW 53	$this->languageTags = $this->config->getLanguageTagsFromConfig();	×
NEW 54	$this->mailchimpKeyOfCrmId = $this->config->getMailchimpKeyOfCrmId();	×
NEW 55	$this->groupForNewMembers = $this->config->getGroupForNewMembers();	×
56
NEW 57	$this->interestsToSync = $this->config->getInterestsToSync();	×
NEW 58	if (empty($this->interestsToSync)) {	×
NEW 59	throw new ConfigException('No interests configured for sync. Please configure it in the config file.');	×
60	}
61	}
62
63	/**
64	* Synchronize all changed Mailchimp members to CRM
65	*
66	* @param int $batchSize Number of members to process per batch
67	* @param int $limit Maximum number of members to process (0 for no limit)
68	* @return array Statistics about the synchronization
69	* @throws \Exception
70	*/
71	public function syncAll(int $batchSize = 100, int $limit = 0): array
72	{
73	$this->log('info', 'Starting Mailchimp to CRM synchronization');	2✔
74
75	$offset = 0;	2✔
76	$totalProcessed = 0;	2✔
77	$totalSuccess = 0;	2✔
78	$totalFailed = 0;	2✔
79	$hasMore = true;	2✔
80	$requestFilterParams = $this->getRequestFilterParams();	2✔
81
82	while ($hasMore && ($limit === 0 \|\| $totalProcessed < $limit)) {	2✔
83	$fetchCount = $limit > 0 ? min($batchSize, $limit - $totalProcessed) : $batchSize;	2✔
84
85	try {
86	$members = $this->mcClient->getListMembers($fetchCount, $offset, $requestFilterParams);	2✔
87
88	if (empty($members)) {	2✔
89	$hasMore = false;	1✔
90	break;	1✔
91	}
92
93	$this->log('info', "Processing batch of " . count($members) . " members (offset: $offset)");	1✔
94
95	foreach ($members as $member) {	1✔
96	$totalProcessed++;	1✔
97
98	$emailForLog = $member['email_address'] ?? '(no email)';	1✔
99	$this->log('info', "Processing member " . $emailForLog);	1✔
100	try {
101	$filterResult = $this->filterSingle($member);	1✔
102	if (!$filterResult) {	1✔
NEW 103	$totalFailed++;	×
NEW 104	continue;	×
105	}
106
107	$syncResult = $this->syncSingle($member);	1✔
108	if ($syncResult) {	1✔
109	$this->log('info', "Member {$member['email_address']} sync to Crm was successful.");	1✔
110	$totalSuccess++;	1✔
111	} else {
NEW 112	$this->log('info', "Member {$member['email_address']} sync to Crm has failed.");	×
113	$totalFailed++;	1✔
114	}
NEW 115	} catch (\Exception $e) {	×
NEW 116	$this->log('error', "Error processing member: " . $e->getMessage());	×
NEW 117	$totalFailed++;	×
118	}
119
120	if ($limit > 0 && $totalProcessed >= $limit) {	1✔
NEW 121	break;	×
122	}
123	}
124
125	$offset += count($members);	1✔
126
127	if (count($members) < $fetchCount) {	1✔
128	$hasMore = false;	1✔
129	}
NEW 130	} catch (\Exception $e) {	×
NEW 131	$this->log('error', "Error in syncAll batch: " . $e->getMessage());	×
NEW 132	$totalFailed++;	×
NEW 133	break;	×
134	}
135	}
136
137	$this->log('info', "Completed Mailchimp to CRM synchronization: $totalProcessed processed, $totalSuccess successful, $totalFailed failed");	2✔
138
139	return [	2✔
140	'processed' => $totalProcessed,	2✔
141	'success' => $totalSuccess,	2✔
142	'failed' => $totalFailed	2✔
143	];	2✔
144	}
145
146	/**
147	* Filter a single Mailchimp member for potential CRM update
148	*
149	* @param array $member The member data to filter
150	* @return bool True if the member should be processed, false otherwise
151	*/
152	private function filterSingle(array $member): bool
153	{
154	$email = $member['email_address'] ?? null;	5✔
155
156	if (!$email) {	5✔
157	$this->log('error', "Missing email in member data");	1✔
158	return false;	1✔
159	}
160
161	if (!empty($member['merge_fields'][$this->mailchimpKeyOfCrmId])) {	4✔
NEW 162	$this->log('debug', "Member " . $member['email_address'] . " has a CRM ID. Skipping.");	×
NEW 163	return false;	×
164	}
165
166	$hasNewsletter = false;	4✔
167	$memberInterests = $member['interests'] ?? [];	4✔
168	foreach ($this->interestsToSync as $interestId) {	4✔
169	if (isset($memberInterests[$interestId]) && $memberInterests[$interestId] === true) {	4✔
170	$hasNewsletter = true;	1✔
171	break;	1✔
172	}
173	}
174	if (!$hasNewsletter) {	4✔
175	$this->log('debug', "Member " . $member['email_address'] . " has no newsletter interests set. Skipping.");	3✔
176	return false;	3✔
177	}
178
179	$configuredNewTag = $this->config->getNewTag();	1✔
180	$newTagFound = false;	1✔
181	if (isset($member['tags']) && is_array($member['tags'])) {	1✔
182	foreach ($member['tags'] as $tag) {	1✔
183	if ($tag['name'] === $configuredNewTag) {	1✔
184	$newTagFound = true;	1✔
185	break;	1✔
186	}
187	}
188	}
189	if (!$newTagFound) {	1✔
NEW 190	$this->log('debug', "Member {$member['email_address']} does not have the configured new tag. Skipping.");	×
NEW 191	return false;	×
192	}
193
194	return true;	1✔
195	}
196
197	/**
198	* Process a single Mailchimp member for potential CRM update
199	*
200	* @param array $member The member data to sync
201	* @return bool True if the member was processed successfully, false otherwise
202	* @throws \Exception
203	*/
204	public function syncSingle(array $member): bool
205	{
206	try {
207	$crmData = $this->mapper->mailchimpToCrm($member, true);	3✔
208	$crmData = $this->addCustomMapping($crmData, $member);	3✔
209
210	$mailchimpId = MailChimpClient::calculateSubscriberId($member['email_address']);	3✔
211	$crmId = $this->upsertToCrm($crmData, $member, 'daily_sync', $mailchimpId);	3✔
212
213	return $crmId !== false;	3✔
NEW 214	} catch (\Exception $e) {	×
NEW 215	$this->log('error', "Error processing member {$member['email_address']}: " . $e->getMessage());	×
NEW 216	return false;	×
217	}
218	}
219
220	/**
221	* Add custom CRM data to the mapped data
222	*
223	* @param array $crmData The mapped CRM data
224	* @param array $member The member data from Mailchimp
225	*
226	* @return array The CRM data with custom fields added
227	*
228	* @throws \Exception
229	*/
230	private function addCustomMapping(array $crmData, array $member): array
231	{
232	$crmData['entryChannel'] = [	3✔
233	'value' => 'Mailchimp import ' . date('Y-m-d H:i:s'),	3✔
234	'mode' => 'replaceEmpty'	3✔
235	];	3✔
236	$crmData['groups'] = [	3✔
237	'value' => (string) $this->groupForNewMembers,	3✔
238	'mode' => 'append'	3✔
239	];	3✔
240
241	$languageTag = $this->determineLanguageFromTags($member);	3✔
242	if ($languageTag) {	3✔
243	$crmData['language'] = [	2✔
244	'value' => $languageTag,	2✔
245	'mode' => 'replaceEmpty'	2✔
246	];	2✔
247	}
248
249	// remove empty WEBLINGID from CRM data
250	if (isset($crmData['id'])) {	3✔
251	unset($crmData['id']);	1✔
252	}
253	return $crmData;	3✔
254	}
255
256	/**
257	* Return the interest filter parameters for the list members request
258	*/
259	private function getRequestFilterParams(): array
260	{
261	$nowUtc = new \DateTimeImmutable('now', new \DateTimeZone('UTC'));	3✔
262	$params = [	3✔
263	'status' => 'subscribed',	3✔
264	'since_last_changed' => $nowUtc->modify('-' . $this->config->getChangedWithinMonths() . ' months')->format(DATE_ATOM),	3✔
265	// Only include subscribers whose opt-in is older than the specified months
266	'before_timestamp_opt' => $nowUtc->modify('-' . $this->config->getOptInOlderThanMonths() . ' months')->format(DATE_ATOM),	3✔
267	];	3✔
268
269	$defaultFields = [	3✔
270	'members.email_address',	3✔
271	'members.merge_fields',	3✔
272	'members.status',	3✔
273	'members.tags',	3✔
274	'members.id',	3✔
275	'members.interests'	3✔
276	];	3✔
277	$params['fields'] = implode(',', $defaultFields);	3✔
278
279	return $params;	3✔
280	}
281
282	/**
283	* Try to upsert a contact to CRM and update Mailchimp with the CRM ID
284	*
285	* @param array $crmData The CRM data to upsert
286	* @param array $member The member data from Mailchimp
287	* @param string $callType The webhook event type
288	* @param string $mailchimpId The Mailchimp ID of the contact
289	*
290	* @return string\|false The CRM ID if successful, false otherwise
291	* @throws Exception
292	*/
293	private function upsertToCrm(array $crmData, array $member, string $callType, string $mailchimpId)
294	{
295	try {
296	$response = $this->crmClient->post('/api/v1/member', $crmData);	3✔
297	$crmId = (string) json_decode((string) $response->getBody(), true);	2✔
298
299	if (!empty($crmId)) {	2✔
300	$this->updateMailchimpWithCrmId($crmId, $mailchimpId);	2✔
301	return $crmId;	2✔
302	} else {
NEW 303	$this->logWebhook('error', $callType, $mailchimpId, "Failed to get CRM ID from upsert response.");	×
NEW 304	return false;	×
305	}
306	} catch (\Exception $e) {	1✔
307	$this->logWebhook('error', $callType, $mailchimpId, "Error upserting to CRM: " . $e->getMessage());	1✔
308	return false;	1✔
309	}
310	}
311
312	/**
313	* Update Mailchimp with the CRM ID
314	*
315	* @param string $crmId The CRM ID to update
316	*/
317	private function updateMailchimpWithCrmId(string $crmId, string $mailchimpId): void
318	{
319	$this->mcClient->updateMergeFields(	2✔
320	$mailchimpId,	2✔
321	[$this->config->getMailchimpKeyOfCrmId() => $crmId]	2✔
322	);	2✔
323
324	$this->mcClient->removeTagFromSubscriber($mailchimpId, $this->config->getNewTag());	2✔
325	}
326
327	/**
328	* Determine the language from subscriber tags
329	*
330	* @param array $subscriberTags The tags of the subscriber
331	*
332	* @return string\|null The language tag or null if not found
333	*/
334	private function determineLanguageFromTags(array $subscriberTags): ?string
335	{
336	if (!isset($subscriberTags['tags']) \|\| !is_array($subscriberTags['tags'])) {	6✔
NEW 337	return null;	×
338	}
339
340	foreach ($subscriberTags['tags'] as $tag) {	6✔
341	if (!isset($tag['name'])) {	6✔
NEW 342	continue;	×
343	}
344
345	$tagName = $tag['name'];	6✔
346	if (in_array($tagName, $this->languageTags)) {	6✔
347	// e.g. Deutsch -> "d", Francais -> "f", Italiano -> "i"
348	return strtolower(substr($tagName, 0, 1));	4✔
349	}
350	}
351
352	return null;	2✔
353	}
354	}

grueneschweiz / mailchimpservice / 17108989360

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous