CommunicationController/CommunicationService massArchive

src/main/java/com/impactupgrade/nucleus/controller/CommunicationController.java

@@ -160,4 +160,44 @@ public Response upsertContact(
 
     return Response.ok().build();
   }
+
+  // Auto archive the MC contact if they don't exist as a subscriber in the CRM. Enable only if the organization
+  // ensures all contacts are first in the CRM. If they still manually upload to MC directly, keep this turned off.
+  @GET
+  @Path("/mass-archive")
+  public Response massArchive(@Context HttpServletRequest request) throws Exception {
+    Environment env = envFactory.init(request);
+
+    Runnable thread = () -> {
+      try {
+        String jobName = "Communication: Mass Archive";
+        env.startJobLog(JobType.EVENT, null, jobName, "Nucleus Portal");
+        boolean success = true;
+
+        for (CommunicationService communicationService : env.allCommunicationServices()) {
+          try {
+            communicationService.massArchive();
+            env.logJobInfo("{}: massArchive done", communicationService.name());
+          } catch (Exception e) {
+            env.logJobError("communication massArchive failed for {}", communicationService.name(), e);
+            env.logJobError(e.getMessage());
+            success = false;
+          }
+        }
+
+        if (success) {
+          env.endJobLog(JobStatus.DONE);
+        } else {
+          env.endJobLog(JobStatus.FAILED);
+        }
+      } catch (Exception e) {
+        env.logJobError("communication massArchive failed", e);
+        env.logJobError(e.getMessage());
+        env.endJobLog(JobStatus.FAILED);
+      }
+    };
+    new Thread(thread).start();
+
+    return Response.ok().build();
+  }
 }

src/main/java/com/impactupgrade/nucleus/controller/MailchimpController.java

@@ -87,8 +87,8 @@ private void processCampaignEvent(AudienceEvent event, Environment env) throws E
     }
 
     // TODO: Clunky way to retrieve the specific account keys, when all we have is the List ID..
-    EnvironmentConfig.Mailchimp mailchimpConfig = null;
-    for (EnvironmentConfig.Mailchimp _mailchimpConfig : env.getConfig().mailchimp) {
+    EnvironmentConfig.CommunicationPlatform mailchimpConfig = null;
+    for (EnvironmentConfig.CommunicationPlatform _mailchimpConfig : env.getConfig().mailchimp) {
       for (EnvironmentConfig.CommunicationList communicationList : _mailchimpConfig.lists) {
         if (event.listId.equalsIgnoreCase(communicationList.id)) {
           mailchimpConfig = _mailchimpConfig;

src/main/java/com/impactupgrade/nucleus/environment/EnvironmentConfig.java

@@ -350,12 +350,7 @@ public static class CommunicationPlatform extends Platform {
     public boolean transactionalSender = false;
   }
 
-  public List<Mailchimp> mailchimp = new ArrayList<>();
-  public static class Mailchimp extends CommunicationPlatform {
-    // Auto archive the MC contact if they don't exist as a subscriber in the CRM. Enable only if the organization
-    // ensures all contacts are first in the CRM. If they still manually upload to MC directly, keep this turned off.
-    public boolean enableCrmBasedArchival = false;
-  }
+  public List<CommunicationPlatform> mailchimp = new ArrayList<>();
 
   public List<CommunicationPlatform> constantContact = new ArrayList<>();
 

src/main/java/com/impactupgrade/nucleus/service/segment/CommunicationService.java

@@ -12,4 +12,5 @@ public interface CommunicationService extends SegmentService {
   void syncUnsubscribes(Calendar lastSync) throws Exception;
   void upsertContact(String contactId) throws Exception;
   // TODO: deleteContact
+  void massArchive() throws Exception;
 }

src/main/java/com/impactupgrade/nucleus/service/segment/ConstantContactCommunicationService.java

@@ -67,4 +67,9 @@ public void upsertContact(String contactId) throws Exception {
       }
     }
   }
+
+  @Override
+  public void massArchive() throws Exception {
+    // TODO
+  }
 }

src/main/java/com/impactupgrade/nucleus/service/segment/MailchimpCommunicationService.java

@@ -52,7 +52,7 @@ public boolean isConfigured(Environment env) {
 
   @Override
   public void syncContacts(Calendar lastSync) throws Exception {
-    for (EnvironmentConfig.Mailchimp mailchimpConfig : env.getConfig().mailchimp) {
+    for (EnvironmentConfig.CommunicationPlatform mailchimpConfig : env.getConfig().mailchimp) {
       for (EnvironmentConfig.CommunicationList communicationList : mailchimpConfig.lists) {
         // clear the cache, since fields differ between audiences
         mergeFieldsNameToTag.clear();
@@ -89,10 +89,6 @@ public void syncContacts(Calendar lastSync) throws Exception {
             }
           } while (resultSet != null);
         }
-
-        if (mailchimpConfig.enableCrmBasedArchival) {
-          massArchive(mailchimpConfig, communicationList, listMembers, seenEmails, mailchimpClient);
-        }
       }
     }
   }
@@ -122,7 +118,7 @@ protected CrmContact asCrmContact(CrmAccount crmAccount) {
     return crmContact;
   }
 
-  protected void syncContacts(PagedResults.ResultSet<CrmContact> resultSet, EnvironmentConfig.Mailchimp mailchimpConfig,
+  protected void syncContacts(PagedResults.ResultSet<CrmContact> resultSet, EnvironmentConfig.CommunicationPlatform mailchimpConfig,
       EnvironmentConfig.CommunicationList communicationList, List<MemberInfo> listMembers, Set<String> mcEmails,
       Set<String> seenEmails, MailchimpClient mailchimpClient) {
     List<CrmContact> contactsToUpsert = new ArrayList<>();
@@ -179,16 +175,39 @@ protected void syncContacts(PagedResults.ResultSet<CrmContact> resultSet, Enviro
     }
   }
 
-  protected void massArchive(EnvironmentConfig.Mailchimp mailchimpConfig,
-      EnvironmentConfig.CommunicationList communicationList, List<MemberInfo> listMembers, Set<String> seenEmails,
-      MailchimpClient mailchimpClient) throws Exception {
-    // get all mc email addresses in the entire audience
-    Set<String> emailsToArchive = listMembers.stream().map(memberInfo -> memberInfo.email_address.toLowerCase(Locale.ROOT)).collect(Collectors.toSet());
-    // remove all CRM emails from the list of MC emails, which leaves us with the list that needs to be archived
-    emailsToArchive.removeAll(seenEmails);
+  @Override
+  public void massArchive() throws Exception {
+    for (EnvironmentConfig.CommunicationPlatform mailchimpConfig : env.getConfig().mailchimp) {
+      for (EnvironmentConfig.CommunicationList communicationList : mailchimpConfig.lists) {
+        MailchimpClient mailchimpClient = env.mailchimpClient(mailchimpConfig);
+        List<MemberInfo> listMembers = mailchimpClient.getListMembers(communicationList.id);
+        // get all mc email addresses in the entire audience
+        Set<String> emailsToArchive = listMembers.stream().map(memberInfo -> memberInfo.email_address.toLowerCase(Locale.ROOT)).collect(Collectors.toSet());
+
+        PagedResults<CrmContact> contactPagedResults = env.primaryCrmService().getEmailContacts(null, communicationList);
+        for (PagedResults.ResultSet<CrmContact> resultSet : contactPagedResults.getResultSets()) {
+          do {
+            // remove all active CRM emails from the list of MC emails, which leaves us with the list that needs to be archived
+            for (CrmContact crmContact : resultSet.getRecords()) {
+              if (crmContact.canReceiveEmail()) {
+                emailsToArchive.remove(crmContact.email.toLowerCase(Locale.ROOT));
+              }
+            }
+
+            if (!Strings.isNullOrEmpty(resultSet.getNextPageToken())) {
+              // next page
+              resultSet = env.primaryCrmService().queryMoreContacts(resultSet.getNextPageToken());
+            } else {
+              resultSet = null;
+            }
+          } while (resultSet != null);
+        }
 
-    String archiveBatchId = mailchimpClient.archiveContactsBatch(communicationList.id, emailsToArchive);
-    mailchimpClient.runBatchOperations(mailchimpConfig, archiveBatchId, 0);
+        env.logJobInfo("massArchiving {} contacts in MC: {}", emailsToArchive.size(), String.join(", ", emailsToArchive));
+//        String archiveBatchId = mailchimpClient.archiveContactsBatch(communicationList.id, emailsToArchive);
+//        mailchimpClient.runBatchOperations(mailchimpConfig, archiveBatchId, 0);
+      }
+    }
   }
 
   protected Map<String, Set<String>> getActiveTags(List<CrmContact> crmContacts, Map<String,
@@ -267,7 +286,7 @@ protected void updateAccountsByEmails(List<String> emails, Consumer<CrmAccount>
   public void upsertContact(String contactId) throws Exception {
     CrmService crmService = env.primaryCrmService();
 
-    for (EnvironmentConfig.Mailchimp mailchimpConfig : env.getConfig().mailchimp) {
+    for (EnvironmentConfig.CommunicationPlatform mailchimpConfig : env.getConfig().mailchimp) {
       for (EnvironmentConfig.CommunicationList communicationList : mailchimpConfig.lists) {
         // clear the cache, since fields differ between audiences
         mergeFieldsNameToTag.clear();
@@ -285,7 +304,7 @@ public void upsertContact(String contactId) throws Exception {
   //  stripping out the full sync and only pushing in the contact's new tags. We skip removing old tags --
   //  instead, let the nightly job do that for everybody. This process is typically only needed when something
   //  needs added, like a campaign tag to kick off a Journey in MC itself.
-  protected void upsertContact(EnvironmentConfig.Mailchimp mailchimpConfig,
+  protected void upsertContact(EnvironmentConfig.CommunicationPlatform mailchimpConfig,
       EnvironmentConfig.CommunicationList communicationList, CrmContact crmContact) throws Exception {
     MailchimpClient mailchimpClient = env.mailchimpClient(mailchimpConfig);
 

src/main/java/com/impactupgrade/nucleus/service/segment/MinistryByTextCommunicationService.java

@@ -80,4 +80,9 @@ public void upsertContact(String contactId) throws Exception {
       }
     }
   }
+
+  @Override
+  public void massArchive() throws Exception {
+    // TODO
+  }
 }

src/main/java/com/impactupgrade/nucleus/service/segment/SendGridCommunicationService.java

@@ -93,6 +93,11 @@ public void upsertContact(String contactId) throws Exception {
     //TODO
   }
 
+  @Override
+  public void massArchive() throws Exception {
+    // TODO
+  }
+
   protected Contact toSendGridContact(CrmContact crmContact, List<String> campaignNames,
       Map<String, String> customFieldsByName, SendGrid sendgridClient,
       EnvironmentConfig.CommunicationPlatform communicationPlatform,

src/test/java/com/impactupgrade/nucleus/it/MailchimpCommunicationServiceIT.java

@@ -81,7 +81,7 @@ public void syncContacts() throws Exception {
     // Sync contacts to MC
     env.communicationService("mailchimp").syncContacts(beforeBulkImport);
 
-    EnvironmentConfig.Mailchimp mailchimp = env.getConfig().mailchimp.get(0);
+    EnvironmentConfig.CommunicationPlatform mailchimp = env.getConfig().mailchimp.get(0);
     String listId = mailchimp.lists.get(0).id;
     MailchimpClient mailchimpClient = env.mailchimpClient(mailchimp);
 
@@ -128,7 +128,7 @@ public void syncUnsubscribes() throws Exception {
     assertTrue(crmContactsEmails.containsAll(emails));
 
     // Subscribe emails to MC list
-    EnvironmentConfig.Mailchimp mailchimp = env.getConfig().mailchimp.get(0);
+    EnvironmentConfig.CommunicationPlatform mailchimp = env.getConfig().mailchimp.get(0);
     String listId = mailchimp.lists.get(0).id;
     MailchimpClient mailchimpClient = env.mailchimpClient(mailchimp);