diff --git a/build.xml b/build.xml index 9449da7552a..f83ef9721c9 100644 --- a/build.xml +++ b/build.xml @@ -1,5 +1,4 @@ - @@ -19,6 +18,15 @@ Starting package creation from war + + + + + + + + + diff --git a/store/src/java-test/com/zimbra/cs/service/mail/ImportContactsTest.java b/store/src/java-test/com/zimbra/cs/service/mail/ImportContactsTest.java new file mode 100644 index 00000000000..3f8eee1e27d --- /dev/null +++ b/store/src/java-test/com/zimbra/cs/service/mail/ImportContactsTest.java @@ -0,0 +1,154 @@ +/* + * ***** BEGIN LICENSE BLOCK ***** + * Zimbra Collaboration Suite Server + * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2016 Synacor, Inc. + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software Foundation, + * version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * You should have received a copy of the GNU General Public License along with this program. + * If not, see . + * ***** END LICENSE BLOCK ***** + */ +package com.zimbra.cs.service.mail; + +import java.io.BufferedReader; +import java.io.StringReader; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.zimbra.cs.account.Account; +import com.zimbra.cs.account.NamedEntry; +import com.zimbra.cs.account.Provisioning; +import com.zimbra.cs.account.SearchDirectoryOptions; +import com.zimbra.cs.ldap.ZLdapFilterFactory.FilterId; +import com.zimbra.cs.mailbox.Contact; +import com.zimbra.cs.mailbox.ContactGroup; +import com.zimbra.cs.mailbox.ContactGroup.Member; +import com.zimbra.cs.mailbox.Mailbox; +import com.zimbra.cs.mailbox.MailboxManager; +import com.zimbra.cs.mailbox.MailboxTestUtil; +import com.zimbra.cs.service.formatter.ContactCSV; +import com.zimbra.cs.service.util.ItemId; + +public class ImportContactsTest { + private static final String USERNAME = "user1@zimbra.com"; + private static final String USERDN = "uid=user1,ou=people,dc=zimbra,dc=com"; + + @BeforeClass + public static void init() throws Exception { + System.setProperty("zimbra.config", "../store/src/java-test/localconfig-test.xml"); + MailboxTestUtil.initServer(); + } + + @Before + public void setUp() throws Exception { + Provisioning prov = Provisioning.getInstance(); + prov.createAccount(USERNAME, "secret", new HashMap()); + Provisioning.setInstance(prov); + } + + @After + public void tearDown() throws Exception { + Provisioning prov = Provisioning.getInstance(); + prov.deleteAccount(USERNAME); + } + + @Test + public void testImportContactsOK() throws Exception { + Provisioning prov = Provisioning.getInstance(); + Account account = prov.getAccountByName(USERNAME); + Mailbox mbox = MailboxManager.getInstance().getMailboxByAccount(account); + + String csvText = getCsvFile(); + BufferedReader reader = new BufferedReader(new StringReader(csvText)); + List> contactsMap = ContactCSV.getContacts(reader, null); + + List ids = ImportContacts.ImportCsvContacts(null, mbox, + new ItemId(mbox, Mailbox.ID_FOLDER_CONTACTS), contactsMap); + + Assert.assertFalse(ids.isEmpty()); + Assert.assertEquals(4, ids.size()); + + Set expectedContactEmails = getExpectedContactEmails(); + Set foundContactGroups = new HashSet<>(); + + for (ItemId id : ids) { + Contact contact = mbox.getContactById(null, id.getId()); + Assert.assertNotNull(contact); + + for (String addr : contact.getEmailAddresses()) { + expectedContactEmails.remove(addr); + } + + if (contact.isContactGroup()) { + foundContactGroups.add(contact); + } + } + + if (!expectedContactEmails.isEmpty()) { + Assert.fail("Missing expected contact emails: " + expectedContactEmails); + } + + Assert.assertEquals("Found exactly one contact group after import", + 1, foundContactGroups.size()); + + ContactGroup group = ContactGroup.init(foundContactGroups.iterator().next(), true); + + for (Member member : group.getMembers()) { + Member.Type memberType = member.getType(); + + if (memberType == Member.Type.INLINE) { + Assert.assertEquals("Found correct inline group member", "delta.user@example.org", + member.getValue()); + } else if (memberType != Member.Type.CONTACT_REF) { + Assert.fail(String.format("Found unexpected group member of type %s with value \"%s\"", memberType, + member.getValue())); + } + } + } + + private String getCsvFile() { + return "" + + "email,fullName,type,dlist\r\n" + + "alpha.user@example.org,Alpha User,inline,\"\"\r\n" + + "bravo.user@example.org,Bravo User,inline,\"\"\r\n" + + "testgroup@example.org,Test Group,group,\"alpha.user@example.org,bravo.user@example.org,charlie.user@example.org,delta.user@example.org\"\r\n" + + "charlie.user@example.org,Charlie User,inline,\"\"\r\n"; + } + + private Set getExpectedContactEmails() { + Set expectedContactEmails = new HashSet<>(); + expectedContactEmails.add("alpha.user@example.org"); + expectedContactEmails.add("bravo.user@example.org"); + expectedContactEmails.add("charlie.user@example.org"); + expectedContactEmails.add("delta.user@example.org"); + expectedContactEmails.add("testgroup@example.org"); + + return expectedContactEmails; + } + + @Test + public void testFoo() throws Exception { + Provisioning prov = Provisioning.getInstance(); +// Account account = prov.getAccountBy(USERDN); + SearchDirectoryOptions sdo = new SearchDirectoryOptions(); + sdo.setFilterString((FilterId) null, USERDN); + List entries = prov.searchDirectory(sdo); + Assert.assertFalse(entries.isEmpty()); + } +} + diff --git a/store/src/java/com/zimbra/cs/mailbox/ContactGroup.java b/store/src/java/com/zimbra/cs/mailbox/ContactGroup.java index 8251860a43d..4ba85c118cb 100644 --- a/store/src/java/com/zimbra/cs/mailbox/ContactGroup.java +++ b/store/src/java/com/zimbra/cs/mailbox/ContactGroup.java @@ -390,7 +390,7 @@ public static Type fromSoap(String soapEncoded) throws ServiceException { } } - private static Member init(Member.Type type, String value) throws ServiceException { + public static Member init(Member.Type type, String value) throws ServiceException { Member member = null; switch (type) { case CONTACT_REF: @@ -890,7 +890,22 @@ public void handle() throws ServiceException { } } + public void migrate(Contact contact) throws ServiceException { + migrate(contact, null); + } + + /** + * Given a "group" type contact, add all its members either Inline or as a + * Contacts reference if found in the passed list of contacts already parsed + * from the source file of the passed contact. + * + * @param contact the contact being migrated + * @param parsedByAddr a list of contacts parsed from same source as above, + * null/empty to add all as inline + * @throws ServiceException + */ + public void migrate(Contact contact, Map parsedByAddr) throws ServiceException { if (!contact.isGroup()) { return; } @@ -903,7 +918,7 @@ public void migrate(Contact contact) throws ServiceException { ContactGroup contactGroup = ContactGroup.init(); - migrate(contactGroup, dlist); + migrate(contactGroup, dlist, parsedByAddr); if (contactGroup.hasMembers()) { ParsedContact pc = new ParsedContact(contact); @@ -924,8 +939,17 @@ public void migrate(Contact contact) throws ServiceException { } } - // add each dlist member as an inlined member in groupMember - static void migrate(ContactGroup contactGroup, String dlist) throws ServiceException { + static void migrate(ContactGroup contactGroup, String dlist) + throws ServiceException { + migrate(contactGroup, dlist, null); + } + + /** + * Add each comma-delimited dlist member to the given contact group, as either + * an inlined member or a contact reference if found in parsedByAddr. + */ + static void migrate(ContactGroup contactGroup, String dlist, Map parsedByAddr) + throws ServiceException { Matcher matcher = PATTERN.matcher(dlist); while (matcher.find()) { String token = matcher.group(); @@ -937,9 +961,15 @@ static void migrate(ContactGroup contactGroup, String dlist) throws ServiceExcep String addr = token.trim(); if (!addr.isEmpty()) { try { - contactGroup.addMember(Member.Type.INLINE, addr); + Contact found = parsedByAddr == null ? null : parsedByAddr.get(addr.toLowerCase()); + if (found == null) { + contactGroup.addMember(Member.Type.INLINE, addr); + } else { + ItemId iid = new ItemId(found); + contactGroup.addMember(Member.Type.CONTACT_REF, iid.toString()); + } } catch (ServiceException e) { - ZimbraLog.contact.info("skipped contact group member %s", addr); + ZimbraLog.contact.info("skipped contact group member %s: %s", addr, e.getMessage()); } } } diff --git a/store/src/java/com/zimbra/cs/service/formatter/ArchiveFormatter.java b/store/src/java/com/zimbra/cs/service/formatter/ArchiveFormatter.java index 3055ea62b7b..b98d3b68127 100644 --- a/store/src/java/com/zimbra/cs/service/formatter/ArchiveFormatter.java +++ b/store/src/java/com/zimbra/cs/service/formatter/ArchiveFormatter.java @@ -80,6 +80,8 @@ import com.zimbra.cs.mailbox.CalendarItem.Instance; import com.zimbra.cs.mailbox.Chat; import com.zimbra.cs.mailbox.Contact; +import com.zimbra.cs.mailbox.ContactGroup; +import com.zimbra.cs.mailbox.ContactGroup.Member; import com.zimbra.cs.mailbox.Conversation; import com.zimbra.cs.mailbox.DeliveryOptions; import com.zimbra.cs.mailbox.Document; @@ -844,12 +846,13 @@ public void saveCallback(UserServletContext context, String contentType, Folder disableJettyTimeout(context); Exception ex = null; - ItemData id = null; + ItemData itemData = null; FolderDigestInfo digestInfo = new FolderDigestInfo(context.opContext); List errs = new LinkedList(); List flist; Map fmap = new HashMap(); Map idMap = new HashMap(); + Set rawContacts = new HashSet<>(); long last = System.currentTimeMillis(); String types = context.getTypesString(); String resolve = context.params.get(PARAM_RESOLVE); @@ -956,11 +959,12 @@ public void saveCallback(UserServletContext context, String contentType, Folder continue; } else if (aie.getName().endsWith(".meta")) { meta = true; - if (id != null) { - addItem(context, fldr, fmap, digestInfo, idMap, ids, searchTypes, r, id, ais, null, errs); + if (itemData != null) { + addItem(context, fldr, fmap, digestInfo, idMap, ids, searchTypes, r, itemData, ais, null, + errs, rawContacts); } try { - id = new ItemData(readArchiveEntry(ais, aie)); + itemData = new ItemData(readArchiveEntry(ais, aie)); } catch (IOException e) { throw ServiceException.FAILURE("Error reading file", e); } catch (Exception e) { @@ -969,30 +973,32 @@ public void saveCallback(UserServletContext context, String contentType, Folder continue; } else if (aie.getName().endsWith(".err")) { addError(errs, FormatterServiceException.MISMATCHED_SIZE(aie.getName())); - } else if (id == null) { + } else if (itemData == null) { if (meta) { addError(errs, FormatterServiceException.MISSING_META(aie.getName())); } else { addData(context, fldr, fmap, searchTypes, r, timestamp == null || !timestamp.equals("0"), ais, aie, errs); } - } else if ((aie.getType() != 0 && id.ud.type != aie.getType()) || (id.ud.getBlobDigest() != null && aie.getSize() != -1 && id.ud.size != aie.getSize())) { + } else if ((aie.getType() != 0 && itemData.ud.type != aie.getType()) || (itemData.ud.getBlobDigest() != null && aie.getSize() != -1 && itemData.ud.size != aie.getSize())) { addError(errs, FormatterServiceException.MISMATCHED_META(aie.getName())); } else { - addItem(context, fldr, fmap, digestInfo, idMap, ids, searchTypes, r, id, ais, aie, errs); + addItem(context, fldr, fmap, digestInfo, idMap, ids, searchTypes, r, itemData, ais, aie, errs, + rawContacts); } - id = null; + itemData = null; } - if (id != null) { - addItem(context, fldr, fmap, digestInfo, idMap, ids, searchTypes, r, id, ais, null, errs); + if (itemData != null) { + addItem(context, fldr, fmap, digestInfo, idMap, ids, searchTypes, r, itemData, ais, null, errs, + rawContacts); } } catch (Exception e) { - if (id == null) { + if (itemData == null) { addError(errs, FormatterServiceException.UNKNOWN_ERROR(e)); } else { - addError(errs, FormatterServiceException.UNKNOWN_ERROR(id.path, e)); + addError(errs, FormatterServiceException.UNKNOWN_ERROR(itemData.path, e)); } - id = null; + itemData = null; } finally { if (ais != null) { ais.close(); @@ -1002,6 +1008,9 @@ public void saveCallback(UserServletContext context, String contentType, Folder } catch (Exception e) { ex = e; } + + postProcessContacts(rawContacts, context, context.targetMailbox, fldr); + try { updateClient(context, ex, errs); } catch (ServiceException e) { @@ -1011,6 +1020,86 @@ public void saveCallback(UserServletContext context, String contentType, Folder } } + private void postProcessContacts(Set rawContacts, UserServletContext context, Mailbox mbox, + Folder folder) { + + Map contactsById = new HashMap<>(); + Map contactGroups = new HashMap<>(); + + for (Contact contact : rawContacts) { + if (contact.isContactGroup()) { + try { + contactGroups.put(Integer.valueOf(contact.getId()), ContactGroup.init(contact, true)); + } catch (ServiceException ex) { + warn(ex); + return; + } + } else { + contactsById.put(contact.getId(), contact); + } + } + + for (Map.Entry entry : contactGroups.entrySet()) { + Integer contactGroupId = entry.getKey(); + ContactGroup contactGroup = entry.getValue(); + List oldMembers = contactGroup.getMembers(); + List newMembers = new ArrayList<>(oldMembers.size()); + + for (Member member : oldMembers) { + Member.Type memberType = member.getType(); + String memberValue = member.getValue(); + +// if (memberValue.contains(":")) { +// memberValue = memberValue.split(":")[1]; +// } + + try { + if (Member.Type.CONTACT_REF.equals(memberType)) { +// Contact rawContact = contactsById.get(Integer.valueOf(memberValue)); + Contact rawContact = contactsById.get(memberValue); + Contact foundContact = findContact(context.opContext, mbox, rawContact, folder); + + if (foundContact == null) { + ZimbraLog.misc.warn("Failed to find newly added contact %s", rawContact); + return; + } + else { + newMembers.add(Member.init(memberType, String.valueOf(foundContact.getId()))); + } + } + else { + newMembers.add(Member.init(memberType, memberValue)); + } + } catch (ServiceException ex) { + warn(ex); + return; + } + } + + contactGroup.removeAllMembers(); + + for (Member member : newMembers) { + try { + contactGroup.addMember(member.getType(), member.getValue()); + } catch (ServiceException ex) { + warn(ex); + return; + } + } + + try { + Contact rawContact = contactsById.get(contactGroupId); + ParsedContact pc = new ParsedContact(rawContact); + + pc.modifyField(ContactConstants.A_groupMember, contactGroup.encode()); + + mbox.modifyContact(context.opContext, contactGroupId, pc); + } catch (ServiceException ex) { + warn(ex); + } + } + } + private void addError(List errs, ServiceException ex) { StringBuilder s = new StringBuilder(ex.getLocalizedMessage() == null ? ex.toString() : ex.getLocalizedMessage()); @@ -1098,34 +1187,41 @@ private void warn(Exception e) { private void addItem(UserServletContext context, Folder fldr, Map fmap, FolderDigestInfo digestInfo, - Map idMap, int[] ids, Set types, Resolve r, ItemData id, + Map idMap, int[] ids, Set types, Resolve r, ItemData itemData, ArchiveInputStream ais, ArchiveInputEntry aie, List errs) throws ServiceException { + addItem(context, fldr, fmap, digestInfo, idMap, ids, types, r, itemData, ais, aie, errs, null); + } + + private void addItem(UserServletContext context, Folder fldr, Map fmap, FolderDigestInfo digestInfo, + Map idMap, int[] ids, Set types, Resolve r, ItemData itemData, + ArchiveInputStream ais, ArchiveInputEntry aie, List errs, Set rawContacts) + throws ServiceException { try { Mailbox mbox = fldr.getMailbox(); - MailItem mi = MailItem.constructItem(mbox, id.ud); + MailItem mi = MailItem.constructItem(mbox, itemData.ud); MailItem newItem = null, oldItem = null; OperationContext octxt = context.opContext; String path; ParsedMessage pm; - boolean root = fldr.getId() == Mailbox.ID_FOLDER_ROOT || fldr.getId() == Mailbox.ID_FOLDER_USER_ROOT || id.path.startsWith(fldr.getPath() + '/'); + boolean root = fldr.getId() == Mailbox.ID_FOLDER_ROOT || fldr.getId() == Mailbox.ID_FOLDER_USER_ROOT || itemData.path.startsWith(fldr.getPath() + '/'); - if ((ids != null && Arrays.binarySearch(ids, id.ud.id) < 0) || (types != null && !types.contains(MailItem.Type.of(id.ud.type)))) + if ((ids != null && Arrays.binarySearch(ids, itemData.ud.id) < 0) || (types != null && !types.contains(MailItem.Type.of(itemData.ud.type)))) return; - if (id.ud.getBlobDigest() != null && aie == null) { - addError(errs, FormatterServiceException.MISSING_BLOB(id.path)); + if (itemData.ud.getBlobDigest() != null && aie == null) { + addError(errs, FormatterServiceException.MISSING_BLOB(itemData.path)); return; } if (root) { - path = id.path; + path = itemData.path; } else { - path = fldr.getPath() + id.path; + path = fldr.getPath() + itemData.path; } if (path.endsWith("/") && !path.equals("/")) { path = path.substring(0, path.length() - 1); } - if (mbox.isImmutableSystemFolder(id.ud.folderId)) + if (mbox.isImmutableSystemFolder(itemData.ud.folderId)) return; switch (mi.getType()) { @@ -1257,6 +1353,9 @@ private void addItem(UserServletContext context, Folder fldr, Map emailAdresses = new HashSet(contact.getEmailAddresses()); + HashSet emailAddresses = new HashSet(contact.getEmailAddresses()); for (String emailId : ct.getEmailAddresses()) { - if (emailAdresses.contains(emailId)) { + if (emailAddresses.contains(emailId)) { return contact; } } diff --git a/store/src/java/com/zimbra/cs/service/mail/ImportContacts.java b/store/src/java/com/zimbra/cs/service/mail/ImportContacts.java index 6be5462d391..f53def7a0ae 100644 --- a/store/src/java/com/zimbra/cs/service/mail/ImportContacts.java +++ b/store/src/java/com/zimbra/cs/service/mail/ImportContacts.java @@ -22,21 +22,22 @@ import java.io.InputStreamReader; import java.io.StringReader; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import com.zimbra.client.ZMailbox; import com.zimbra.common.service.ServiceException; -import com.zimbra.common.soap.MailConstants; import com.zimbra.common.soap.Element; +import com.zimbra.common.soap.MailConstants; import com.zimbra.common.util.StringUtil; import com.zimbra.cs.mailbox.Contact; import com.zimbra.cs.mailbox.ContactGroup; -import com.zimbra.cs.mailbox.MailItem; import com.zimbra.cs.mailbox.MailServiceException; import com.zimbra.cs.mailbox.Mailbox; import com.zimbra.cs.mailbox.OperationContext; -import com.zimbra.cs.mailbox.Tag; import com.zimbra.cs.mailbox.util.TagUtil; import com.zimbra.cs.mime.ParsedContact; import com.zimbra.cs.service.FileUploadServlet; @@ -45,9 +46,7 @@ import com.zimbra.cs.service.formatter.ContactCSV.ParseException; import com.zimbra.cs.service.util.ItemId; import com.zimbra.cs.service.util.ItemIdFormatter; -import com.zimbra.client.ZMailbox; import com.zimbra.soap.ZimbraSoapContext; -import com.zimbra.soap.JaxbUtil; import com.zimbra.soap.mail.message.ImportContactsRequest; import com.zimbra.soap.mail.message.ImportContactsResponse; import com.zimbra.soap.mail.type.Content; @@ -151,14 +150,33 @@ private static BufferedReader parseUploadedContent(ZimbraSoapContext lc, String public static List ImportCsvContacts(OperationContext oc, Mailbox mbox, ItemId iidFolder, List> csvContacts) throws ServiceException { - List createdIds = new LinkedList(); - for (Map contact : csvContacts) { - String[] tags = TagUtil.decodeTags(ContactCSV.getTags(contact)); - Contact c = mbox.createContact(oc, new ParsedContact(contact), iidFolder.getId(), tags); - createdIds.add(new ItemId(c)); + List createdContacts = new LinkedList<>(); + Map createdContactsByEmail = new HashMap<>(); + + for (Map csvContact : csvContacts) { + String[] tags = TagUtil.decodeTags(ContactCSV.getTags(csvContact)); + Contact contact = mbox.createContact(oc, new ParsedContact(csvContact), iidFolder.getId(), tags); + createdContacts.add(contact); + + if (!contact.isGroup()) { + for (String addr : contact.getEmailAddresses()) { + createdContactsByEmail.put(addr.toLowerCase(), contact); + } + // createdContactsById.put(contact.getId(), contact) to match the tail end of + // the ID in contact group metadata from a TGZ meta file. Except this is + // explicitly for importing a CSV source, and CSV doesn't contain any ID. + } + } + + for (Contact contact : createdContacts) { + if (!contact.isGroup()) { + continue; + } + ContactGroup.MigrateContactGroup mcg = new ContactGroup.MigrateContactGroup(mbox); - mcg.migrate(c); + mcg.migrate(contact, createdContactsByEmail); } - return createdIds; + + return createdContacts.stream().map(ItemId::new).collect(Collectors.toList()); } }