1) Make ImapTreeDiff configurable. Remove hardcoded values.

2) Add cache config section.
3) Add log.imapdebug option to enable JavaMail debug output.
4) Change file pattern of logging to the same as in output used.
5) Bug of copy reproduced in test.bug_1, test.bug_2. Mailed to Giles.
Reported new Larch bug: rgrove/larch#73

src/ImapTreeDiff.groovy

@@ -1,5 +1,6 @@
 #!/opt/groovy-2.3.6/bin/groovy
 import com.sun.mail.imap.IMAPFolder
+import groovy.util.logging.Log4j2
 @Grab(group = 'commons-cli', module = 'commons-cli', version = '1.2')
 
 // TMP
@@ -16,9 +17,10 @@ import info.hubbitus.imaptree.diff.FolderMessagesCopier
 import info.hubbitus.imaptree.diff.FolderMessagesDiff
 import info.hubbitus.imaptree.utils.cache.memcached.ImapTreeTranscoder
 import info.hubbitus.imaptree.utils.cache.memcached.MemcachedClientExtended
-import java.util.logging.Level
+import net.spy.memcached.MemcachedClient
 
 import java.util.concurrent.TimeUnit
+import java.util.logging.Level
 
 /**
  * Simple example to just print folder sizes recursively
@@ -32,9 +34,18 @@ cli.h(longOpt: 'help', 'This usage information', required: false)
 cli.D(longOpt: 'config', '''Change configured options from command line. Allow runtime override. May appear multiple times - processed in that order. For example:
 	-D log.fullXmlCache="some.file" --config operations.printFolderSizes.folderProcess='{true}' -D operations.printFolderSizes.messageProcess='{m-> println "SUBJ: ${m.subject}"}' --config "operations.printFolderSizes.treeTraverseOrder='breadthFirst'"
 	Values trimmed - use quotes and escapes where appropriate''', required: false, args: 2, valueSeparator: '=', argName: 'property=value')
+cli.s(longOpt: 'account-src', 'Source account for proceed (account1)', required: true, args: 1)
+cli.d(longOpt: 'account-dst', 'Destination account for proceed (account2)', required: true, args: 1)
+cli.m(longOpt: 'memcached', 'Use memcached. See cache.memcached settings in config file. If there present actual (not expired) information for that name of account it will be used. Otherwise gathering on IMAP performed and such cache filled. Be careful, no any cache invalidation performed (for example if underlied Mail data changed of even settings)', required: false)
+cli.c(longOpt: 'copy', 'Copy missed mails. Only Source->Destination copy performed.', required: false)
+cli.r(longOpt: 'recheck', 'Implied --copy (does not run if no copy performed). Run again initial check. Now value of log.diff.FolderMessagesDiffLoggerFiles.dirRecheck will be used for log.diff.FolderMessagesDiffLoggerFiles.dirRecheck if such logging enabled at all.', required: false)
+
 OptionAccessor opt = cli.parse(args)
 GlobalConf.opt = opt;
 
+@Log4j2
+class ImapTreeDiffLog{}
+
 if(opt.h /*|| opt.arguments().isEmpty()*/ ) {
 	cli.usage()
 }
@@ -53,38 +64,56 @@ else {
 //println "GlobalConf.log.diff.FolderMessagesDiffLoggerFiles.files.perAnomaly('QWERTY')=${GlobalConf.log.diff.FolderMessagesDiffLoggerFiles.files.perAnomaly('QWERTY')}"
 //System.exit(101);
 
-// use memcached to do not long await start and information gathering:
-def mem = new MemcachedClientExtended(new InetSocketAddress('127.0.0.1', 11211));
+	ImapTreeSize tree1, tree2;
+	MemcachedClient mem;
+	if (opt.m){
+		// use memcached to do not long await start and information gathering:
+		mem = new MemcachedClientExtended(new InetSocketAddress(GlobalConf.memcached.host, GlobalConf.memcached.port));
+		ImapAccount accountSrc = GlobalConf.accounts[opt.'account-src'] as ImapAccount;
+		ImapAccount accountDst = GlobalConf.accounts[opt.'account-dst'] as ImapAccount;
+		tree1 = mem.getOrCreate("ImapTreeDiff.tree1.${accountSrc.name}", GlobalConf.memcached.cachetime, new ImapTreeTranscoder(accountSrc)) {
+			new ImapTreeSize(accountSrc)
+		} as ImapTreeSize;
+		tree2 = mem.getOrCreate("ImapTreeDiff.tree2.${accountDst.name}", GlobalConf.memcached.cachetime, new ImapTreeTranscoder(accountDst)) {
+			new ImapTreeSize(accountDst)
+		} as ImapTreeSize;
+	}
+	else{
+		tree1 = new ImapTreeSize((ImapAccount)GlobalConf.accounts[opt.'account-src'])
+		tree2 = new ImapTreeSize((ImapAccount)GlobalConf.accounts[opt.'account-dst'])
+	}
+
 
-ImapTreeSize tree1 = mem.getOrCreate('tree1', new ImapTreeTranscoder((ImapAccount)GlobalConf.accounts.Ant)) {
-	new ImapTreeSize(GlobalConf.accounts.Ant)
-};
-ImapTreeSize tree2 = mem.getOrCreate('tree2', new ImapTreeTranscoder((ImapAccount)GlobalConf.accounts.PahanTest)) {
-	new ImapTreeSize(GlobalConf.accounts.PahanTest)
-};
+//((IMAPFolder)tree2.tree.@folder).store.session.setDebug(true);
+//((IMAPFolder)tree2.tree.@folder).store.session.setDebugOut(System.err);
+//((IMAPFolder)tree2.tree.@folder).store.session.logger.logger.level = Level.ALL
 
-((IMAPFolder)tree2.tree.@folder).store.session.setDebug(true);
-((IMAPFolder)tree2.tree.@folder).store.session.setDebugOut(System.err);
-((IMAPFolder)tree2.tree.@folder).store.session.logger.logger.level = Level.ALL
+	FolderMessagesDiff foldersDiff = new FolderMessagesDiff(tree1.tree.@folder, tree2.tree.@folder);
+	foldersDiff.dump('Before copy missed messages');
 
-FolderMessagesDiff foldersDiff = new FolderMessagesDiff(tree1.tree.@folder, tree2.tree.@folder);
-foldersDiff.dump('Before copy missed messages');
+	// Change path to do not overwrite. It must be before copier created because it use config for log changes in destination
+	GlobalConf.log.diff.FolderMessagesDiffLoggerFiles.dir = GlobalConf.log.diff.FolderMessagesDiffLoggerFiles.dirRecheck
+	if(opt.copy){
+		ImapTreeDiffLog.log.info('Copying requested. Run.');
+		FolderMessagesCopier copier = new FolderMessagesCopier(foldersDiff);
+		copier.copyMissedMessagesToFolder2();
 
-// Change path to do not overwrite. It must be before copier created because it use config for log changes in destination
-GlobalConf.log.diff.FolderMessagesDiffLoggerFiles.dir = GlobalConf.log.diff.FolderMessagesDiffLoggerFiles.dirRecheck
-FolderMessagesCopier copier = new FolderMessagesCopier(foldersDiff);
-copier.copyMissedMessagesToFolder2();
+		// Recheck results
+		if (opt.recheck){
+			ImapTreeDiffLog.log.info('Copying requested. Run.');
+			new FolderMessagesDiff(foldersDiff.folder1messages.folder, foldersDiff.folder2messages.folder).dump('After copy missed messages');
+		}
 
-// Recheck results
-new FolderMessagesDiff(foldersDiff.folder1messages.folder, foldersDiff.folder2messages.folder).dump('After copy missed messages');
+		// Wait run message change listener thread. Unfortunately Java finalize is not same as destructor, so it code outside
+		ThreadGroup mainGroup = Thread.currentThread().threadGroup;
+		Thread[] activeThreads = new Thread[mainGroup.activeCount()];
+		mainGroup.enumerate(activeThreads, true);
+		activeThreads.findAll{ it != Thread.currentThread() }*.join(5000); // Wait all at least 5 seconds
+	}
 
-// Wait run message change listener thread
-ThreadGroup mainGroup = Thread.currentThread().threadGroup;
-Thread[] activeThreads = new Thread[mainGroup.activeCount()];
-mainGroup.enumerate(activeThreads, true);
-activeThreads.findAll{ it != Thread.currentThread() }*.join(10000); // Wait all at least 5 seconds
 
-println 'Done'
-// To do not hang
-mem.shutdown(10, TimeUnit.MILLISECONDS);
+	println 'Done'
+	// To do not hang
+	if(mem)
+		mem.shutdown(10, TimeUnit.MILLISECONDS);
 }

src/TODO

@@ -1,9 +1,9 @@
 TODO IMPLEMENT:
 (1) Extend CliBuilder like CliBuilderAutoWidth
 (2) Implement Traits chain messages logging and 2 base loggers: LogFile (base) and detailed per-file.
-3) Make ImapTreeDiff true configurable
+(3) Make ImapTreeDiff true configurable
 4) Fix sha1 in message caching
-5) Find minimum stable Gmail append problem (ideal 2 messages)
+(5) Find minimum stable Gmail append problem (ideal 2 messages) (see test.bug_1, test.bug_2).
 (6) Bring Config singleton in Groovy style (from ASA) - GlobalConf
 7) Investigate and fill bug from info.hubbitus.imaptree.utils.ConfigExtended.overrideFromListPropertiesPairs processing on command line options quoting
 8) Class SpentTest. Run test. Write tests.

src/info/hubbitus/imaptree/ImapTreeSize.groovy

@@ -17,6 +17,7 @@ import com.thoughtworks.xstream.core.util.QuickWriter
 import com.thoughtworks.xstream.io.HierarchicalStreamWriter
 import com.thoughtworks.xstream.io.xml.PrettyPrintWriter
 import com.thoughtworks.xstream.io.xml.StaxDriver
+import info.hubbitus.imaptree.config.GlobalConf
 import info.hubbitus.imaptree.config.ImapAccount
 import info.hubbitus.imaptree.config.Operation
 import info.hubbitus.imaptree.utils.bench.ProgressLogger
@@ -52,6 +53,7 @@ class ImapTreeSize {
 			)
 			, null
 		);
+		session.setDebug(GlobalConf.log.imapdebug);
 		Store store = session.getStore(account.type);
 		store.connect(account.host, account.login, account.password);
 		store
@@ -121,7 +123,9 @@ class ImapTreeSize {
 			finally {
 				f.close(false)
 			}
-		} else return new Size(onlyFolders: true, bytes: 0, messages: 0)
+		}
+		else
+			return new Size(onlyFolders: true, bytes: 0, messages: 0)
 	}
 
 	// Store text as CDATA sections for readability

src/info/hubbitus/imaptree/utils/cache/MessagesCache.groovy

@@ -53,24 +53,18 @@ class MessagesCache{
 
 	String messageToJson(IMAPMessage m, List<String> headers = [], boolean prettyPrint = false){
 //???	assert m.getSize() == m.getMimeStream().text.size()
-		Map essential
-		if(m.expunged){ // Only base info and prefetched early data
-			essential = extractMessageEssentials(m, headers);
-			IMAPMessage mByUid = uids[m.getUIDsafe()];
-			if (mByUid){
-				essential.cachedMessageWithThatUIDwas = extractMessageEssentials(mByUid, headers);
-			}
-			IMAPMessage mByMessageNumber = msgNumbers[m.getMessageNumber()];
-			if(mByMessageNumber){
-				essential.cachedMessageWithThatMessageNumberWas = extractMessageEssentials(mByMessageNumber, headers);
-			}
-			if (mByUid && mByMessageNumber){
-				assert mByUid.getUIDsafe() == mByMessageNumber.getUIDsafe()
-				assert mByUid.getMessageNumber() == mByMessageNumber.getMessageNumber()
-			}
+		Map essential = extractMessageEssentials(m, headers);
+		IMAPMessage mByUid = uids[m.getUIDsafe()];
+		if (mByUid){
+			essential.cachedMessageWithThatUIDwas = extractMessageEssentials(mByUid, headers);
+		}
+		IMAPMessage mByMessageNumber = msgNumbers[m.getMessageNumber()];
+		if(mByMessageNumber){
+			essential.cachedMessageWithThatMessageNumberWas = extractMessageEssentials(mByMessageNumber, headers);
 		}
-		else {
-			essential = extractMessageEssentials(m, headers);
+		if (mByUid && mByMessageNumber){
+			assert mByUid.getUIDsafe() == mByMessageNumber.getUIDsafe()
+			assert mByUid.getMessageNumber() == mByMessageNumber.getMessageNumber()
 		}
 		return (prettyPrint ? JsonOutput.prettyPrint(JsonOutput.toJson(essential)) : JsonOutput.toJson(essential));
 	}

src/log4j2.xml

@@ -11,7 +11,7 @@ http://logging.apache.org/log4j/2.0/manual/migration.html
 			<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{1} - %msg%n"/>
 		</Console>
 		<File name="LogFile" fileName="process.log" append="false">
-			<PatternLayout pattern="%t %-5p %c{2} - %m%n"/>
+			<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{1} - %msg%n"/>
 		</File>
 	</Appenders>
 	<Loggers>