Option to download url list

- Adds an option to download an url list instead of the binary files
themselves (#42).

src/TumblThree/SharedAssemblyInfo.cs

@@ -15,5 +15,5 @@
 
 [assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.MainAssembly)]
 
-[assembly: AssemblyVersion("1.0.4.33")]
-[assembly: AssemblyFileVersion("1.0.4.33")]
+[assembly: AssemblyVersion("1.0.4.34")]
+[assembly: AssemblyFileVersion("1.0.4.34")]

src/TumblThree/TumblThree.Applications/Controllers/DetailsController.cs

@@ -118,6 +118,7 @@ public TumblrBlog CreateFromMultiple(IEnumerable<TumblrBlog> blogFiles)
                 ForceSize = false,
                 ForceRescan = false,
                 CheckDirectoryForFiles = false,
+                DownloadUrlList = false,
                 Dirty = false
             };
         }
@@ -162,6 +163,7 @@ private void SaveChanges(TumblrBlog blogFile)
                     blog.ForceSize = blogFile.ForceSize;
                     blog.ForceRescan = blogFile.ForceRescan;
                     blog.CheckDirectoryForFiles = blogFile.CheckDirectoryForFiles;
+                    blog.DownloadUrlList = blogFile.DownloadUrlList;
                     blog.Dirty = true;                
                 }
             }

src/TumblThree/TumblThree.Applications/Downloader/Downloader.cs

@@ -19,7 +19,10 @@ public class Downloader : IDownloader
     {
         private readonly IBlog blog;
         private readonly IShellService shellService;
-        protected readonly object lockObject;
+        protected readonly object lockObjectDb;
+        protected readonly object lockObjectDirectory;
+        protected readonly object lockObjectDownload;
+
 
         public Downloader(IShellService shellService): this(shellService, null)
         {
@@ -29,7 +32,9 @@ public Downloader(IShellService shellService, IBlog blog)
         {
             this.shellService = shellService;
             this.blog = blog;
-            this.lockObject = new object();
+            this.lockObjectDb = new object();
+            this.lockObjectDirectory = new object();
+            this.lockObjectDownload = new object();
             SetUp();
         }
 
@@ -133,13 +138,13 @@ protected virtual void SetUp()
         protected virtual bool CheckIfFileExistsInDB(string url)
         {
             var fileName = url.Split('/').Last();
-            Monitor.Enter(lockObject);
+            Monitor.Enter(lockObjectDb);
             if (blog.Links.Contains(fileName))
             {
-                Monitor.Exit(lockObject);
+                Monitor.Exit(lockObjectDb);
                 return true;
             }
-            Monitor.Exit(lockObject);
+            Monitor.Exit(lockObjectDb);
             return false;
         }
 
@@ -155,14 +160,14 @@ protected virtual bool CheckIfBlogShouldCheckDirectory(string url)
         protected virtual bool CheckIfFileExistsInDirectory(string url)
         {
             var fileName = url.Split('/').Last();
-            Monitor.Enter(lockObject);
+            Monitor.Enter(lockObjectDirectory);
             string blogPath = Path.Combine(Directory.GetParent(blog.Location).FullName, blog.Name);
             if (System.IO.File.Exists(Path.Combine(blogPath, fileName)))
             {
-                Monitor.Exit(lockObject);
+                Monitor.Exit(lockObjectDirectory);
                 return true;
             }
-            Monitor.Exit(lockObject);
+            Monitor.Exit(lockObjectDirectory);
             return false;
         }
 
@@ -200,9 +205,12 @@ protected virtual bool AppendToTextFile(string fileLocation, string text)
         {
             try
             {
-                using (StreamWriter sw = new StreamWriter(fileLocation, true))
+                lock (lockObjectDownload)
                 {
-                    sw.WriteLine(text);
+                    using (StreamWriter sw = new StreamWriter(fileLocation, true))
+                    {
+                        sw.WriteLine(text);
+                    }
                 }
                 return true;
             }

src/TumblThree/TumblThree.Applications/Downloader/TumblrDownloader.cs

@@ -30,7 +30,6 @@ public class TumblrDownloader : Downloader
         private readonly TumblrBlog blog;
         private TumblrFiles files;
         private readonly object lockObjectProgress;
-        private readonly object lockObjectDownload;
         private readonly List<Tuple<PostTypes, string, string>> downloadList;
         private readonly BlockingCollection<Tuple<PostTypes, string, string>> sharedDownloads;
 
@@ -42,7 +41,6 @@ public TumblrDownloader(IShellService shellService, ICrawlerService crawlerServi
             this.blog = (TumblrBlog)blog;
             this.files = LoadTumblrFiles();
             this.lockObjectProgress = new object();
-            this.lockObjectDownload = new object();
             this.downloadList = new List<Tuple<PostTypes, string, string>>();
             this.sharedDownloads = new BlockingCollection<Tuple<PostTypes, string, string>>();
         }
@@ -240,13 +238,13 @@ private ulong GetHighestPostId()
         protected override bool CheckIfFileExistsInDB(string url)
         {
             var fileName = url.Split('/').Last();
-            Monitor.Enter(lockObject);
+            Monitor.Enter(lockObjectDb);
             if (files.Links.Contains(fileName))
             {
-                Monitor.Exit(lockObject);
+                Monitor.Exit(lockObjectDb);
                 return true;
             }
-            Monitor.Exit(lockObject);
+            Monitor.Exit(lockObjectDb);
             return false;
         }
 
@@ -657,6 +655,18 @@ private TumblrFiles LoadTumblrFiles()
             }
         }
 
+        protected virtual bool DownloadBinaryFile(string fileLocation, string fileLocationUrlList, string url)
+        {
+            if (!blog.DownloadUrlList)
+            {
+                return DownloadBinaryFile(fileLocation, url);
+            }
+            else
+            {
+                return AppendToTextFile(fileLocationUrlList, url);
+            }
+        }
+
         private bool DownloadTumblrBlog(IProgress<DataModels.DownloadProgress> progress, CancellationToken ct, PauseToken pt)
         {
             int downloadedFiles = blog.DownloadedImages;
@@ -690,6 +700,7 @@ private bool DownloadTumblrBlog(IProgress<DataModels.DownloadProgress> progress,
                                 string fileName = String.Empty;
                                 string url = String.Empty;
                                 string fileLocation = String.Empty;
+                                string fileLocationUrlList = String.Empty;
                                 string postId = String.Empty;
 
                                 // FIXME: Conditional with Polymorphism
@@ -699,11 +710,12 @@ private bool DownloadTumblrBlog(IProgress<DataModels.DownloadProgress> progress,
                                         fileName = currentImageUrl.Item2.Split('/').Last();
                                         url = currentImageUrl.Item2;
                                         fileLocation = Path.Combine(Path.Combine(blogPath, blog.Name), fileName);
+                                        fileLocationUrlList = Path.Combine(Path.Combine(blogPath, blog.Name), string.Format(CultureInfo.CurrentCulture, Resources.FileNamePhotos));
 
                                         if (!(CheckIfFileExistsInDB(url) || CheckIfBlogShouldCheckDirectory(GetTumblrCoreImageUrl(url))))
                                         {
                                             UpdateProgressQueueInformation(progress, fileName);
-                                            DownloadBinaryFile(fileLocation, url);
+                                            DownloadBinaryFile(fileLocation, fileLocationUrlList, url);
                                             UpdateBlogCounter(ref downloadedPhotos, ref downloadedFiles);
                                             UpdateBlogProgress(fileName, ref downloadedFiles);
                                             blog.DownloadedPhotos = downloadedPhotos;
@@ -720,11 +732,12 @@ private bool DownloadTumblrBlog(IProgress<DataModels.DownloadProgress> progress,
                                         fileName = currentImageUrl.Item2.Split('/').Last();
                                         url = currentImageUrl.Item2;
                                         fileLocation = Path.Combine(Path.Combine(blogPath, blog.Name), fileName);
+                                        fileLocationUrlList = Path.Combine(Path.Combine(blogPath, blog.Name), string.Format(CultureInfo.CurrentCulture, Resources.FileNameVideos));
 
                                         if (!(CheckIfFileExistsInDB(url) || CheckIfBlogShouldCheckDirectory(url)))
                                         {
                                             UpdateProgressQueueInformation(progress, fileName);
-                                            DownloadBinaryFile(fileLocation, url);
+                                            DownloadBinaryFile(fileLocation, fileLocationUrlList, url);
                                             UpdateBlogCounter(ref downloadedVideos, ref downloadedFiles);
                                             UpdateBlogProgress(fileName, ref downloadedFiles);
                                             blog.DownloadedVideos = downloadedVideos;
@@ -738,11 +751,12 @@ private bool DownloadTumblrBlog(IProgress<DataModels.DownloadProgress> progress,
                                         fileName = currentImageUrl.Item2.Split('/').Last();
                                         url = currentImageUrl.Item2;
                                         fileLocation = Path.Combine(Path.Combine(blogPath, blog.Name), currentImageUrl.Item3 + ".swf");
+                                        fileLocationUrlList = Path.Combine(Path.Combine(blogPath, blog.Name), string.Format(CultureInfo.CurrentCulture, Resources.FileNameAudios));
 
                                         if (!(CheckIfFileExistsInDB(url) || CheckIfBlogShouldCheckDirectory(url)))
                                         {
                                             UpdateProgressQueueInformation(progress, fileName);
-                                            DownloadBinaryFile(fileLocation, url);
+                                            DownloadBinaryFile(fileLocation, fileLocationUrlList, url);
                                             UpdateBlogCounter(ref downloadedAudios, ref downloadedFiles);
                                             UpdateBlogProgress(fileName, ref downloadedFiles);
                                             blog.DownloadedAudios = downloadedAudios;

src/TumblThree/TumblThree.Applications/Properties/AppSettings.cs

@@ -169,6 +169,9 @@ public AppSettings()
         [DataMember]
         public bool CheckDirectoryForFiles { get; set; }
 
+        [DataMember]
+        public bool DownloadUrlList { get; set; }
+
         [DataMember]
         public string ProxyHost { get; set; }
 
@@ -254,6 +257,7 @@ private void Initialize()
             TimerInterval = "22:40:00";
             ForceSize = false;
             CheckDirectoryForFiles = false;
+            DownloadUrlList = false;
             ProxyHost = String.Empty;
             ProxyPort = String.Empty;
             ColumnWidths = new Dictionary<object, Tuple<int, double>>();

src/TumblThree/TumblThree.Applications/Properties/Resources.de.resx

@@ -273,4 +273,13 @@
   <data name="NewVersionAvailable" xml:space="preserve">
     <value>Neuere Version erhältlich : {0}</value>
   </data>
+  <data name="FileNameAudios" xml:space="preserve">
+    <value>audios_url.txt</value>
+  </data>
+  <data name="FileNamePhotos" xml:space="preserve">
+    <value>bilder_url.txt</value>
+  </data>
+  <data name="FileNameVideos" xml:space="preserve">
+    <value>videos_url.txt</value>
+  </data>
 </root>

src/TumblThree/TumblThree.Applications/Properties/Resources.resx

@@ -189,6 +189,9 @@
   <data name="DiskFull" xml:space="preserve">
     <value>There is not enough space on the disk.</value>
   </data>
+  <data name="FileNameAudios" xml:space="preserve">
+    <value>audios_url.txt</value>
+  </data>
   <data name="FileNameConversations" xml:space="preserve">
     <value>conversations.txt</value>
   </data>
@@ -204,12 +207,18 @@
   <data name="FileNameMetaVideo" xml:space="preserve">
     <value>videos.txt</value>
   </data>
+  <data name="FileNamePhotos" xml:space="preserve">
+    <value>images_url.txt</value>
+  </data>
   <data name="FileNameQuotes" xml:space="preserve">
     <value>quotes.txt</value>
   </data>
   <data name="FileNameTexts" xml:space="preserve">
     <value>texts.txt</value>
   </data>
+  <data name="FileNameVideos" xml:space="preserve">
+    <value>videos_url.txt</value>
+  </data>
   <data name="Id3Album" xml:space="preserve">
     <value>Id3: Album: {0}</value>
   </data>

src/TumblThree/TumblThree.Applications/ViewModels/SettingsViewModel.cs

@@ -52,6 +52,7 @@ public class SettingsViewModel : ViewModel<ISettingsView>
         private bool removeIndexAfterCrawl;
         private bool forceSize;
         private bool checkDirectoryForFiles;
+        private bool downloadUrlList;
         private string proxyHost;
         private string proxyPort;
         private bool downloadImages;
@@ -273,6 +274,12 @@ public bool CheckDirectoryForFiles
             set { SetProperty(ref checkDirectoryForFiles, value); }
         }
 
+        public bool DownloadUrlList
+        {
+            get { return downloadUrlList; }
+            set { SetProperty(ref downloadUrlList, value); }
+        }
+
         public string ProxyHost
         {
             get { return proxyHost; }
@@ -495,6 +502,7 @@ private void LoadSettings()
                 AutoDownload = settings.AutoDownload;
                 ForceSize = settings.ForceSize;
                 CheckDirectoryForFiles = settings.CheckDirectoryForFiles;
+                DownloadUrlList = settings.DownloadUrlList;
                 ProxyHost = settings.ProxyHost;
                 ProxyPort = settings.ProxyPort;
                 TimerInterval = settings.TimerInterval;
@@ -539,6 +547,7 @@ private void LoadSettings()
                 AutoDownload = false;
                 ForceSize = false;
                 CheckDirectoryForFiles = false;
+                DownloadUrlList = false;
                 ProxyHost = String.Empty;
                 ProxyPort = String.Empty;
                 TimerInterval = "22:40:00";
@@ -590,6 +599,7 @@ private void SaveSettings()
             settings.AutoDownload = AutoDownload;
             settings.ForceSize = ForceSize;
             settings.CheckDirectoryForFiles = CheckDirectoryForFiles;
+            settings.DownloadUrlList = DownloadUrlList;
             settings.ProxyHost = ProxyHost;
             settings.ProxyPort = ProxyPort;
             settings.TimerInterval = TimerInterval;

src/TumblThree/TumblThree.Domain/Models/Blog.cs

@@ -24,6 +24,7 @@ public abstract class Blog : Model, IBlog
         private bool online;
         private bool dirty;
         private bool checkDirectoryForFiles;
+        private bool downloadUrlList;
         private string notes;
         private IList<string> links;
         private Exception loadError;
@@ -130,6 +131,12 @@ public bool CheckDirectoryForFiles
             set { SetProperty(ref checkDirectoryForFiles, value); Dirty = true; }
         }
 
+        public bool DownloadUrlList
+        {
+            get { return downloadUrlList; }
+            set { SetProperty(ref downloadUrlList, value); Dirty = true; }
+        }
+
         public bool Dirty
         {
             get { return dirty; }