Merge pull request stevencohn#1235 from stevencohn/1233-phantom-pages

delete phantom hashtag and hashtag_pages

OneMore/Commands/Tagging/HashtagCommand.cs

@@ -29,7 +29,7 @@ public override async Task Execute(params object[] args)
 		{
 			if (dialog != null)
 			{
-				logger.WriteLine("+++ dialog exists");
+				// single instance
 				dialog.ForceTopMost();
 				dialog.Activate();
 				dialog.TopMost = false;

OneMore/Commands/Tagging/HashtagDialog.cs

@@ -8,6 +8,7 @@ namespace River.OneMoreAddIn.Commands
 	using River.OneMoreAddIn.UI;
 	using System;
 	using System.Collections.Generic;
+	using System.Diagnostics;
 	using System.Drawing;
 	using System.Globalization;
 	using System.Linq;
@@ -306,8 +307,15 @@ private async void ScanNow(object sender, EventArgs e)
 			ShowScanTimes();
 
 			using var scanner = new HashtagScanner();
+
+			var clock = new Stopwatch();
+			clock.Start();
+
 			var totalPages = await scanner.Scan();
-			logger.WriteLine($"scanned {totalPages} pages");
+
+			clock.Stop();
+			var time = clock.ElapsedMilliseconds;
+			logger.WriteLine($"scanned {totalPages} pages in {time}ms");
 
 			PopulateTags(sender, e);
 		}

OneMore/Commands/Tagging/HashtagProvider.cs

@@ -2,6 +2,8 @@
 // Copyright © 2023 Steven M Cohn. All rights reserved.
 //************************************************************************************************
 
+#define xEnableUpgrade
+
 namespace River.OneMoreAddIn.Commands
 {
 	using River.OneMoreAddIn.Properties;
@@ -18,6 +20,8 @@ namespace River.OneMoreAddIn.Commands
 	/// </summary>
 	internal class HashtagProvider : Loggable, IDisposable
 	{
+		private const int ScannerID = 0;
+
 		private readonly SQLiteConnection con;
 		private readonly string timestamp;
 		private bool disposed;
@@ -40,6 +44,12 @@ public HashtagProvider()
 			{
 				RefreshDatabase();
 			}
+#if EnableUpgrade
+			else
+			{
+				UpgradeDatabase();
+			}
+#endif
 
 			timestamp = DateTime.Now.ToZuluString();
 		}
@@ -84,6 +94,105 @@ private void RefreshDatabase()
 		}
 
 
+		#region UpgradeDatabase
+#if EnableUpgrade
+		private void UpgradeDatabase()
+		{
+			using var cmd = con.CreateCommand();
+			cmd.CommandType = CommandType.Text;
+			cmd.CommandText = $"SELECT version FROM hashtag_scanner WHERE scannerID = {ScannerID}";
+
+			var version = 0;
+			try
+			{
+				using var reader = cmd.ExecuteReader();
+				if (reader.Read())
+				{
+					version = reader.GetInt32(0);
+				}
+			}
+			catch (Exception exc)
+			{
+				logger.WriteLine("error reading scanner version", exc);
+				return;
+			}
+
+			// upgrade incrementally...
+
+			if (version == 1)
+			{
+				version = Upgrade1to2(con);
+			}
+
+			//if (version == 2)
+			//{
+			//	version = Upgrade2to3(con);
+			//}
+		}
+
+
+		private int Upgrade1to2(SQLiteConnection con)
+		{
+			logger.WriteLine("upgrading database to version 2");
+			using var cmd = con.CreateCommand();
+
+			cmd.CommandType = CommandType.Text;
+			cmd.CommandText = "ALTER TABLE hashtag_page ADD COLUMN token NUMBER DEFAULT 0";
+
+			using var transaction = con.BeginTransaction();
+
+			try
+			{
+				cmd.ExecuteNonQuery();
+			}
+			catch (Exception exc)
+			{
+				logger.WriteLine("error upgrading hashtag_page v2", exc);
+				return 0;
+			}
+
+			try
+			{
+				cmd.CommandText = "CREATE INDEX IF NOT EXISTS IDX_token ON hashtag_page (token)";
+				cmd.ExecuteNonQuery();
+			}
+			catch (Exception exc)
+			{
+				logger.WriteLine("error upgrading hashtag_page v2", exc);
+				transaction.Rollback();
+				return 0;
+			}
+
+			try
+			{
+				cmd.CommandText = "UPDATE hashtag_scanner " +
+					$"SET version = 2 WHERE scannerID = {ScannerID}";
+
+				cmd.ExecuteNonQuery();
+			}
+			catch (Exception exc)
+			{
+				logger.WriteLine("error upgrading hashtag_page v2", exc);
+				transaction.Rollback();
+				return 0;
+			}
+
+			try
+			{
+				transaction.Commit();
+			}
+			catch (Exception exc)
+			{
+				logger.WriteLine("error committing hashtag_page v2", exc);
+				return 0;
+			}
+
+			return 2;
+		}
+#endif
+		#endregion UpgradeDatabase
+
+
 		protected virtual void Dispose(bool disposing)
 		{
 			if (!disposed)
@@ -105,6 +214,64 @@ public void Dispose()
 		}
 
 
+		/// <summary>
+		/// Compares the recorded pages against the list of knownIDs and deletes any
+		/// records no longer in that list.
+		/// </summary>
+		/// <param name="knownIDs"></param>
+		public void DeletePhantoms(List<string> knownIDs, string sectionID, string sectionPath)
+		{
+			using var cmd = con.CreateCommand();
+			cmd.CommandType = CommandType.Text;
+			cmd.CommandText = "SELECT moreID, pageID FROM hashtag_page WHERE sectionID = @sid";
+			cmd.Parameters.AddWithValue("@sid", sectionID);
+
+			using var dtagcmd = con.CreateCommand();
+			dtagcmd.CommandType = CommandType.Text;
+			dtagcmd.CommandText = "DELETE FROM hashtag WHERE moreID = @mid";
+			dtagcmd.Parameters.Add("@mid", DbType.String);
+
+			using var dpagcmd = con.CreateCommand();
+			dpagcmd.CommandType = CommandType.Text;
+			dpagcmd.CommandText = "DELETE FROM hashtag_page WHERE pageID = @pid";
+			dpagcmd.Parameters.Add("@pid", DbType.String);
+
+			using var transaction = con.BeginTransaction();
+			var count = 0;
+
+			try
+			{
+				using var reader = cmd.ExecuteReader();
+				while (reader.Read())
+				{
+					var pageID = reader.GetString(1);
+					if (!knownIDs.Contains(pageID))
+					{
+						var moreID = reader.GetString(0);
+
+						dtagcmd.Parameters["@mid"].Value = moreID;
+						dtagcmd.ExecuteNonQuery();
+
+						dpagcmd.Parameters["@pid"].Value = pageID;
+						dpagcmd.ExecuteNonQuery();
+						count++;
+					}
+				}
+
+				if (count > 0)
+				{
+					transaction.Commit();
+					logger.WriteLine($"deleted {count} phantom pages from {sectionPath}");
+				}
+			}
+			catch (Exception exc)
+			{
+				transaction.Rollback();
+				logger.WriteLine("error deleting phantom pages", exc);
+			}
+		}
+
+
 		/// <summary>
 		/// Deletes the specified tags
 		/// </summary>
@@ -179,14 +346,14 @@ private void CleanupPages()
 		public string ReadScanTime()
 		{
 			using var cmd = con.CreateCommand();
-			cmd.CommandText = "SELECT version, scanTime FROM hashtag_scanner WHERE scannerID = 0";
+			cmd.CommandText = 
+				$"SELECT version, scanTime FROM hashtag_scanner WHERE scannerID = {ScannerID}";
 
 			try
 			{
 				using var reader = cmd.ExecuteReader();
 				if (reader.Read())
 				{
-					// var version = reader.GetInt32(0); // upgrade?
 					return reader.GetString(1);
 				}
 			}

OneMore/Commands/Tagging/HashtagScanner.cs

@@ -6,6 +6,7 @@ namespace River.OneMoreAddIn.Commands
 {
 	using River.OneMoreAddIn.Models;
 	using System;
+	using System.Collections.Generic;
 	using System.Linq;
 	using System.Threading.Tasks;
 	using System.Xml.Linq;
@@ -114,19 +115,27 @@ private async Task<int> Scan(XElement parent, string notebookID, string path)
 					var pages = section.Elements(ns + "Page")
 						.Where(e => e.Attribute("isInRecycleBin") == null);
 
-					totalPages += pages.Count();
+					if (pages.Any())
+					{
+						totalPages += pages.Count();
+						var pids = new List<string>();
 
-					var sectionID = section.Attribute("ID").Value;
-					var sectionPath = $"{path}/{section.Attribute("name").Value}";
-					//logger.Verbose($"scanning section {sectionPath} ({pages.Count()} pages)");
+						var sectionID = section.Attribute("ID").Value;
+						var sectionPath = $"{path}/{section.Attribute("name").Value}";
+						logger.Verbose($"scanning section {sectionPath} ({pages.Count()} pages)");
 
-					foreach (var page in pages)
-					{
-						if (page.Attribute("lastModifiedTime").Value.CompareTo(lastTime) > 0)
+						foreach (var page in pages)
 						{
-							await ScanPage(
-								page.Attribute("ID").Value, notebookID, sectionID, sectionPath);
+							var pid = page.Attribute("ID").Value;
+							pids.Add(pid);
+
+							if (page.Attribute("lastModifiedTime").Value.CompareTo(lastTime) > 0)
+							{
+								await ScanPage(pid, notebookID, sectionID, sectionPath);
+							}
 						}
+
+						provider.DeletePhantoms(pids, sectionID, sectionPath);
 					}
 				}
 			}

OneMore/Properties/Resources.ar-SA.resx

@@ -2371,7 +2371,9 @@ ISO-code then comma then language name</comment>
     <comment>combobox</comment>
   </data>
   <data name="ResizeImagesDialog_noImages" xml:space="preserve">
-    <value>لم يتم العثور على صور في هذه الصفحة</value>
+    <value>لم يتم العثور على صور في هذه الصفحة.
+
+يجب أن تكون الصور في حاوية المخطط التفصيلي. لا يمكن تعديل صور الخلفية.</value>
     <comment>message box</comment>
   </data>
   <data name="ResizeImagesDialog_pctRadio.Text" xml:space="preserve">

OneMore/Properties/Resources.de-DE.resx

@@ -2365,7 +2365,9 @@ Vergrößere kleinere Bilder nicht</value>
     <comment>combobox</comment>
   </data>
   <data name="ResizeImagesDialog_noImages" xml:space="preserve">
-    <value>Auf dieser Seite wurden keine Bilder gefunden</value>
+    <value>Auf dieser Seite wurden keine Bilder gefunden.
+
+Bilder müssen sich in einem Gliederungscontainer befinden. Hintergrundbilder können nicht angepasst werden.</value>
     <comment>message box</comment>
   </data>
   <data name="ResizeImagesDialog_pctRadio.Text" xml:space="preserve">

OneMore/Properties/Resources.es-ES.resx

@@ -2371,7 +2371,9 @@ No ampliar imágenes más pequeñas</value>
     <comment>combobox</comment>
   </data>
   <data name="ResizeImagesDialog_noImages" xml:space="preserve">
-    <value>No se encontraron imágenes en esta página.</value>
+    <value>No se encontraron imágenes en esta página.
+
+Las imágenes deben estar en un contenedor de contorno. Las imágenes de fondo no se pueden ajustar.</value>
     <comment>message box</comment>
   </data>
   <data name="ResizeImagesDialog_pctRadio.Text" xml:space="preserve">

OneMore/Properties/Resources.fr-FR.resx

@@ -2367,7 +2367,9 @@ Ne pas agrandir les petites images</value>
     <comment>combobox</comment>
   </data>
   <data name="ResizeImagesDialog_noImages" xml:space="preserve">
-    <value>Aucune image n'a été trouvée sur cette page</value>
+    <value>Aucune image n'a été trouvée sur cette page.
+
+Les images doivent être dans un conteneur de plan. Les images d'arrière-plan ne peuvent pas être ajustées.</value>
     <comment>message box</comment>
   </data>
   <data name="ResizeImagesDialog_pctRadio.Text" xml:space="preserve">

OneMore/Properties/Resources.he-IL.resx

@@ -2390,7 +2390,9 @@ ISO-code then comma then language name</comment>
     <comment>combobox</comment>
   </data>
   <data name="ResizeImagesDialog_noImages" xml:space="preserve">
-    <value>לא נמצאו תמונות בדף זה</value>
+    <value>לא נמצאו תמונות בדף זה.
+
+התמונות חייבות להיות במיכל מתאר. לא ניתן לכוונן תמונות רקע.</value>
     <comment>message box</comment>
   </data>
   <data name="ResizeImagesDialog_pctRadio.Text" xml:space="preserve">

OneMore/Properties/Resources.nl-NL.resx

@@ -2372,7 +2372,9 @@ Kleinere afbeeldingen niet vergroten</value>
     <comment>combobox</comment>
   </data>
   <data name="ResizeImagesDialog_noImages" xml:space="preserve">
-    <value>Er zijn geen afbeeldingen gevonden op deze pagina</value>
+    <value>Er zijn geen afbeeldingen gevonden op deze pagina.
+
+Afbeeldingen moeten zich in een overzichtscontainer bevinden. Achtergrondafbeeldingen kunnen niet worden aangepast.</value>
     <comment>message box</comment>
   </data>
   <data name="ResizeImagesDialog_pctRadio.Text" xml:space="preserve">

OneMore/Properties/Resources.pl-PL.resx

@@ -2384,7 +2384,9 @@ Nie powiększaj mniejszych obrazów</value>
     <comment>combobox</comment>
   </data>
   <data name="ResizeImagesDialog_noImages" xml:space="preserve">
-    <value>Na tej stronie nie znaleziono żadnych obrazów</value>
+    <value>Na tej stronie nie znaleziono żadnych obrazów.
+
+Obrazy muszą znajdować się w kontenerze konspektu. Nie można regulować obrazów tła.</value>
     <comment>message box</comment>
   </data>
   <data name="ResizeImagesDialog_pctRadio.Text" xml:space="preserve">