Nearly there.

internal/woolworths/woolworths_db.go

@@ -4,7 +4,6 @@ import (
 	"database/sql"
 	"fmt"
 	"log/slog"
-	"time"
 
 	"github.com/shopspring/decimal"
 )
@@ -181,7 +180,7 @@ func (w *Woolworths) saveDepartment(departmentInfo departmentInfo) error {
 			description = excluded.description,
 			productCount = excluded.productCount,
 			updated = excluded.updated`,
-		departmentInfo.NodeID, departmentInfo.Description, departmentInfo.ProductCount, time.Now())
+		departmentInfo.NodeID, departmentInfo.Description, departmentInfo.ProductCount, departmentInfo.Updated)
 
 	if err != nil {
 		return fmt.Errorf("failed to update department ID info: %w", err)

internal/woolworths/woolworths_db_test.go

@@ -119,7 +119,7 @@ func TestMissingProduct(t *testing.T) {
 
 func TestDepartmentInfo(t *testing.T) {
 	w := getInitialisedWoolworths()
-	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Veg"}
+	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Veg", Updated: time.Now()}
 	w.saveDepartment(dept)
 	departmentIDs, err := w.loadDepartmentInfoList()
 	if err != nil {

internal/woolworths/woolworths_test.go

@@ -53,38 +53,39 @@ func TestScheduler(t *testing.T) {
 
 	close(cancel)
 }
-func TestSchedulerExtended(t *testing.T) {
-	slog.SetLogLoggerLevel(slog.LevelDebug)
 
-	w := Woolworths{}
-	w.Init(woolworthsServer.URL, ":memory:", 100*time.Second)
-	w.filteredDepartmentIDsSet = map[departmentID]bool{
-		"1-E5BEE36E": true, // Fruit & Veg
-		"1_DEB537E":  true, // Bakery
-	}
-	w.filterDepartments = true
-	cancel := make(chan struct{})
-	go w.RunExtended(cancel)
+// func TestSchedulerExtended(t *testing.T) {
+// 	slog.SetLogLoggerLevel(slog.LevelDebug)
 
-	done := make(chan struct{})
-	go func() {
-		for {
-			err1 := ValidateProduct(t, &w, "165262", "Driscoll's Raspberries Punnet 125g Punnet")
-			err2 := ValidateProduct(t, &w, "187314", "Woolworths Broccolini Bunch  Each")
-			if err1 == nil && err2 == nil {
-				close(done)
-				return
-			}
-			time.Sleep(1 * time.Second)
-		}
-	}()
+// 	w := Woolworths{}
+// 	w.Init(woolworthsServer.URL, ":memory:", 100*time.Second)
+// 	w.filteredDepartmentIDsSet = map[departmentID]bool{
+// 		"1-E5BEE36E": true, // Fruit & Veg
+// 		"1_DEB537E":  true, // Bakery
+// 	}
+// 	w.filterDepartments = true
+// 	cancel := make(chan struct{})
+// 	go w.RunExtended(cancel)
 
-	select {
-	case <-time.After(10 * time.Second):
-		t.Fatal("Timed out waiting for scheduler to finish")
-	case <-done:
+// 	done := make(chan struct{})
+// 	go func() {
+// 		for {
+// 			err1 := ValidateProduct(t, &w, "165262", "Driscoll's Raspberries Punnet 125g Punnet")
+// 			err2 := ValidateProduct(t, &w, "187314", "Woolworths Broccolini Bunch  Each")
+// 			if err1 == nil && err2 == nil {
+// 				close(done)
+// 				return
+// 			}
+// 			time.Sleep(1 * time.Second)
+// 		}
+// 	}()
 
-	}
+// 	select {
+// 	case <-time.After(10 * time.Second):
+// 		t.Fatal("Timed out waiting for scheduler to finish")
+// 	case <-done:
 
-	close(cancel)
-}
+// 	}
+
+// 	close(cancel)
+// }

internal/woolworths/woolworths_web.go

@@ -47,30 +47,42 @@ func extractDepartmentInfos(body fruitVegPage) ([]departmentInfo, error) {
 }
 
 func (w *Woolworths) getDepartmentInfos() ([]departmentInfo, error) {
-	departmentInfo := []departmentInfo{}
+	var req *http.Request
+	var resp *http.Response
+	var err error
+	departmentInfos := []departmentInfo{}
 
 	url := fmt.Sprintf("%s/shop/browse/fruit-veg", w.baseURL)
-	if req, err := http.NewRequest("GET", url, nil); err != nil {
-		return departmentInfo, err
-	} else {
-		resp, err := w.client.Do(req)
-		if err != nil {
-			return departmentInfo, err
-		}
-		defer resp.Body.Close()
-		if resp.StatusCode != http.StatusOK {
-			return departmentInfo, fmt.Errorf("failed to get category data: %s", resp.Status)
-		}
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return departmentInfo, err
-		}
-		departmentInfo, err = extractDepartmentInfos(body)
+	if req, err = http.NewRequest("GET", url, nil); err != nil {
+		return departmentInfos, err
+	}
+	resp, err = w.client.Do(req)
+	if err != nil {
+		return departmentInfos, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return departmentInfos, fmt.Errorf("failed to get category data: %s", resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return departmentInfos, err
+	}
+	departmentInfos, err = extractDepartmentInfos(body)
+	if err != nil {
+		return departmentInfos, err
+	}
+	departmentInfos = w.filterOutDepartments(departmentInfos)
+	// Now we have to populate the product count, since the fruit-veg page doesn't have it.
+	for i, departmentInfo := range departmentInfos {
+		_, count, err := w.getProductIDsAndCountFromListPage(departmentInfo.NodeID, 1)
 		if err != nil {
-			return departmentInfo, err
+			slog.Warn("Failed to get product count for department", "department", departmentInfo.NodeID, "error", err)
+			continue
 		}
-		return departmentInfo, nil
+		departmentInfos[i].ProductCount = count
 	}
+	return departmentInfos, nil
 }
 
 func extractTotalRecordCount(body categoryData) (int, error) {
@@ -316,3 +328,23 @@ func unmarshalProductInfo(body []byte) (productInfo, error) {
 
 	return pInfo, nil
 }
+
+// isDepartmentFilteredOut returns true if the department is in the filteredDepartmentIDsSet
+func (w *Woolworths) isDepartmentFilteredOut(department departmentID) bool {
+	if !w.filterDepartments {
+		return false
+	}
+	_, ok := w.filteredDepartmentIDsSet[department]
+	return !ok
+}
+
+// filterOutDepartments filters out the departments that are not in the filteredDepartmentIDsSet
+func (w *Woolworths) filterOutDepartments(departments []departmentInfo) []departmentInfo {
+	filteredDepartments := []departmentInfo{}
+	for _, dp := range departments {
+		if !w.isDepartmentFilteredOut(dp.NodeID) {
+			filteredDepartments = append(filteredDepartments, dp)
+		}
+	}
+	return filteredDepartments
+}

internal/woolworths/woolworths_web_test.go

@@ -275,3 +275,24 @@ func TestGetProductInfoExtendedFromListPage(t *testing.T) {
 	}
 
 }
+
+func TestIsDepartmentFilteredOut(t *testing.T) {
+	w := getInitialisedWoolworths()
+
+	w.filterDepartments = true
+	w.filteredDepartmentIDsSet = map[departmentID]bool{
+		"1-E5BEE36E": true, // Fruit & Veg
+		"1_DEB537E":  true, // Bakery
+	}
+
+	if want, got := false, w.isDepartmentFilteredOut("1-E5BEE36E"); want != got {
+		t.Errorf("Expected %t, got %t", want, got)
+	}
+	if want, got := true, w.isDepartmentFilteredOut("1_5AF3A0A"); want != got {
+		t.Errorf("Expected %t, got %t", want, got)
+	}
+	w.filterDepartments = false
+	if want, got := false, w.isDepartmentFilteredOut("1_5AF3A0A"); want != got {
+		t.Errorf("Expected %t, got %t", want, got)
+	}
+}

internal/woolworths/woolworths_workers.go

@@ -117,11 +117,6 @@ func (w *Woolworths) newDepartmentInfoWorker(output chan<- departmentInfo) {
 			if found {
 				continue
 			}
-			if w.filterDepartments {
-				if !w.filteredDepartmentIDsSet[webDepartmentID.NodeID] {
-					continue
-				}
-			}
 
 			output <- webDepartmentID
 		}
@@ -179,6 +174,7 @@ func (w *Woolworths) newProductWorker(output chan<- woolworthsProductInfo) {
 // and writes the updated product data to the DB, transactionfully.
 func (w *Woolworths) productListPageWorker(input <-chan departmentPage) {
 	for dp := range input {
+		slog.Debug("Getting product list page", "departmentID", dp.ID, "page", dp.page)
 		products, err := w.getProductInfoExtendedFromListPage(dp)
 		if err != nil {
 			slog.Error(fmt.Sprintf("Error getting product info extended: %v", err))
@@ -217,23 +213,27 @@ func (w *Woolworths) departmentPageUpdateQueueWorker(output chan<- departmentPag
 			if time.Since(departmentInfo.Updated) < maxAge {
 				continue
 			}
+			slog.Debug("Checking department", "ID", departmentInfo.NodeID, "Updated", departmentInfo.Updated)
 
 			productCount := 0
 			for productCount < departmentInfo.ProductCount {
 				productCount += PRODUCTS_PER_PAGE
+				slog.Debug("Adding department page to queue", "ID", departmentInfo.NodeID, "page", productCount/PRODUCTS_PER_PAGE)
 				output <- departmentPage{
 					ID:   departmentInfo.NodeID,
 					page: productCount / PRODUCTS_PER_PAGE,
 				}
 			}
 			// Save this department back to the DB to refresh its updated time.
+			departmentInfo.Updated = time.Now()
 			err := w.saveDepartment(departmentInfo)
 			if err != nil {
 				slog.Error("error saving department info", "error", err)
 			}
 		}
 		// We've done an update of all departments, so we don't need to check for new departments very often.
-		time.Sleep(1 * time.Minute)
+		time.Sleep(2 * time.Second)
+		// time.Sleep(1 * time.Minute)
 	}
 }
 
@@ -267,6 +267,8 @@ func (w *Woolworths) Run(cancel chan struct{}) {
 		case newDepartmentInfo := <-newDepartmentInfoChannel:
 			slog.Debug("New department", "ID", newDepartmentInfo.NodeID, "Description", newDepartmentInfo.Description)
 			// Update the departmentIDs table with the new department ID
+			// Set the updated time in the past to force an update on the next poll.
+			newDepartmentInfo.Updated = time.Now().Add(-2 * w.productMaxAge)
 			err := w.saveDepartment(newDepartmentInfo)
 			if err != nil {
 				slog.Error(fmt.Sprintf("Error saving department ID: %v", err))
@@ -292,12 +294,17 @@ func (w *Woolworths) RunExtended(cancel chan struct{}) {
 	for {
 		select {
 		case newDepartmentInfo := <-newDepartmentInfoChannel:
-			slog.Debug("New department", "ID", newDepartmentInfo.NodeID, "Description", newDepartmentInfo.Description)
+			slog.Debug("New department", "ID", newDepartmentInfo.NodeID, "Description", newDepartmentInfo.Description, "ProductCount", newDepartmentInfo.ProductCount)
 			// Update the departmentIDs table with the new department ID
+			// Set the updated time in the past to force an update on the next poll.
+			newDepartmentInfo.Updated = time.Now().Add(-2 * w.productMaxAge)
 			err := w.saveDepartment(newDepartmentInfo)
 			if err != nil {
 				slog.Error(fmt.Sprintf("Error saving department ID: %v", err))
+				continue
 			}
+			slog.Debug("Saved department", "ID", newDepartmentInfo.NodeID)
+
 		case <-cancel:
 			slog.Info("Exiting scheduler")
 			return

internal/woolworths/woolworths_workers_test.go

@@ -49,7 +49,7 @@ func TestNewDepartmentIDWorker(t *testing.T) {
 
 	// Pre-load one existing department ID to check we're only being notified of
 	// new ones
-	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Vegetables"}
+	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Vegetables", Updated: time.Now()}
 	w.saveDepartment(dept)
 
 	departmentIDChannel := make(chan departmentInfo)
@@ -72,7 +72,7 @@ func TestNewProductWorker(t *testing.T) {
 	w := getInitialisedWoolworths()
 
 	// Set up a department to scan products from
-	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Vegetables"}
+	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Vegetables", Updated: time.Now()}
 	w.saveDepartment(dept)
 
 	productIDChannel := make(chan woolworthsProductInfo)
@@ -98,7 +98,7 @@ func TestProductListPageWorker(t *testing.T) {
 	w := getInitialisedWoolworths()
 
 	// Set up a department to scan products from
-	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Vegetables"}
+	dept := departmentInfo{NodeID: "1-E5BEE36E", Description: "Fruit & Vegetables", Updated: time.Now()}
 	w.saveDepartment(dept)
 
 	departmentPageChannel := make(chan departmentPage)
@@ -143,6 +143,12 @@ func TestDepartmentPageUpdateQueueWorker(t *testing.T) {
 			break
 		}
 	}
+	// Now test there are no more products waiting.
+	select {
+	case dept := <-departmentPageChannel:
+		t.Fatal("Expected no more products, got", dept)
+	case <-time.After(1 * time.Second):
+	}
 	// if want, got := 3, pageIndex; want != got {
 	// 	t.Errorf("Expected %d, got %d", want, got)
 	// }