Added debug logging to sitemap_filter (#45)

DataONEorg · Nov 10, 2023 · a2235cb · a2235cb
1 parent 3e16aac
commit a2235cb
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/soscan/spiders/jsonldspider.py b/soscan/spiders/jsonldspider.py
@@ -135,6 +135,7 @@ def sitemap_filter(self, entries):
 
                 if self.lastmod_filter is not None and ts is not None:
                     if ts > self.lastmod_filter:
+                        self.logger.debug(f'Yielding record {i}: {entry}')
                         yield entry
                     else:
                         self.logger.debug(f'lastmod_filter skipping record {i}: {entry}')
@@ -143,6 +144,7 @@ def sitemap_filter(self, entries):
             if (self.start_point is not None) and (self.start_point < i):
                 if i == 1:
                     self.logger.info(f'Skipping to start_point at record {self.start_point}')
+                self.logger.debug(f'start_point skipping record {i}: {entry}')
 
     def parse(self, response, **kwargs):
         """