Ignore resource WARC records for now

src/warc2zim/converter.py

@@ -485,7 +485,7 @@ def add_items_for_warc_record(self, record):
                 logger.debug(f"Skipping url {url}, outside included domains")
                 return
 
-        if record.rec_type != "revisit":
+        if record.rec_type == "response":
             if self.is_self_redirect(record, url):
                 logger.debug("Skipping self-redirect: " + url)
                 return
@@ -510,7 +510,8 @@ def add_items_for_warc_record(self, record):
             self.indexed_urls.add(normalized_url)
 
         elif (
-            record.rec_headers["WARC-Refers-To-Target-URI"] != url
+            record.rec_type == "revisit"
+            and record.rec_headers["WARC-Refers-To-Target-URI"] != url
             and normalized_url not in self.revisits
         ):  # pragma: no branch
             self.revisits[normalized_url] = normalize(

tests/test_warc_to_zim.py

@@ -159,7 +159,7 @@ def verify_warc_and_zim(self, warcfile, zimfile, verify_scraper_suffix):
                 # We should check with the content of the targeted record...
                 # But difficult to test as we don't have it
                 assert payload
-            else:
+            elif record.rec_type == "response":
                 # We must have a payload
                 assert payload
                 payload_content = payload.content.tobytes()
@@ -168,6 +168,9 @@ def verify_warc_and_zim(self, warcfile, zimfile, verify_scraper_suffix):
                 # have exact match
                 if payload.mimetype.startswith("text/html"):
                     assert head_insert in payload_content
+            elif record.rec_type == "resource":
+                # we do not want to embed resources "as-is"
+                assert not payload
 
             warc_urls.add(url)