Merge pull request #277 from openzim/scraper_suffix

Pass scraper suffix to warc2zim

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - New `--version` flag to display Zimit version
 - New `--logging` flag to adjust Browsertrix Crawler logging (#273)
+- Use new `--scraper-suffix` flag of warc2zim to enhance ZIM "Scraper" metadata (#275)
 
 ### Changed
 

src/zimit/__about__.py

@@ -1 +1 @@
-__version__ = "2.0.0-dev0"
+__version__ = "2.0.0-dev1"

src/zimit/zimit.py

@@ -360,6 +360,25 @@ def run(raw_args):
 
  zimit_args, warc2zim_args = parser.parse_known_args(raw_args)
 
+ logger.info("Checking browsertrix-crawler version")
+ crawl_version_cmd = ["crawl", "--version"]
+ try:
+ crawl = subprocess.run(
+ crawl_version_cmd, check=True, capture_output=True, text=True
+ )
+ except Exception:
+ logger.error("Failed to get Browsertrix crawler version")
+ raise
+ crawler_version = crawl.stdout
+ logger.info(f"Browsertrix crawler: version {crawler_version}")
+
+ # pass a scraper suffix to warc2zim so that both zimit, warc2zim and crawler
+ # versions are associated with the ZIM
+ warc2zim_args.append("--scraper-suffix")
+ warc2zim_args.append(
+ f" + zimit {__version__} + Browsertrix crawler {crawler_version}"
+ )
+
  # pass url and output to warc2zim also
  if zimit_args.output:
  warc2zim_args.append("--output")
@@ -522,9 +541,11 @@ def check_url(url: str, user_agent: str, scope: str | None = None):
  "your homepage might be out-of-scope. Please check!".format(
  parsed_url.geturl(),
  actual_url.geturl(),
- "is"
- if get_fld(parsed_url.geturl()) == get_fld(actual_url.geturl())
- else "is not",
+ (
+ "is"
+ if get_fld(parsed_url.geturl()) == get_fld(actual_url.geturl())
+ else "is not"
+ ),
  scope,
  )
  )

tests-integration/integration.py

@@ -2,13 +2,8 @@
 import json
 import os
 
-import libzim.reader
 from warcio import ArchiveIterator
-
-
-def get_zim_main_entry(zimfile):
- zim_fh = libzim.reader.Archive(zimfile)
- return zim_fh.main_entry
+from zimscraperlib.zim import Archive
 
 
 def test_is_file():
@@ -20,11 +15,22 @@ def test_zim_main_page():
  """Main page specified, http://isago.rskg.org/, was a redirect to https
  Ensure main page is the redirected page"""
 
- main_entry = get_zim_main_entry("/output/isago.zim")
+ main_entry = Archive("/output/isago.zim").main_entry
  assert main_entry.is_redirect
  assert main_entry.get_redirect_entry().path == "isago.rskg.org/"
 
 
+def test_zim_scraper():
+ """Main page specified, http://isago.rskg.org/, was a redirect to https
+ Ensure main page is the redirected page"""
+
+ zim_fh = Archive("/output/isago.zim")
+ scraper = zim_fh.get_text_metadata("Scraper")
+ assert "zimit " in scraper
+ assert "warc2zim " in scraper
+ assert "Browsertrix crawler " in scraper
+
+
 def test_user_agent():
  """Test that mobile user agent was used