More statistics and conversion notes

As discussed during the challenge meeting, a few additional
fields would be useful for evaluating. These are now stored
(somewhat redundantly) in each stats block. Additionally, a
new argument `--conversion-notes` adds text to the _creator
field.

This does not yet record memory usage. `psutil` will need adding
for cross-platform checks.

see: ome#31 (comment)

src/ome2024_ngff_challenge/resave.py

@@ -5,6 +5,8 @@
 import json
 import logging
 import math
+import multiprocessing
+import os
 import random
 import time
 import warnings
@@ -136,6 +138,9 @@ def convert_array(
         "read": after.read(),
         "written": after.written(),
         "elapsed": after.elapsed(),
+        "threads": threads,
+        "cpu_count": multiprocessing.cpu_count(),
+        "sched_affinity": os.sched_getaffinity(0),
     }
     LOGGER.info(f"""Re-encode (tensorstore) {input_config} to {output_config}
         read: {stats["read"]}
@@ -181,6 +186,7 @@ def convert_image(
     output_write_details: bool,
     output_script: bool,
     threads: int,
+    notes: str | None,
 ):
     dimension_names = None
     # top-level version...
@@ -194,7 +200,7 @@ def convert_image(
         ome_attrs[key] = value
 
     # Add _creator - NB: this will overwrite existing _creator info
-    add_creator(ome_attrs)
+    add_creator(ome_attrs, notes)
 
     if output_config.zr_group is not None:  # otherwise dry-run
         # dev2: everything is under 'ome' key
@@ -307,6 +313,7 @@ def convert_image(
                 output_write_details,
                 output_script,
                 threads,
+                notes,
             )
 
 
@@ -430,6 +437,7 @@ def main(ns: argparse.Namespace) -> int:
             ns.output_write_details,
             ns.output_script,
             ns.output_threads,
+            ns.conversion_notes,
         )
         converted += 1
 
@@ -441,7 +449,7 @@ def main(ns: argparse.Namespace) -> int:
             strip_version(value)
             ome_attrs[key] = value
 
-        add_creator(ome_attrs)
+        add_creator(ome_attrs, ns.conversion_notes)
 
         if output_config.zr_group is not None:  # otherwise dry run
             # dev2: everything is under 'ome' key
@@ -486,6 +494,7 @@ def main(ns: argparse.Namespace) -> int:
                     ns.output_write_details,
                     ns.output_script,
                     ns.output_threads,
+                    ns.conversion_notes,
                 )
                 converted += 1
     # Note: plates can *also* contain this metadata
@@ -497,7 +506,7 @@ def main(ns: argparse.Namespace) -> int:
             strip_version(value)
             ome_attrs[key] = value
 
-        add_creator(ome_attrs)
+        add_creator(ome_attrs, ns.conversion_notes)
 
         if output_config.zr_group is not None:  # otherwise dry run
             # dev2: everything is under 'ome' key
@@ -531,6 +540,7 @@ def main(ns: argparse.Namespace) -> int:
                 ns.output_write_details,
                 ns.output_script,
                 ns.output_threads,
+                ns.conversion_notes,
             )
             converted += 1
     else:
@@ -630,6 +640,7 @@ def cli(subparsers: argparse._SubParsersAction):
     Set number of parallel threads           {cmd} --cc-by in.zarr out.zarr --output-threads=128
     Increase logging                         {cmd} --cc-by in.zarr out.zarr --log=debug
     Increase logging even more               {cmd} --cc-by in.zarr out.zarr --log=trace
+    Record details about the conversion      {cmd} --cc-by in.zarr out.zarr --conversion-notes="run on a virtual machine"
     """
     parser = subparsers.add_parser(
         "resave",
@@ -737,6 +748,10 @@ def __call__(self, parser, args, *unused, **ignore):  # noqa: ARG002
         help="comma separated list of shards sizes for all subresolutions",
         type=csv_int,
     )
+    parser.add_argument(
+        "--conversion-notes",
+        help="free-text notes on this conversion (e.g., 'run on AWS EC2 instance in docker')",
+    )
     parser.add_argument("input_path", type=Path)
     parser.add_argument("output_path", type=Path)
 

src/ome2024_ngff_challenge/utils.py

@@ -122,10 +122,16 @@ def strip_version(possible_dict) -> None:
         del possible_dict["version"]
 
 
-def add_creator(json_dict) -> None:
+def add_creator(json_dict: dict, notes: str | None = None) -> None:
     # Add _creator - NB: this will overwrite any existing _creator info
     pkg_version = lib_version("ome2024-ngff-challenge")
-    json_dict["_creator"] = {"name": "ome2024-ngff-challenge", "version": pkg_version}
+    json_dict["_creator"] = {
+        "name": "ome2024-ngff-challenge",
+        "version": pkg_version,
+        "notes": notes,
+    }
+    if notes:
+        json_dict["_creator"]["notes"] = notes
 
 
 class TextBuffer(Buffer):