Merge branch 'main' into text_dev

.all-contributorsrc

@@ -31,7 +31,7 @@
         "research",
         "ideas",
         "talk",
-        "doc",
+        "doc"
       ]
     },
     {
@@ -107,12 +107,22 @@
         "mentoring",
         "review"
       ]
+    },
+    {
+      "login": "mradamcox",
+      "name": "Adam Cox",
+      "avatar_url": "https://avatars.githubusercontent.com/u/10427268?v=4",
+      "profile": "http://mradamcox.github.io",
+      "contributions": [
+        "code",
+        "test"
+      ]
     }
   ],
   "contributorsPerLine": 7,
   "skipCi": true,
   "repoType": "github",
   "repoHost": "https://github.com",
   "projectName": "MapReader",
-  "projectOwner": "Living-with-machines"
+  "projectOwner": "maps-as-data"
 }

.github/workflows/mr_ci.yml

@@ -1,7 +1,10 @@
 ---
 name: Units Tests
 
-on: [push]
+on:
+  pull_request:
+    branches:
+      - main
 
 jobs:
 

.github/workflows/mr_ci_text_spotting.yml

@@ -1,7 +1,10 @@
 ---
 name: Units Tests - Text Spotting
 
-on: [push]
+on:
+  pull_request:
+    branches:
+      - main
 
 # Run linter with github actions for quick feedbacks.
 jobs:

.github/workflows/publish-to-conda-forge.yml

@@ -4,6 +4,8 @@ name: Publish to Conda Forge
 on:
   workflow_dispatch:
   push:
+    branches:
+      - main
     tags:
       - v*
 

.github/workflows/publish-to-pypi.yml

@@ -4,6 +4,8 @@ name: Publish Tagged Python 🐍 distributions 📦 to PyPI
 on:
   workflow_dispatch:
   push:
+    branches:
+      - main
     tags:
       - v*
 

CHANGELOG.md

@@ -21,6 +21,7 @@ _Add new changes here_
 
 - Added ability to save and reload text predictions ([#536](https://github.com/maps-as-data/MapReader/pull/536)
 - Added minimal dataclasses for text predictions ([#536](https://github.com/maps-as-data/MapReader/pull/536)
+- Added `skip_blank_patches` argument to `MapImages.patchify_all()` ([#540](https://github.com/maps-as-data/MapReader/pull/540))
 
 ## [v1.6.1](https://github.com/Living-with-machines/MapReader/releases/tag/v1.6.1) (2024-11-18)
 

README.md

@@ -115,16 +115,17 @@ Maps above reproduced with the permission of the National Library of Scotland ht
 <table>
   <tbody>
     <tr>
-      <td align="center" valign="top" width="14.28%"><a href="https://www.turing.ac.uk/people/researchers/katherine-mcdonough"><img src="https://avatars.githubusercontent.com/u/20363927?v=4?s=100" width="100px;" alt="Katie McDonough"/><br /><sub><b>Katie McDonough</b></sub></a><br /><a href="#research-kmcdono2" title="Research">🔬</a> <a href="#ideas-kmcdono2" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/Living-with-machines/MapReader/commits?author=kmcdono2" title="Documentation">📖</a> <a href="#eventOrganizing-kmcdono2" title="Event Organizing">📋</a> <a href="#projectManagement-kmcdono2" title="Project Management">📆</a> <a href="https://github.com/Living-with-machines/MapReader/pulls?q=is%3Apr+reviewed-by%3Akmcdono2" title="Reviewed Pull Requests">👀</a> <a href="#talk-kmcdono2" title="Talks">📢</a> <a href="#tutorial-kmcdono2" title="Tutorials">✅</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="http://danielwilson.info"><img src="https://avatars.githubusercontent.com/u/34318222?v=4?s=100" width="100px;" alt="Daniel C.S. Wilson"/><br /><sub><b>Daniel C.S. Wilson</b></sub></a><br /><a href="#research-dcsw2" title="Research">🔬</a> <a href="#ideas-dcsw2" title="Ideas, Planning, & Feedback">🤔</a> <a href="#talk-dcsw2" title="Talks">📢</a> <a href="https://github.com/Living-with-machines/MapReader/commits?author=dcsw2" title="Documentation">📖</a> <a href="#eventOrganizing-dcsw2" title="Event Organizing">📋</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kasra-hosseini"><img src="https://avatars.githubusercontent.com/u/1899856?v=4?s=100" width="100px;" alt="Kasra Hosseini"/><br /><sub><b>Kasra Hosseini</b></sub></a><br /><a href="https://github.com/Living-with-machines/MapReader/commits?author=kasra-hosseini" title="Code">💻</a> <a href="#ideas-kasra-hosseini" title="Ideas, Planning, & Feedback">🤔</a> <a href="#research-kasra-hosseini" title="Research">🔬</a> <a href="https://github.com/Living-with-machines/MapReader/pulls?q=is%3Apr+reviewed-by%3Akasra-hosseini" title="Reviewed Pull Requests">👀</a> <a href="#talk-kasra-hosseini" title="Talks">📢</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rwood-97"><img src="https://avatars.githubusercontent.com/u/72076688?v=4?s=100" width="100px;" alt="Rosie Wood"/><br /><sub><b>Rosie Wood</b></sub></a><br /><a href="https://github.com/Living-with-machines/MapReader/commits?author=rwood-97" title="Code">💻</a> <a href="https://github.com/Living-with-machines/MapReader/commits?author=rwood-97" title="Documentation">📖</a> <a href="#ideas-rwood-97" title="Ideas, Planning, & Feedback">🤔</a> <a href="#talk-rwood-97" title="Talks">📢</a> <a href="#tutorial-rwood-97" title="Tutorials">✅</a> <a href="https://github.com/Living-with-machines/MapReader/pulls?q=is%3Apr+reviewed-by%3Arwood-97" title="Reviewed Pull Requests">👀</a> <a href="#maintenance-rwood-97" title="Maintenance">🚧</a> <a href="#research-rwood-97" title="Research">🔬</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="http://www.westerling.nu"><img src="https://avatars.githubusercontent.com/u/7298727?v=4?s=100" width="100px;" alt="Kalle Westerling"/><br /><sub><b>Kalle Westerling</b></sub></a><br /><a href="https://github.com/Living-with-machines/MapReader/commits?author=kallewesterling" title="Code">💻</a> <a href="https://github.com/Living-with-machines/MapReader/commits?author=kallewesterling" title="Documentation">📖</a> <a href="#maintenance-kallewesterling" title="Maintenance">🚧</a> <a href="https://github.com/Living-with-machines/MapReader/pulls?q=is%3Apr+reviewed-by%3Akallewesterling" title="Reviewed Pull Requests">👀</a> <a href="#talk-kallewesterling" title="Talks">📢</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://www.turing.ac.uk/people/researchers/katherine-mcdonough"><img src="https://avatars.githubusercontent.com/u/20363927?v=4?s=100" width="100px;" alt="Katie McDonough"/><br /><sub><b>Katie McDonough</b></sub></a><br /><a href="#research-kmcdono2" title="Research">🔬</a> <a href="#ideas-kmcdono2" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/maps-as-data/MapReader/commits?author=kmcdono2" title="Documentation">📖</a> <a href="#projectManagement-kmcdono2" title="Project Management">📆</a> <a href="https://github.com/maps-as-data/MapReader/pulls?q=is%3Apr+reviewed-by%3Akmcdono2" title="Reviewed Pull Requests">👀</a> <a href="#talk-kmcdono2" title="Talks">📢</a> <a href="#tutorial-kmcdono2" title="Tutorials">✅</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://danielwilson.info"><img src="https://avatars.githubusercontent.com/u/34318222?v=4?s=100" width="100px;" alt="Daniel C.S. Wilson"/><br /><sub><b>Daniel C.S. Wilson</b></sub></a><br /><a href="#research-dcsw2" title="Research">🔬</a> <a href="#ideas-dcsw2" title="Ideas, Planning, & Feedback">🤔</a> <a href="#talk-dcsw2" title="Talks">📢</a> <a href="https://github.com/maps-as-data/MapReader/commits?author=dcsw2" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kasra-hosseini"><img src="https://avatars.githubusercontent.com/u/1899856?v=4?s=100" width="100px;" alt="Kasra Hosseini"/><br /><sub><b>Kasra Hosseini</b></sub></a><br /><a href="https://github.com/maps-as-data/MapReader/commits?author=kasra-hosseini" title="Code">💻</a> <a href="#ideas-kasra-hosseini" title="Ideas, Planning, & Feedback">🤔</a> <a href="#research-kasra-hosseini" title="Research">🔬</a> <a href="https://github.com/maps-as-data/MapReader/pulls?q=is%3Apr+reviewed-by%3Akasra-hosseini" title="Reviewed Pull Requests">👀</a> <a href="#talk-kasra-hosseini" title="Talks">📢</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rwood-97"><img src="https://avatars.githubusercontent.com/u/72076688?v=4?s=100" width="100px;" alt="Rosie Wood"/><br /><sub><b>Rosie Wood</b></sub></a><br /><a href="https://github.com/maps-as-data/MapReader/commits?author=rwood-97" title="Code">💻</a> <a href="https://github.com/maps-as-data/MapReader/commits?author=rwood-97" title="Documentation">📖</a> <a href="#ideas-rwood-97" title="Ideas, Planning, & Feedback">🤔</a> <a href="#talk-rwood-97" title="Talks">📢</a> <a href="#tutorial-rwood-97" title="Tutorials">✅</a> <a href="https://github.com/maps-as-data/MapReader/pulls?q=is%3Apr+reviewed-by%3Arwood-97" title="Reviewed Pull Requests">👀</a> <a href="#maintenance-rwood-97" title="Maintenance">🚧</a> <a href="#research-rwood-97" title="Research">🔬</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://www.westerling.nu"><img src="https://avatars.githubusercontent.com/u/7298727?v=4?s=100" width="100px;" alt="Kalle Westerling"/><br /><sub><b>Kalle Westerling</b></sub></a><br /><a href="https://github.com/maps-as-data/MapReader/commits?author=kallewesterling" title="Code">💻</a> <a href="https://github.com/maps-as-data/MapReader/commits?author=kallewesterling" title="Documentation">📖</a> <a href="#maintenance-kallewesterling" title="Maintenance">🚧</a> <a href="https://github.com/maps-as-data/MapReader/pulls?q=is%3Apr+reviewed-by%3Akallewesterling" title="Reviewed Pull Requests">👀</a> <a href="#talk-kallewesterling" title="Talks">📢</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://maps.nls.uk"><img src="https://avatars.githubusercontent.com/u/3666702?v=4?s=100" width="100px;" alt="Chris Fleet"/><br /><sub><b>Chris Fleet</b></sub></a><br /><a href="#data-ChrisFleet" title="Data">🔣</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kasparvonbeelen"><img src="https://avatars.githubusercontent.com/u/11618160?v=4?s=100" width="100px;" alt="Kaspar Beelen"/><br /><sub><b>Kaspar Beelen</b></sub></a><br /><a href="#ideas-kasparvonbeelen" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/Living-with-machines/MapReader/pulls?q=is%3Apr+reviewed-by%3Akasparvonbeelen" title="Reviewed Pull Requests">👀</a> <a href="#research-kasparvonbeelen" title="Research">🔬</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kasparvonbeelen"><img src="https://avatars.githubusercontent.com/u/11618160?v=4?s=100" width="100px;" alt="Kaspar Beelen"/><br /><sub><b>Kaspar Beelen</b></sub></a><br /><a href="#ideas-kasparvonbeelen" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/maps-as-data/MapReader/pulls?q=is%3Apr+reviewed-by%3Akasparvonbeelen" title="Reviewed Pull Requests">👀</a> <a href="#research-kasparvonbeelen" title="Research">🔬</a></td>
     </tr>
     <tr>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andrewphilipsmith"><img src="https://avatars.githubusercontent.com/u/5346065?v=4?s=100" width="100px;" alt="Andy Smith"/><br /><sub><b>Andy Smith</b></sub></a><br /><a href="https://github.com/Living-with-machines/MapReader/commits?author=andrewphilipsmith" title="Code">💻</a> <a href="https://github.com/Living-with-machines/MapReader/commits?author=andrewphilipsmith" title="Documentation">📖</a> <a href="#mentoring-andrewphilipsmith" title="Mentoring">🧑‍🏫</a> <a href="https://github.com/Living-with-machines/MapReader/pulls?q=is%3Apr+reviewed-by%3Aandrewphilipsmith" title="Reviewed Pull Requests">👀</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/andrewphilipsmith"><img src="https://avatars.githubusercontent.com/u/5346065?v=4?s=100" width="100px;" alt="Andy Smith"/><br /><sub><b>Andy Smith</b></sub></a><br /><a href="https://github.com/maps-as-data/MapReader/commits?author=andrewphilipsmith" title="Code">💻</a> <a href="https://github.com/maps-as-data/MapReader/commits?author=andrewphilipsmith" title="Documentation">📖</a> <a href="#mentoring-andrewphilipsmith" title="Mentoring">🧑‍🏫</a> <a href="https://github.com/maps-as-data/MapReader/pulls?q=is%3Apr+reviewed-by%3Aandrewphilipsmith" title="Reviewed Pull Requests">👀</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://mradamcox.github.io"><img src="https://avatars.githubusercontent.com/u/10427268?v=4?s=100" width="100px;" alt="Adam Cox"/><br /><sub><b>Adam Cox</b></sub></a><br /><a href="https://github.com/maps-as-data/MapReader/commits?author=mradamcox" title="Code">💻</a> <a href="https://github.com/maps-as-data/MapReader/commits?author=mradamcox" title="Tests">⚠️</a></td>
     </tr>
   </tbody>
 </table>

docs/source/using-mapreader/step-by-step-guide/2-load.rst

@@ -211,6 +211,7 @@ This will create 1024 x 1024 pixel patches with 10% overlap between each patch.
     - ``square_cuts`` - This is a deprecated method and no longer recommended for use. By default, this is set to ``False`` and padding is added to patches at the edges of the parent image to ensure square patches. If you set ``square_cuts=True``, instead of padding, there will be some overlap between edge patches.
     - ``add_to_parent`` - By default, this is set to ``True`` so that each time you run ``patchify_all`` your patches are added to your ``MapImages`` object. Setting it to ``False`` (by specifying ``add_to_parent=False``) will mean your patches are created, but not added to your ``MapImages`` object. This can be useful for testing out different patch sizes.
     - ``rewrite`` - By default, this is set to ``False`` so that if your patches already exist they are not overwritten. Setting it to ``True`` (by specifying ``rewrite=True``) will mean already existing patches are recreated and overwritten.
+    - ``skip_blank_patches`` - By default, this is set to ``False``. Setting to ``True`` will omit any patches that only contain ``0`` values, which can speed up processing on irregularly shaped map images that have empty regions. The `Image.getbbox() <https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.Image.getbbox>`_ method is used to determine if a patch is blank.
 
 If you would like to save your patches as geo-referenced tiffs (i.e. geotiffs), use:
 

mapreader/load/images.py

@@ -1208,6 +1208,7 @@ def patchify_all(
         patch_size: int | None = 100,
         tree_level: str | None = "parent",
         path_save: str | None = None,
+        skip_blank_patches: bool = False,
         add_to_parents: bool | None = True,
         square_cuts: bool | None = False,
         resize_factor: bool | None = False,
@@ -1234,6 +1235,8 @@ def patchify_all(
             Directory to save the patches.
             If None, will be set as f"patches_{patch_size}_{method}" (e.g. "patches_100_pixel").
             By default None.
+        skip_blank_patches : bool
+            If True, any patch that only contains 0 values will be skipped, by default ``False``. Uses PIL.Image().get_bbox().
         add_to_parents : bool, optional
             If True, patches will be added to the MapImages instance's
             ``images`` dictionary, by default ``True``.
@@ -1313,6 +1316,7 @@ def patchify_all(
                     add_to_parents=add_to_parents,
                     resize_factor=resize_factor,
                     output_format=output_format,
+                    skip_blank_patches=skip_blank_patches,
                     rewrite=rewrite,
                     verbose=verbose,
                     overlap=overlap,
@@ -1326,6 +1330,7 @@ def _patchify_by_pixel(
         add_to_parents: bool | None = True,
         resize_factor: bool | None = False,
         output_format: str | None = "png",
+        skip_blank_patches: bool = False,
         rewrite: bool | None = False,
         verbose: bool | None = False,
         overlap: int | None = 0,
@@ -1347,6 +1352,8 @@ def _patchify_by_pixel(
             If True, resize the images before patchifying, by default ``False``.
         output_format : str, optional
             Format to use when writing image files, by default ``"png"``.
+        skip_blank_patches : bool
+            If True, any patch that only contains 0 values will be skipped, by default ``False``. Uses PIL.Image().get_bbox().
         rewrite : bool, optional
             If True, existing patches will be rewritten, by default ``False``.
         verbose : bool, optional
@@ -1370,6 +1377,7 @@ def _patchify_by_pixel(
             )
 
         height, width = img.height, img.width
+        overlap_pixels = int(patch_size * overlap)
 
         x = 0
         while x < width:
@@ -1389,6 +1397,15 @@ def _patchify_by_pixel(
 
                 else:
                     patch = img.crop((x, y, max_x, max_y))
+
+                    # skip if blank and don't add to parents
+                    if skip_blank_patches and patch.getbbox() is None:
+                        self._print_if_verbose(
+                            f"[INFO] Skipping empty patch: {patch_id}.", verbose
+                        )
+                        y = y + patch_size - overlap_pixels
+                        continue
+
                     if max_x == width:
                         patch = ImageOps.pad(
                             patch, (patch_size, patch.height), centering=(0, 0)
@@ -1416,7 +1433,6 @@ def _patchify_by_pixel(
                     self._add_patch_coords_id(patch_id)
                     self._add_patch_polygons_id(patch_id)
 
-                overlap_pixels = int(patch_size * overlap)
                 y = y + patch_size - overlap_pixels
             x = x + patch_size - overlap_pixels
 

tests/test_load/test_images.py

@@ -515,6 +515,26 @@ def test_patchify_pixels(sample_dir, image_id, tmp_path):
     assert os.path.isfile(f"{tmp_path}/patch-0-0-3-3-#{image_id}#.png")
 
 
+def test_patchify_pixels_skip_blank_rgb(sample_dir, tmp_path):
+    maps = MapImages(f"{sample_dir}/cropped_blank_corners_rgb.tif")
+    maps.patchify_all(patch_size=3, path_save=tmp_path, skip_blank_patches=True)
+    parent_list = maps.list_parents()
+    patch_list = maps.list_patches()
+    assert len(parent_list) == 1
+    assert len(patch_list) == 5
+    assert os.path.isfile(f"{tmp_path}/patch-0-3-3-6-#cropped_blank_corners_rgb.tif#.png")
+
+
+def test_patchify_pixels_skip_blank_rgba(sample_dir, tmp_path):
+    maps = MapImages(f"{sample_dir}/cropped_blank_corners_rgba.tif")
+    maps.patchify_all(patch_size=3, path_save=tmp_path, skip_blank_patches=True)
+    parent_list = maps.list_parents()
+    patch_list = maps.list_patches()
+    assert len(parent_list) == 1
+    assert len(patch_list) == 5
+    assert os.path.isfile(f"{tmp_path}/patch-0-3-3-6-#cropped_blank_corners_rgba.tif#.png")
+
+
 def test_patchify_pixels_square(sample_dir, image_id, tmp_path):
     maps = MapImages(f"{sample_dir}/{image_id}")
     maps.patchify_all(patch_size=5, path_save=f"{tmp_path}_square", square_cuts=True)