tests: Add xfail_windows decorator for failing tests on Windows (#4362

)

* grass.gunittest: Add xfail_windows decorator for Windows expected failures

* lib/imagery: Add xfail_windows decorator to test functions

* Fix spacing issue in xfail_windows warning message

* Add stacklevel=2 to xfail_windows decorator warning message

* Update utils.py

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Format end of file

* Comment out warning in decorator

* i.maxlik: Add xfail_windows decorator to SuccessTest class

* lib/gis: Add xfail_windows decorator to TestNewlinesWithGetlFunctions class

* Update warning stacklevel to 3

* Update utils.py

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update utils.py

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update utils.py

* tests: Add expected failures to failing tests on windows

* tests: Apply xfail_windows decorator to setUpClass in lib/gis getl tests

* tests: Apply xfail_windows decorator to parser json tests in lib/gis

* tests: Apply xfail_windows decorator to failing test in lib/imagery

* Remove wrong import of xfail_windows

* tests: Temporarily remove all gunittest exclusions

* tests: Apply xfail_windows decorator to failing test in test_gcmd.py

* tests: Apply xfail_windows decorator to failing test in lib/init in test_grass_tmp_mapset.py

* CI(OSGeo4W): Use an OSGeo4W-specific gunittest.cfg config file

* Revert "tests: Temporarily remove all gunittest exclusions"

This reverts commit 20eb947.

* v.out.lidar: Apply xfail_windows decorator to one failing test

* t.rast.series: Apply xfail_windows decorator to failing tests

* t.connect: Apply xfail_windows decorator to three failing tests

* g.search.modules: Apply xfail_windows decorator to a failing test

* grass.temporal: Apply xfail_windows decorator to failing tests in unittests_temporal_raster_algebra_equal_ts

* grass.gunittest: Apply xfail_windows decorator to failing test test_assertVectorEqualsAscii_by_import

* tests: Re-enable excluded tests that are not failing on Windows

* tests: Re-enable passing tests from .gunittest.cfg exclusion list

* Update test_gis_lib_getl.py

* Update test_raster_img.py

* tests: Try re-enabling python/grass/script/testsuite/test_script_doctests.py

* i.maxlik: Add xfail_windows decorator to all tests in class

* tests: Exclude more tests that are failing on Windows

* v.univar: Apply xfail_windows decorator to failing tests

* t.rast.univar: Apply xfail_windows decorator to a failing test

* r.in.gdal: Try avoiding error in calling gdal-config in skipif decorator on Windows

* CI(OSGeo4W): Increase min-success to 96% for gunittest

* Add line in test_grass_tmp_mapset.py

* Update test_r_tileset.py

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/osgeo4w_gunittest.cfg

@@ -0,0 +1,32 @@
+[gunittest]
+
+# Files (or wildcard patterns) to exclude from testing separated by newline or
+# space. This would be ideally empty or it would include just special cases,
+# but it includes mainly tests which can (and should) be fixed.
+exclude =
+    gui/wxpython/core/testsuite/toolboxes.sh
+    lib/init/testsuite/test_grass_tmp_mapset.py
+    python/grass/gunittest/testsuite/test_gunitest_doctests.py
+    python/grass/pygrass/raster/testsuite/test_pygrass_raster_doctests.py
+    python/grass/pygrass/rpc/testsuite/test_pygrass_rpc_doctests.py
+    python/grass/script/testsuite/test_script_doctests.py
+    python/grass/temporal/testsuite/unittests_temporal_raster_conditionals_complement_else.py
+    temporal/t.info/testsuite/test.t.info.sh
+    temporal/t.rast.accdetect/testsuite/test.t.rast.accdetect.reverse.sh
+    temporal/t.rast.accdetect/testsuite/test.t.rast.accdetect.sh
+    temporal/t.rast.aggregate/testsuite/test_aggregation_relative.py
+    vector/v.edit/testsuite/select_all_flag.sh
+    vector/v.in.lidar/testsuite/decimation_test.py
+    vector/v.in.lidar/testsuite/mask_test.py
+    vector/v.in.lidar/testsuite/test_v_in_lidar_basic.py
+    vector/v.in.lidar/testsuite/test_v_in_lidar_filter.py
+    vector/v.what.rast3/testsuite/test.v.what.rast3.sh
+    vector/v.what/testsuite/test_vwhat_layers.py
+    vector/v.what/testsuite/test_vwhat_ncspm.py
+
+# Maximum time for execution of one test file (not a test function)
+# after which test is terminated (which may not terminate child processes
+# from that test).
+# Using 5 minutes as maximum (average test time should be much shorter,
+# couple seconds per file, median should be around one second).
+timeout = 300

.github/workflows/test_thorough.bat

@@ -2,4 +2,4 @@ set grass=%1
 set python=%2
 
 call %grass% --tmp-project XY --exec g.download.project url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=%USERPROFILE%
-call %grass% --tmp-project XY --exec %python% -m grass.gunittest.main --grassdata %USERPROFILE% --location nc_spm_full_v2alpha2 --location-type nc --min-success 86
+call %grass% --tmp-project XY --exec %python% -m grass.gunittest.main --grassdata %USERPROFILE% --location nc_spm_full_v2alpha2 --location-type nc --min-success 96 --config .github\workflows\osgeo4w_gunittest.cfg

gui/wxpython/core/testsuite/test_gcmd.py

@@ -1,5 +1,6 @@
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 
 
 class Rcv:
@@ -16,6 +17,8 @@ def recv(self):
 
 
 class Recv_SomeTest(TestCase):
+
+    @xfail_windows
     def test_decode(self):
         """
         Multibyte chars should not be split

imagery/i.maxlik/testsuite/test_i_maxlik.py

@@ -19,6 +19,7 @@
 
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 
 from grass.lib.gis import G_mapset_path
 from grass.lib.raster import Rast_write_semantic_label
@@ -172,6 +173,7 @@ def tearDownClass(cls):
         )
         cls.runModule("g.remove", flags="f", type="group", name=cls.group, quiet=True)
 
+    @xfail_windows
     def test_v1(self):
         """Test v1 signature"""
         self.assertModule(
@@ -193,6 +195,7 @@ def test_v1(self):
         self.assertEqual(res.get_cat(0)[1], 1)
         res.close()
 
+    @xfail_windows
     def test_v2(self):
         """Test v2 signature"""
         self.assertModule(

lib/gis/testsuite/test_parser_json.py

@@ -10,11 +10,14 @@
 
 import subprocess
 from grass.gunittest.case import TestCase
+from grass.gunittest.utils import xfail_windows
 from grass.script import decode
 import json
 
 
 class TestParserJson(TestCase):
+
+    @xfail_windows
     def test_r_slope_aspect_json(self):
         args = [
             "r.slope.aspect",
@@ -58,6 +61,7 @@ def test_r_slope_aspect_json(self):
         self.assertEqual(json_code["inputs"], inputs)
         self.assertEqual(json_code["outputs"], outputs)
 
+    @xfail_windows
     def test_v_out_ascii(self):
         args = [
             "v.out.ascii",
@@ -91,6 +95,7 @@ def test_v_out_ascii(self):
         self.assertEqual(json_code["inputs"], inputs)
         self.assertEqual(json_code["outputs"], outputs)
 
+    @xfail_windows
     def test_v_info(self):
         args = ["v.info", "map=hospitals@PERMANENT", "-c", "--json"]
 

lib/imagery/testsuite/test_imagery_signature_management.py

@@ -15,6 +15,7 @@
 
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 
 from grass.script.core import tempname
 import grass.script as gs
@@ -44,16 +45,19 @@
 
 
 class GetSignaturesDirTestCase(TestCase):
+    @xfail_windows
     def test_get_sig(self):
         cdir = ctypes.create_string_buffer(GNAME_MAX)
         I_get_signatures_dir(cdir, I_SIGFILE_TYPE_SIG)
         self.assertEqual(utils.decode(cdir.value), f"signatures{HOST_DIRSEP}sig")
 
+    @xfail_windows
     def test_get_sigset(self):
         cdir = ctypes.create_string_buffer(GNAME_MAX)
         I_get_signatures_dir(cdir, I_SIGFILE_TYPE_SIGSET)
         self.assertEqual(utils.decode(cdir.value), f"signatures{HOST_DIRSEP}sigset")
 
+    @xfail_windows
     def test_get_libsvm(self):
         elem = ctypes.create_string_buffer(GNAME_MAX)
         I_get_signatures_dir(elem, I_SIGFILE_TYPE_LIBSVM)

lib/init/testsuite/test_grass_tmp_mapset.py

@@ -18,6 +18,7 @@
 import os
 import shutil
 import subprocess
+from grass.gunittest.utils import xfail_windows
 
 
 # Note that unlike rest of GRASS GIS, here we are using unittest package
@@ -43,6 +44,7 @@ def tearDown(self):
         """Deletes the location"""
         shutil.rmtree(self.location, ignore_errors=True)
 
+    @xfail_windows
     def test_command_runs(self):
         """Check that correct parameters are accepted"""
         return_code = subprocess.call(
@@ -57,6 +59,7 @@ def test_command_runs(self):
             ),
         )
 
+    @xfail_windows
     def test_command_fails_without_location(self):
         """Check that the command fails with a nonexistent location"""
         return_code = subprocess.call(
@@ -78,6 +81,7 @@ def test_command_fails_without_location(self):
             ),
         )
 
+    @xfail_windows
     def test_mapset_metadata_correct(self):
         """Check that metadata is readable and have expected value (XY CRS)"""
         output = subprocess.check_output(
@@ -91,6 +95,7 @@ def test_mapset_metadata_correct(self):
             ),
         )
 
+    @xfail_windows
     def test_mapset_deleted(self):
         """Check that mapset is deleted at the end of execution"""
         subprocess.check_call(

python/grass/grassdb/testsuite/test_manage.py

@@ -18,6 +18,7 @@
 from grass.gunittest.case import TestCase
 from grass.gunittest.gmodules import call_module
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 
 
 class TestMapsetPath(TestCase):
@@ -38,6 +39,7 @@ def test_mapset_from_path_object(self):
         self.assertEqual(mapset_path.mapset, mapset_name)
         self.assertEqual(mapset_path.path, Path(path) / location_name / mapset_name)
 
+    @xfail_windows
     def test_mapset_from_str(self):
         """Check with path from str and database directory as Path"""
         path = "does/not/exist"
@@ -60,6 +62,7 @@ def test_mapset_from_str(self):
 class TestSplitMapsetPath(TestCase):
     """Check that split works with different parameters"""
 
+    @xfail_windows
     def test_split_path(self):
         """Check that pathlib.Path is correctly split"""
         ref_db = "does/not/exist"
@@ -71,6 +74,7 @@ def test_split_path(self):
         self.assertEqual(new_location, ref_location)
         self.assertEqual(new_mapset, ref_mapset)
 
+    @xfail_windows
     def test_split_str(self):
         """Check that path as str is correctly split"""
         ref_db = "does/not/exist"
@@ -82,6 +86,7 @@ def test_split_str(self):
         self.assertEqual(new_location, ref_location)
         self.assertEqual(new_mapset, ref_mapset)
 
+    @xfail_windows
     def test_split_str_trailing_slash(self):
         """Check that path as str with a trailing slash is correctly split"""
         ref_db = "does/not/exist"

python/grass/gunittest/testsuite/test_assertions.py

@@ -11,6 +11,7 @@
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
 from grass.gunittest.gmodules import SimpleModule
+from grass.gunittest.utils import xfail_windows
 
 
 class TestTextAssertions(TestCase):
@@ -34,6 +35,7 @@ def test_assertLooksLike(self):
     def test_assertLooksLike_multiline(self):
         self.assertLooksLike("a=123\nb=456\nc=789", "a=...\nb=...\nc=...")
 
+    @xfail_windows
     def test_assertLooksLike_multiline_platform_dependent(self):
         self.assertLooksLike(
             "a=123\nb=456\nc=789", "a=...{nl}b=...{nl}c=...".format(nl=os.linesep)
@@ -384,6 +386,7 @@ def test_assertFileExists_empty_file(self):
             self.failureException, self.assertFileExists, filename=self.emtpy_file
         )
 
+    @xfail_windows
     def test_assertFileMd5(self):
         self.assertFileMd5(filename=self.file_with_md5, md5=self.file_md5)
         self.assertRaises(

python/grass/gunittest/testsuite/test_assertions_vect.py

@@ -5,6 +5,7 @@
 from grass.exceptions import CalledModuleError
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 
 
 V_UNIVAR_SCHOOLS_WIDTH_SUBSET = """n=144
@@ -281,6 +282,7 @@ def test_assertVectorAsciiEqualsVectorAscii_diff_content(self):
         self.assertFileExists(self.simple_base_file)
         self.assertFileExists(self.simple_modified_file)
 
+    @xfail_windows
     def test_assertVectorEqualsAscii_by_import(self):
         amap = "simple_vector_map_imported_base"
         self.runModule(

python/grass/gunittest/testsuite/test_checkers.py

@@ -24,6 +24,7 @@
     file_md5,
     text_file_md5,
 )
+from grass.gunittest.utils import xfail_windows
 
 
 class TestValuesEqual(TestCase):
@@ -386,6 +387,7 @@ def tearDownClass(cls):
         try_remove(cls.correct_file_name_unix_nl)
         try_remove(cls.wrong_file_name)
 
+    @xfail_windows
     def test_text_file_binary(self):
         r"""File with ``\n`` (LF) newlines as binary (MD5 has ``\n``)."""
         self.assertEqual(

python/grass/gunittest/utils.py

@@ -14,6 +14,8 @@
 from pathlib import Path
 import shutil
 import sys
+from unittest import expectedFailure
+import warnings
 
 
 def ensure_dir(directory):
@@ -80,3 +82,17 @@ def safe_repr(obj, short=False):
     if not short or len(result) < _MAX_LENGTH:
         return result
     return result[:_MAX_LENGTH] + " [truncated]..."
+
+
+def xfail_windows(test_item):
+    """Marks a test as an expected failure or error only on Windows
+    Equivalent to applying @unittest.expectedFailure only when running
+    on Windows.
+    """
+    if not sys.platform.startswith("win"):
+        return lambda func: func
+    warnings.warn(
+        "Once the test is fixed and passing, remove the @xfail_windows decorator",
+        stacklevel=2,
+    )
+    return expectedFailure(test_item)

python/grass/jupyter/testsuite/map3d_test.py

@@ -26,6 +26,7 @@
 import grass.jupyter as gj
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 
 
 def can_import_ipython():
@@ -81,12 +82,14 @@ def tearDown(self):
             else:
                 file.unlink(missing_ok=True)
 
+    @xfail_windows
     def test_defaults(self):
         """Check that default settings work"""
         renderer = gj.Map3D()
         renderer.render(elevation_map="elevation", color_map="elevation")
         self.assertFileExists(renderer.filename)
 
+    @xfail_windows
     def test_filename(self):
         """Check that custom filename works"""
         custom_filename = "test_filename.png"
@@ -96,12 +99,14 @@ def test_filename(self):
         renderer.render(elevation_map="elevation", color_map="elevation")
         self.assertFileExists(custom_filename)
 
+    @xfail_windows
     def test_hw(self):
         """Check that custom width and height works"""
         renderer = gj.Map3D(width=200, height=400)
         renderer.render(elevation_map="elevation", color_map="elevation")
         self.assertFileExists(renderer.filename)
 
+    @xfail_windows
     def test_overlay(self):
         """Check that overlay works"""
         renderer = gj.Map3D()

python/grass/pygrass/raster/testsuite/test_category.py

@@ -6,6 +6,7 @@
 
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 
 from grass.pygrass.raster import RasterRow
 from grass.pygrass.raster.category import Category
@@ -76,6 +77,7 @@ def testFirstCat(self):
         self.assertEqual(cats[7], cat7)
         self.assertEqual(cats[15], cat15)
 
+    @xfail_windows
     def testWrite(self):
         tmpfile = tempfile(False)
         cats = Category(self.name)

python/grass/pygrass/raster/testsuite/test_numpy.py

@@ -6,6 +6,7 @@
 
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
+from grass.gunittest.utils import xfail_windows
 from numpy.random import default_rng
 from grass.pygrass.raster import raster2numpy, numpy2raster, RasterRow
 
@@ -48,6 +49,7 @@ def test_len(self):
         self.assertTrue(len(self.numpy_obj), 40)
         self.assertTrue(len(self.numpy_obj[0]), 60)
 
+    @xfail_windows
     def test_write(self):
         rng = default_rng()
         numpy2raster(rng.random([40, 60]), "FCELL", self.name, True)