[SPARK-45390][PYTHON] Remove distutils usage

LICENSE

@@ -221,6 +221,7 @@ connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaR
 Python Software Foundation License
 ----------------------------------
 
+python/pyspark/loose_version.py
 python/docs/source/_static/copybutton.js
 
 BSD 3-Clause

LICENSE-binary

@@ -421,6 +421,12 @@ This section summarizes those components and their licenses. See licenses-binary
 for text of these licenses.
 
 
+Python Software Foundation License
+----------------------------------
+
+python/pyspark/loose_version.py
+
+
 BSD 2-Clause
 ------------
 

dev/.rat-excludes

@@ -142,3 +142,4 @@ empty.proto
 LimitedInputStream.java
 TimSort.java
 xml-resources/*
+loose_version.py

python/pyspark/__init__.py

@@ -49,11 +49,6 @@
 from functools import wraps
 import types
 from typing import cast, Any, Callable, Optional, TypeVar, Union
-from warnings import filterwarnings
-
-filterwarnings(
-    "ignore", message="distutils Version classes are deprecated. Use packaging.version instead."
-)
 
 from pyspark.conf import SparkConf
 from pyspark.rdd import RDD, RDDBarrier

python/pyspark/loose_version.py

@@ -0,0 +1,75 @@
+# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# https://github.com/python/cpython/blob/3.11/LICENSE
+# File originates from the cpython source
+# https://github.com/python/cpython/blob/3.11/Lib/distutils/version.py
+
+import re
+from typing import Optional
+
+
+class LooseVersion:
+    component_re = re.compile(r"(\d+ | [a-z]+ | \.)", re.VERBOSE)
+
+    def __init__(self, vstring: Optional[str]) -> None:
+        if vstring:
+            self.parse(vstring)
+
+    def parse(self, vstring: str) -> None:
+        self.vstring = vstring
+        components = [x for x in self.component_re.split(vstring) if x and x != "."]
+        for i, obj in enumerate(components):
+            try:
+                components[i] = int(obj)
+            except ValueError:
+                pass
+
+        self.version = components
+
+    def __str__(self) -> str:
+        return self.vstring
+
+    def __repr__(self) -> str:
+        return "LooseVersion ('%s')" % str(self)
+
+    def __eq__(self, other):  # type: ignore[no-untyped-def]
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return c
+        return c == 0
+
+    def __lt__(self, other):  # type: ignore[no-untyped-def]
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return c
+        return c < 0
+
+    def __le__(self, other):  # type: ignore[no-untyped-def]
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return c
+        return c <= 0
+
+    def __gt__(self, other):  # type: ignore[no-untyped-def]
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return c
+        return c > 0
+
+    def __ge__(self, other):  # type: ignore[no-untyped-def]
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return c
+        return c >= 0
+
+    def _cmp(self, other):  # type: ignore[no-untyped-def]
+        if isinstance(other, str):
+            other = LooseVersion(other)
+        elif not isinstance(other, LooseVersion):
+            return NotImplemented
+
+        if self.version == other.version:
+            return 0
+        if self.version < other.version:
+            return -1
+        if self.version > other.version:
+            return 1

python/pyspark/pandas/plot/matplotlib.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-from distutils.version import LooseVersion
+from pyspark.loose_version import LooseVersion
 
 import matplotlib as mat
 import numpy as np

python/pyspark/pandas/supported_api_gen.py

@@ -19,7 +19,7 @@
 Generate 'Supported pandas APIs' documentation file
 """
 import warnings
-from distutils.version import LooseVersion
+from pyspark.loose_version import LooseVersion
 from enum import Enum, unique
 from inspect import getmembers, isclass, isfunction, signature
 from typing import Any, Callable, Dict, List, NamedTuple, Set, TextIO, Tuple

python/pyspark/pandas/tests/computation/test_any_all.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 
 import numpy as np

python/pyspark/pandas/tests/computation/test_corrwith.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 
 

python/pyspark/pandas/tests/computation/test_cov.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 import decimal
 

python/pyspark/pandas/tests/data_type_ops/test_date_ops.py

@@ -17,7 +17,6 @@
 
 import datetime
 import unittest
-from distutils.version import LooseVersion
 
 import pandas as pd
 from pandas.api.types import CategoricalDtype

python/pyspark/pandas/tests/diff_frames_ops/test_cov_corrwith.py

@@ -14,12 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 
 import pandas as pd
 import numpy as np
 
 from pyspark import pandas as ps
+from pyspark.loose_version import LooseVersion
 from pyspark.pandas.config import set_option, reset_option
 from pyspark.testing.pandasutils import PandasOnSparkTestCase
 from pyspark.testing.sqlutils import SQLTestUtils

python/pyspark/pandas/tests/groupby/test_aggregate.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 import unittest
-from distutils.version import LooseVersion
 
 import pandas as pd
 

python/pyspark/pandas/tests/groupby/test_apply_func.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 
 import numpy as np

python/pyspark/pandas/tests/groupby/test_head_tail.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 
 import numpy as np

python/pyspark/pandas/tests/groupby/test_index.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 
 import pandas as pd

python/pyspark/pandas/tests/groupby/test_split_apply.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 
 import pandas as pd

python/pyspark/pandas/tests/groupby/test_stat.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from distutils.version import LooseVersion
 import unittest
 
 import numpy as np

python/pyspark/pandas/tests/indexes/test_base.py

@@ -17,13 +17,13 @@
 
 import inspect
 import unittest
-from distutils.version import LooseVersion
 from datetime import datetime, timedelta
 
 import numpy as np
 import pandas as pd
 
 import pyspark.pandas as ps
+from pyspark.loose_version import LooseVersion
 from pyspark.pandas.exceptions import PandasNotImplementedError
 from pyspark.pandas.missing.indexes import (
     MissingPandasLikeDatetimeIndex,

python/pyspark/pandas/tests/indexes/test_category.py

@@ -16,12 +16,12 @@
 #
 
 import unittest
-from distutils.version import LooseVersion
 
 import pandas as pd
 from pandas.api.types import CategoricalDtype
 
 import pyspark.pandas as ps
+from pyspark.loose_version import LooseVersion
 from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
 
 

python/pyspark/pandas/tests/indexes/test_datetime.py

@@ -17,8 +17,6 @@
 
 import datetime
 
-from distutils.version import LooseVersion
-
 import numpy as np
 import pandas as pd
 

python/pyspark/pandas/tests/indexes/test_indexing.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 import unittest
-from distutils.version import LooseVersion
 
 import numpy as np
 import pandas as pd

python/pyspark/pandas/tests/indexes/test_reindex.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 import unittest
-from distutils.version import LooseVersion
 
 import numpy as np
 import pandas as pd

python/pyspark/pandas/tests/indexes/test_timedelta.py

@@ -17,7 +17,6 @@
 
 import unittest
 from datetime import timedelta
-from distutils.version import LooseVersion
 
 import pandas as pd
 

python/pyspark/pandas/tests/series/test_compute.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 import unittest
-from distutils.version import LooseVersion
 from itertools import product
 
 import numpy as np

python/pyspark/pandas/tests/test_csv.py

@@ -20,7 +20,6 @@
 import tempfile
 import unittest
 from contextlib import contextmanager
-from distutils.version import LooseVersion
 
 import pandas as pd
 import numpy as np

python/pyspark/pandas/tests/test_dataframe_conversion.py

@@ -21,7 +21,6 @@
 import tempfile
 import unittest
 import sys
-from distutils.version import LooseVersion
 
 import numpy as np
 import pandas as pd

python/pyspark/pandas/tests/test_dataframe_spark_io.py

@@ -18,12 +18,12 @@
 import unittest
 import glob
 import os
-from distutils.version import LooseVersion
 
 import numpy as np
 import pandas as pd
 
 from pyspark import pandas as ps
+from pyspark.loose_version import LooseVersion
 from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
 
 

python/pyspark/pandas/tests/test_ops_on_diff_frames.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 
-from distutils.version import LooseVersion
 from itertools import product
 import unittest
 

python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py

@@ -16,7 +16,6 @@
 #
 
 import unittest
-from distutils.version import LooseVersion
 
 import pandas as pd
 

python/pyspark/pandas/tests/test_rolling.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 #
 import unittest
-from distutils.version import LooseVersion
 
 import numpy as np
 import pandas as pd

python/pyspark/pandas/tests/test_series_conversion.py

@@ -17,7 +17,6 @@
 
 import unittest
 import sys
-from distutils.version import LooseVersion
 
 import pandas as pd
 

python/pyspark/pandas/tests/test_series_datetime.py

@@ -17,7 +17,6 @@
 
 import datetime
 import unittest
-from distutils.version import LooseVersion
 
 import numpy as np
 import pandas as pd

python/pyspark/pandas/tests/test_series_string.py

@@ -19,7 +19,6 @@
 import numpy as np
 import re
 import unittest
-from distutils.version import LooseVersion
 
 from pyspark import pandas as ps
 from pyspark.testing.pandasutils import PandasOnSparkTestCase

python/pyspark/pandas/tests/test_stats.py

@@ -16,7 +16,6 @@
 #
 
 import unittest
-from distutils.version import LooseVersion
 import numpy as np
 import pandas as pd
 

python/pyspark/sql/connect/client/core.py

@@ -19,6 +19,7 @@
     "SparkConnectClient",
 ]
 
+from pyspark.loose_version import LooseVersion
 from pyspark.sql.connect.utils import check_dependencies
 
 check_dependencies(__name__)
@@ -31,7 +32,6 @@
 import urllib.parse
 import uuid
 import sys
-from distutils.version import LooseVersion
 from types import TracebackType
 from typing import (
     Iterable,