Revert "Revert "Update to python3.11 for plpython (#17135)"" (#17190)

This reverts commit 721a760.

Author: yihong0618

README.Rhel-Rocky.bash

@@ -2,20 +2,17 @@
 
 # Install needed packages. Please add to this list if you discover additional prerequisites
 sudo yum group install -y "Development Tools"
-INSTALL_PKGS="apr-devel bison bzip2-devel cmake3 epel-release flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libzstd-devel openssl-devel python39 python39-devel python39-psutil python3-pip perl-ExtUtils-MakeMaker.noarch perl-ExtUtils-Embed.noarch readline-devel rsync xerces-c-devel zlib-devel python3-psutil python3-pyyaml python3-psycopg2"
+INSTALL_PKGS="apr-devel bison bzip2-devel cmake3 epel-release flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libzstd-devel openssl-devel python3.11 python3.11-devel python3.11-psutil python3.11-pip perl-ExtUtils-MakeMaker.noarch perl-ExtUtils-Embed.noarch readline-devel rsync xerces-c-devel zlib-devel python3-psutil python3-pyyaml python3-psycopg2"
 
 sudo yum install -y $INSTALL_PKGS
 
 sudo yum --enablerepo=powertools install -y libyaml-devel
 
 sudo yum install -y postgresql 
 sudo yum install -y postgresql-devel
+sudo yum install -y python3-pip
 
-pip3.9 install -r python-dependencies.txt
-
-#Remove the python3 softlink because on Rocky8 its pointing to python3.6 by default. We need python3.9 for gpdb7.
-sudo rm -rf /usr/bin/python3
-sudo ln -s /usr/bin/python3.9 /usr/bin/python3
+pip3 install -r python-dependencies.txt
 
 #For all Greenplum Database host systems running RHEL, CentOs or Rocky8, SELinux must either be Disabled or configured to allow unconfined access to Greenplum processes, directories, and the gpadmin user.
 setenforce 0

README.macOS.arm.bash

@@ -50,10 +50,10 @@ brew link --force libxml2
 brew install postgresql
 
 # Installing python3 libraries
-brew install python@3.9
+brew install python@3.11
 python_version=$(echo `ls  /opt/homebrew/Cellar/python@3.9/`)
-ln -s /opt/homebrew/Cellar/python@3.9/$python_version/bin/python3.9 /opt/homebrew/bin/python3
-ln -s /opt/homebrew/Cellar/python@3.9/$python_version/bin/pip3.9 /opt/homebrew/bin/pip3
+ln -s /opt/homebrew/Cellar/python@3.11/$python_version/bin/python3.11 /opt/homebrew/bin/python3
+ln -s /opt/homebrew/Cellar/python@3.11/$python_version/bin/pip3.11 /opt/homebrew/bin/pip3
 pip3 install -r python-dependencies.txt
 brew install pyenv
 pyenv install $python_version

concourse/scripts/common.bash

@@ -146,7 +146,7 @@ function configure() {
     # The full set of configure options which were used for building the
     # tree must be used here as well since the toplevel Makefile depends
     # on these options for deciding what to test. Since we don't ship
-    ./configure --prefix=/usr/local/greenplum-db-devel --disable-orca --enable-gpcloud --enable-orafce --enable-tap-tests --with-gssapi --with-libxml --with-openssl --with-perl --with-python --with-uuid=e2fs --with-llvm --with-zstd PYTHON=python3.9 PKG_CONFIG_PATH="${GPHOME}/lib/pkgconfig" ${CONFIGURE_FLAGS}
+    ./configure --prefix=/usr/local/greenplum-db-devel --disable-orca --enable-gpcloud --enable-orafce --enable-tap-tests --with-gssapi --with-libxml --with-openssl --with-perl --with-python --with-uuid=e2fs --with-llvm --with-zstd PYTHON=python3.11 PKG_CONFIG_PATH="${GPHOME}/lib/pkgconfig" ${CONFIGURE_FLAGS}
 
     popd
 }

concourse/scripts/ic_gpdb_resgroup.bash

@@ -63,7 +63,7 @@ run_resgroup_test() {
         export CPPFLAGS="-I\${GPHOME}/include"
 
         cd /home/gpadmin/gpdb_src
-        PYTHON=python3.9 ./configure --prefix=/usr/local/greenplum-db-devel \
+        PYTHON=python3.11 ./configure --prefix=/usr/local/greenplum-db-devel \
             --without-zlib --without-rt --without-libcurl \
             --without-libedit-preferred --without-readline \
             --disable-gpcloud --disable-gpfdist --disable-orca \

concourse/scripts/ic_gpdb_resgroup_v2.bash

@@ -37,7 +37,7 @@ run_resgroup_test() {
         export CPPFLAGS="-I\${GPHOME}/include"
 
         cd /home/gpadmin/gpdb_src
-        PYTHON=python3.9 ./configure --prefix=/usr/local/greenplum-db-devel \
+        PYTHON=python3.11 ./configure --prefix=/usr/local/greenplum-db-devel \
             --without-zlib --without-rt --without-libcurl \
             --without-libedit-preferred --without-readline \
             --disable-gpcloud --disable-gpfdist --disable-orca \

concourse/scripts/ic_resgroup.bash

@@ -92,7 +92,7 @@ run_resgroup_icw_test() {
         ./configure --prefix=/usr/local/greenplum-db-devel --disable-orca --enable-gpfdist \
          --enable-gpcloud --enable-orafce --enable-tap-tests \
          --with-gssapi --with-libxml --with-openssl --with-perl --with-python \
-         --with-uuid=e2fs --with-llvm --with-zstd PYTHON=python3.9 PKG_CONFIG_PATH="${GPHOME}/lib/pkgconfig" ${CONFIGURE_FLAGS}
+         --with-uuid=e2fs --with-llvm --with-zstd PYTHON=python3.11 PKG_CONFIG_PATH="${GPHOME}/lib/pkgconfig" ${CONFIGURE_FLAGS}
 
         LANG=en_US.utf8 make create-demo-cluster WITH_MIRRORS=${WITH_MIRRORS:-true}
         source /home/gpadmin/gpdb_src/gpAux/gpdemo/gpdemo-env.sh

gpAux/Makefile

@@ -139,7 +139,7 @@ endif
 
 
 ifneq (false, ${PG_LANG})
-CONFIGFLAGS+= --with-perl --with-python PYTHON=python3.9
+CONFIGFLAGS+= --with-perl --with-python PYTHON=python3.11
 ifdef TCL_CFG
 CONFIGFLAGS+= --with-tcl-config=${TCL_CFG}
 endif

gpdb-doc/markdown/analytics/pl_python.html.md

@@ -18,7 +18,7 @@ PL/Python is a loadable procedural language. With the Greenplum Database PL/Pyth
 
 You can run PL/Python code blocks as anonymous code blocks. See the [DO](../ref_guide/sql_commands/DO.html) command in the *Greenplum Database Reference Guide*.
 
-The Greenplum Database PL/Python extension is installed by default with Greenplum Database. `plpython3u` supports developing functions using Python 3.9. The VMware Greenplum installation process installs a Python 3.9 environment to your system as one of its dependencies.
+The Greenplum Database PL/Python extension is installed by default with Greenplum Database. `plpython3u` supports developing functions using Python 3.11. The VMware Greenplum installation process installs a Python 3.11 environment to your system as one of its dependencies.
 
 ### <a id="topic3"></a>Greenplum Database PL/Python Limitations
 
@@ -34,7 +34,7 @@ The PL/Python language is installed with Greenplum Database. To create and run a
 
 For each database that requires its use, register the PL/Python language with the SQL command `CREATE EXTENSION`. Because PL/Python is an untrusted language, only superusers can register PL/Python with a database.
 
-Run this command as the `gpadmin` user to register PL/Python with Python 3.9 support:
+Run this command as the `gpadmin` user to register PL/Python with Python 3.11 support:
 
 ```
 $ psql -d testdb -c 'CREATE EXTENSION plpython3u;'
@@ -45,7 +45,7 @@ PL/Python is registered as an untrusted language.
 ### <a id="topic6"></a>Removing PL/Python Support
 
 
-Run this command as the `gpadmin` user to remove support for PL/Python for Python 3.9:
+Run this command as the `gpadmin` user to remove support for PL/Python for Python 3.11:
 
 ```
 $ psql -d testdb -c 'DROP EXTENSION plpython3u;'
@@ -332,41 +332,41 @@ A transaction cannot be ended when an explicit subtransaction is active.
 
 When you install a Python module for development with PL/Python, the Greenplum Database Python environment must have the module added to it across all segment hosts and mirror hosts in the cluster. When expanding Greenplum Database, you must add the Python modules to the new segment hosts.
 
-Greenplum Database provides a collection of data science-related Python modules that you can use to easily develop PL/Python functions in Greenplum. The modules are provided as two `.gppkg` format files that can be installed into a Greenplum cluster using the `gppkg` utility, with one package supporting development with Python 3.9. See [Python Data Science Module Packages](/oss/install_guide/install_python_dsmod.html) for installation instructions and descriptions of the provided modules.
+Greenplum Database provides a collection of data science-related Python modules that you can use to easily develop PL/Python functions in Greenplum. The modules are provided as two `.gppkg` format files that can be installed into a Greenplum cluster using the `gppkg` utility, with one package supporting development with Python 3.11. See [Python Data Science Module Packages](/oss/install_guide/install_python_dsmod.html) for installation instructions and descriptions of the provided modules.
 
 To develop with modules that are not part of the Python Data Science Module packages, you can use Greenplum utilities such as `gpssh` and `gpsync` to run commands or copy files to all hosts in the Greenplum cluster. These sections describe how to use those utilities to install and use additional Python modules:
 
 -   [Verifying the Python Environment](#about_python_env)
 -   [Installing Python pip](#topic_yx3_yjq_rt)
--   [Installing Python Packages for Python 3.9](#pip39)
+-   [Installing Python Packages for Python 3.11](#pip311)
 -   [Building and Installing Python Modules Locally](#topic_j53_5jq_rt)
 -   [Testing Installed Python Modules](#topic_e4p_gcw_vt)
 
 ### <a id="about_python_env"></a>Verifying the Python Environment
 
-The plpython3u is built with Python 3.9. To check the Python environment, you can use the `which` command:
+The plpython3u is built with Python 3.11. To check the Python environment, you can use the `which` command:
 
 ```
-which python3.9
+which python3.11
 ```
 
 When running shell commands on remote hosts with `gpssh`, specify the `-s` option to source the `greenplum_path.sh` file before running commands on the remote hosts. For example, this command should display the Python installed with Greenplum Database on each host specified in the `gpdb_hosts` file.
 
 ```
-gpssh -s -f gpdb_hosts which python3.9
+gpssh -s -f gpdb_hosts which python3.11
 ```
 
 To display the list of currently installed Python modules, run this command.
 
 ```
-python3.9 -c "help('modules')"
+python3.11 -c "help('modules')"
 ```
 
 You can optionally run `gpssh` in interactive mode to display Python modules on remote hosts. This example starts `gpssh` in interactive mode and lists the Python modules on the Greenplum Database host `sdw1`.
 
 ```
 $ gpssh -s -h sdw1
-=> python3.9 -c "help('modules')"
+=> python3.11 -c "help('modules')"
 . . .
 => exit
 $
@@ -376,24 +376,24 @@ $
 
 The Python utility `pip` installs Python packages that contain Python modules and other resource files from versioned archive files.
 
-For Python 3.9, use:
+For Python 3.11, use:
 ```
-python3.9 -m ensurepip --default-pip
+python3.11 -m ensurepip --default-pip
 ```
 
 The command runs the `ensurepip` module to bootstrap \(install and configure\) the `pip` utility from the local Python installation.
 
 You can run this command to ensure the `pip`, `setuptools` and `wheel` projects are current. Current Python projects ensure that you can install Python packages from source distributions or pre-built distributions \(wheels\).
 
 ```
-python3.9 -m pip install --upgrade pip setuptools wheel
+python3.11 -m pip install --upgrade pip setuptools wheel
 ```
 
 You can use `gpssh` to run the commands on the Greenplum Database hosts. This example runs `gpssh` in interactive mode to install `pip` on the hosts listed in the file `gpdb_hosts`.
 
 ```
 $ gpssh -s -f gpdb_hosts
-=> python3.9 -m ensurepip --default-pip
+=> python3.11 -m ensurepip --default-pip
 [centos6-cdw1] Ignoring indexes: https://pypi.python.org/simple
 [centos6-cdw1] Collecting setuptools
 [centos6-cdw1] Collecting pip
@@ -422,7 +422,7 @@ For more information about installing Python packages, see [https://packaging.py
 
 
 ```
-python3.9 -m pip install --user numpy scipy
+python3.11 -m pip install --user numpy scipy
 ```
 
 The `--user` option attempts to avoid conflicts when installing Python packages.
@@ -431,15 +431,15 @@ You can use `gpssh` to run the command on the Greenplum Database hosts.
 
 For information about these and other Python packages, see [References](#topic12).
 
-### <a id="pip39"></a>Installing Python Packages to a Non-Standard Location
+### <a id="pip311"></a>Installing Python Packages to a Non-Standard Location
 
-You can optionally install Python 3.9 modules to a non-standard location by using the `--prefix` option with `pip`. For example:
+You can optionally install Python 3.11 modules to a non-standard location by using the `--prefix` option with `pip`. For example:
 
 ```
 gpssh -s -f gpdb_hosts
 => unset PYTHONHOME
 => unset PYTHONPATH
-=> $GPHOME/ext/python3.9 -m pip install --prefix=/home/gpadmin/my_python numpy scipy
+=> $GPHOME/ext/python3.11 -m pip install --prefix=/home/gpadmin/my_python numpy scipy
 ```
 
 If you use this option, keep in mind that the `PYTHONPATH` environment variable setting is cleared before initializing or executing functions using `plpython3u`. If you want to use modules installed to a custom location, you must configure the paths to those modules using the Greenplum configuration parameter `plpython3.python_path` instead of  `PYTHONPATH`. For example:
@@ -512,14 +512,14 @@ If `FAILURE` is returned, these are some possible causes:
     Make sure you get no errors when running command on the segment host as the `gpadmin` user. This `gpssh` command tests importing the numpy module on the segment host `cdw1`.
 
     ```
-    gpssh -s -h cdw1 python3.9 -c "import numpy"
+    gpssh -s -h cdw1 python3.11 -c "import numpy"
     ```
 
 -   The `plpython3.python_path` has not been set to the correct location.
 
 ## <a id="topic11"></a>Examples
 
-This PL/Python function example uses Python 3.9 and returns the value of pi using the `numpy` module:
+This PL/Python function example uses Python 3.11 and returns the value of pi using the `numpy` module:
 
 ```
 CREATE OR REPLACE FUNCTION testpi()

gpdb-doc/markdown/install_guide/install_python_dsmod.html.md

@@ -6,17 +6,17 @@ Greenplum Database provides a collection of data science-related Python modules
 
 This section contains the following information:
 
--   [Data Science Package for Python 3.9 Modules](#topic_pydatascimod3)
+-   [Data Science Package for Python 3.11 Modules](#topic_pydatascimod3)
 -   [Installing a Data Science Package for Python](#topic_instpdsm)
 -   [Uninstalling a Data Science Package for Python](#topic_removepdsm)
 
 For information about the Greenplum Database PL/Python Language, see [Greenplum PL/Python Language Extension](../analytics/pl_python.html).
 
 **Parent topic:** [Installing Optional Extensions \(VMware Greenplum\)](data_sci_pkgs.html)
 
-## <a id="topic_pydatascimod3"></a>Data Science Package for Python 3.9 Modules 
+## <a id="topic_pydatascimod3"></a>Data Science Package for Python 3.11 Modules 
 
-The following table lists the modules that are provided in the Data Science Package for Python 3.9.
+The following table lists the modules that are provided in the Data Science Package for Python 3.11.
 
 |Module Name|Description/Used For|
 |-----------|--------------------|
@@ -193,14 +193,14 @@ $ sudo yum install tk
 
 1.  Locate the Data Science Package for Python that you built or downloaded.
 
-    The file name format of the package is `DataSciencePython<pythonversion>-gp7-rhel<n>-x86_64.gppkg`.  For example: `DataSciencePython3.9-3.0.0-gp7-rhel8_x86_64.gppkg`.
+    The file name format of the package is `DataSciencePython<pythonversion>-<data-science-bundle-version>-gp7-rhel<n>-x86_64.gppkg`.  For example: `DataSciencePython3.11-3.0.0-gp7-rhel8_x86_64.gppkg`.
 
 2.  Copy the package to the Greenplum Database coordinator host.
 3.  Follow the instructions in [Verifying the Greenplum Database Software Download](../install_guide/verify_sw.html) to verify the integrity of the *Greenplum Procedural Languages Python Data Science Package* software.
 4.  Use the `gppkg` command to install the package. For example:
 
     ```
-    $ gppkg install DataSciencePython3.9-1.2.0-gp7-el8_x86_64.gppkg
+    $ gppkg install DataSciencePython3.11-1.2.0-gp7-el8_x86_64.gppkg
     ```
 
     `gppkg` installs the Data Science Package for Python modules on all nodes in your Greenplum Database cluster. The command also updates the `PYTHONPATH`, `PATH`, and `LD_LIBRARY_PATH` environment variables in your `greenplum_path.sh` file.
@@ -216,7 +216,7 @@ $ sudo yum install tk
 The Data Science Package for Python modules are installed in the following directory:
 
 ```
-$GPHOME/ext/DataSciencePython/lib/python3.9/site-packages/
+$GPHOME/ext/DataSciencePython/lib/python3.11/site-packages/
 ```
 
 ## <a id="topic_removepdsm"></a>Uninstalling a Data Science Package for Python 

gpdb-doc/markdown/install_guide/platform-requirements-overview.html.md

@@ -46,7 +46,8 @@ Greenplum Database 7 requires the following software packages on RHEL systems. T
 -   python3-psycopg2
 -   python3-psutil
 -   python3-pyyaml
--   python39
+-   python3.11
+-   python3.11-devel
 -   readline
 -   rsync
 -   sed
@@ -83,12 +84,12 @@ Greenplum Databased 7 supports these Java versions for PL/Java and PXF:
 
 #### <a id="topic_xbl_mkx_python"></a>Python
 
-Greenplum Database uses the system default `python3` for the Greenplum management utilities, and `python3.9` for the [PL/Python module](../analytics/pl_python.html). For most of the supported OS versions, the system default `python3` is `python3.9`. If you are installing Greenplum Database on Rocky Linux 8, the default `python3` version included is `python3.6`. You may want to unify the `python3` versions to `python3.9` by running the following commands:
+Greenplum Database uses the system default `python3` for the Greenplum management utilities, and `python3.11` for the [PL/Python module](../analytics/pl_python.html). For most of the supported OS versions, the system default `python3` is `python3.9`. If you are installing Greenplum Database on Rocky Linux 8, the default `python3` version included is `python3.6`. You may want to unify the `python3` versions to `python3.11` by running the following commands:
 
 ```
-sudo yum install python39-psycopg2 python39-pyyaml python39-psutil
-sudo update-alternatives set python3 /usr/bin/python39
-sudo update-alternatives set python /usr/bin/python39
+sudo yum install python3.11 python3.11-devel python3.11-psycopg2 python3.11-pyyaml python3.11-psutil
+sudo update-alternatives set python3 /usr/bin/python3.11
+sudo update-alternatives set python /usr/bin/python3.11
 ```
 
 ## <a id="topic31"></a>VMware Greenplum Tools and Extensions Compatibility
@@ -126,7 +127,7 @@ This table lists the versions of the Greenplum Extensions that are compatible wi
 <td class="entry cell-norowborder" style="vertical-align:top;" headers="d78288e689 ">Supports Java 8 and 11.</td>
 </tr>
 <tr class="row">
-<td class="entry nocellnorowborder" style="vertical-align:top;" headers="d78288e683 "><a class="xref" href="../install_guide/install_python_dsmod.html">Python 3.9 Data Science Module Package</a></td>
+<td class="entry nocellnorowborder" style="vertical-align:top;" headers="d78288e683 "><a class="xref" href="../install_guide/install_python_dsmod.html">Python 3.11 Data Science Module Package</a></td>
 <td class="entry nocellnorowborder" style="vertical-align:top;" headers="d78288e686 ">1.2</td>
 <td class="entry cell-norowborder" style="vertical-align:top;" headers="d78288e689 "> </td>
 </tr>

gpdb-doc/markdown/ref_guide/modules/postgresml.html.md

@@ -6,14 +6,14 @@ The `postgresml` module provides PostgresML functions for using tens of thousand
 
 ## <a id="prereqs"></a>Before Registering the `postgresml` Module
 
-Before registering the `postgresml` module, you must install the Data Science bundle for Python3.9, add the `pgml` library to the set of libraries the VMware Greenplum server loads at startup, and set the Python virtual environment:
+Before registering the `postgresml` module, you must install the Data Science bundle for Python3.11, add the `pgml` library to the set of libraries the VMware Greenplum server loads at startup, and set the Python virtual environment:
 
-1. Install the Data Science bundle for Python 3.9.
+1. Install the Data Science bundle for Python 3.11.
 
     For example:
 
     ```
-    gppkg install DataSciencePython3.9-x.x.x-gp7-el8_x86_64.gppkg 
+    gppkg install DataSciencePython3.11-x.x.x-gp7-el8_x86_64.gppkg 
     ```
 
     where x.x.x is the version string.
@@ -33,13 +33,13 @@ Before registering the `postgresml` module, you must install the Data Science bu
     At the session level:
 
     ```
-    SET pgml.venv='$GPHOME/ext/DataSciencePython3.9';
+    SET pgml.venv='$GPHOME/ext/DataSciencePython3.11';
     ```
 
     To last beyond a session:
 
     ```
-    gpconfig -c pgml.venv -v '$GPHOME/ext/DataSciencePython3.9'
+    gpconfig -c pgml.venv -v '$GPHOME/ext/DataSciencePython3.11'
     gpstop -u
     ```