Merge pull request #1686 from rstudio/python-3-13-compat

Updates for Python 3.13

.github/workflows/R-CMD-check.yaml

@@ -21,26 +21,26 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - { os: macOS-latest  , r: 'release', python: '3.10' }
-          - { os: windows-latest, r: 'release', python: '3.10' }
-          - { os: ubuntu-latest , r: 'release', python: '3.10' }
+          - { os: macOS-latest  , r: 'release', python: '3.11' }
+          - { os: windows-latest, r: 'release', python: '3.11' }
+          - { os: ubuntu-latest , r: 'release', python: '3.11' }
 
           # test R oldrel on mac too: https://github.com/RcppCore/RcppArmadillo/issues/447
-          - { os: macOS-latest , r: 'oldrel'  , python: '3.10' }
+          - { os: macOS-latest , r: 'oldrel'  , python: '3.11' }
 
-          - { os: ubuntu-latest, r: 'oldrel-1', python: '3.9' }
-          - { os: ubuntu-latest, r: 'oldrel-2', python: '3.9' }
-          - { os: ubuntu-20.04,  r: 'oldrel-3', python: '3.8' }
-          - { os: ubuntu-20.04,  r: 'oldrel-4', python: '3.8' }
+          - { os: ubuntu-latest, r: 'oldrel-1', python: '3.10' }
+          - { os: ubuntu-latest, r: 'oldrel-2', python: '3.10' }
+          - { os: ubuntu-20.04,  r: 'oldrel-3', python: '3.9' }
+          - { os: ubuntu-20.04,  r: 'oldrel-4', python: '3.9' }
           # https://api.r-hub.io/rversions/resolve/oldrel/4
 
-          - { os: ubuntu-latest, r: 'release', python: '3.7'  }
           - { os: ubuntu-latest, r: 'release', python: '3.8'  }
           - { os: ubuntu-latest, r: 'release', python: '3.9' }
           - { os: ubuntu-latest, r: 'release', python: '3.11' }
           - { os: ubuntu-latest, r: 'release', python: '3.12' }
+          - { os: ubuntu-latest, r: 'release', python: '3.13' }
 
-          - { os: ubuntu-latest, r: 'devel', python: '3.12', http-user-agent: 'release' }
+          - { os: ubuntu-latest, r: 'devel', python: '3.13', http-user-agent: 'release' }
 
           # test with one debug build of python
           # disabled; failing presently

NEWS.md

@@ -1,5 +1,19 @@
 # reticulate (development version)
 
+- The S3 classes for some (rarely encountered) Python objects have changed.
+  Only Python objects with non-standard `__module__` values are affected.
+  If a Python object’s parent class’s `__module__` attribute does not resolve to a string,
+  reticulate:
+    - Attempts to resolve it from the class's class, if it's a metaclass.
+    - If no string can be resolved, reticulate no longer implicitly prepends
+      'python.builtin.' as the class prefix, instead it uses just the `__name__`.
+  (See #1686 for more context)
+
+- Added support for Python 3.13. Note that Python 3.13 removed support
+  for `classmethod` descriptors, which may affect the S3 class of
+  some Python objects that use metaclass properties to resolve a class’s
+  `__module__` or `__name__` attribute. (#1686)
+
 - `py_is_null_xptr()` and `[[` now load delayed modules (#1688).
 
 - Fixed error when attempting to use a python venv created with `uv` (#1678)

R/RcppExports.R

@@ -102,8 +102,8 @@ py_clear_error <- function() {
     invisible(.Call(`_reticulate_py_clear_error`))
 }
 
-py_initialize <- function(python, libpython, pythonhome, virtualenv_activate, python3, interactive, numpy_load_error) {
-    invisible(.Call(`_reticulate_py_initialize`, python, libpython, pythonhome, virtualenv_activate, python3, interactive, numpy_load_error))
+py_initialize <- function(python, libpython, pythonhome, virtualenv_activate, python_major_version, python_minor_version, interactive, numpy_load_error) {
+    invisible(.Call(`_reticulate_py_initialize`, python, libpython, pythonhome, virtualenv_activate, python_major_version, python_minor_version, interactive, numpy_load_error))
 }
 
 py_finalize <- function() {

R/config.R

@@ -896,7 +896,7 @@ python_config <- function(python,
     executable           = config$Executable, # sys.executable
     base_executable      = config$BaseExecutable, # sys._base_executable; exe for venv starter
     version_string       = version_string,
-    version              = version,
+    version              = as.package_version(version),
     architecture         = architecture,
     anaconda             = anaconda,
     conda                = config$IsConda,

R/package.R

@@ -204,7 +204,8 @@ initialize_python <- function(required_module = NULL, use_environment = NULL) {
                     config$libpython,
                     config$pythonhome,
                     config$virtualenv_activate,
-                    config$version >= "3.0",
+                    config$version$major,
+                    config$version$minor,
                     interactive(),
                     numpy_load_error)
 

src/RcppExports.cpp

@@ -200,18 +200,19 @@ BEGIN_RCPP
 END_RCPP
 }
 // py_initialize
-void py_initialize(const std::string& python, const std::string& libpython, const std::string& pythonhome, const std::string& virtualenv_activate, bool python3, bool interactive, const std::string& numpy_load_error);
-RcppExport SEXP _reticulate_py_initialize(SEXP pythonSEXP, SEXP libpythonSEXP, SEXP pythonhomeSEXP, SEXP virtualenv_activateSEXP, SEXP python3SEXP, SEXP interactiveSEXP, SEXP numpy_load_errorSEXP) {
+void py_initialize(const std::string& python, const std::string& libpython, const std::string& pythonhome, const std::string& virtualenv_activate, int python_major_version, int python_minor_version, bool interactive, const std::string& numpy_load_error);
+RcppExport SEXP _reticulate_py_initialize(SEXP pythonSEXP, SEXP libpythonSEXP, SEXP pythonhomeSEXP, SEXP virtualenv_activateSEXP, SEXP python_major_versionSEXP, SEXP python_minor_versionSEXP, SEXP interactiveSEXP, SEXP numpy_load_errorSEXP) {
 BEGIN_RCPP
     Rcpp::RNGScope rcpp_rngScope_gen;
     Rcpp::traits::input_parameter< const std::string& >::type python(pythonSEXP);
     Rcpp::traits::input_parameter< const std::string& >::type libpython(libpythonSEXP);
     Rcpp::traits::input_parameter< const std::string& >::type pythonhome(pythonhomeSEXP);
     Rcpp::traits::input_parameter< const std::string& >::type virtualenv_activate(virtualenv_activateSEXP);
-    Rcpp::traits::input_parameter< bool >::type python3(python3SEXP);
+    Rcpp::traits::input_parameter< int >::type python_major_version(python_major_versionSEXP);
+    Rcpp::traits::input_parameter< int >::type python_minor_version(python_minor_versionSEXP);
     Rcpp::traits::input_parameter< bool >::type interactive(interactiveSEXP);
     Rcpp::traits::input_parameter< const std::string& >::type numpy_load_error(numpy_load_errorSEXP);
-    py_initialize(python, libpython, pythonhome, virtualenv_activate, python3, interactive, numpy_load_error);
+    py_initialize(python, libpython, pythonhome, virtualenv_activate, python_major_version, python_minor_version, interactive, numpy_load_error);
     return R_NilValue;
 END_RCPP
 }
@@ -854,7 +855,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"_reticulate_py_activate_virtualenv", (DL_FUNC) &_reticulate_py_activate_virtualenv, 1},
     {"_reticulate_main_process_python_info", (DL_FUNC) &_reticulate_main_process_python_info, 0},
     {"_reticulate_py_clear_error", (DL_FUNC) &_reticulate_py_clear_error, 0},
-    {"_reticulate_py_initialize", (DL_FUNC) &_reticulate_py_initialize, 7},
+    {"_reticulate_py_initialize", (DL_FUNC) &_reticulate_py_initialize, 8},
     {"_reticulate_py_finalize", (DL_FUNC) &_reticulate_py_finalize, 0},
     {"_reticulate_py_is_none", (DL_FUNC) &_reticulate_py_is_none, 1},
     {"_reticulate_py_compare_impl", (DL_FUNC) &_reticulate_py_compare_impl, 3},

src/libpython.cpp

@@ -159,6 +159,7 @@ void initialize_type_objects(bool python3) {
   if (builtins == NULL) goto error;
   PyExc_KeyboardInterrupt = PyObject_GetAttrString(builtins, "KeyboardInterrupt"); // new ref
   PyExc_RuntimeError = PyObject_GetAttrString(builtins, "RuntimeError"); // new ref
+  PyExc_AttributeError = PyObject_GetAttrString(builtins, "AttributeError"); // new ref
 
   if (PyErr_Occurred()) { error:
      // Should never happen. If you see this please report a bug.
@@ -174,28 +175,40 @@ if (!loadSymbol(pLib_, #name, (void**)&as, pError)) \
 if (!loadSymbol(pLib_, #name, (void**) &libpython::name, pError)) \
   return false;
 
-bool SharedLibrary::load(const std::string& libPath, bool python3, std::string* pError)
+bool SharedLibrary::load(const std::string& libPath, int major_ver, int minor_ver, std::string* pError)
 {
   if (!loadLibrary(libPath, &pLib_, pError))
     return false;
 
-  return loadSymbols(python3, pError);
+  return loadSymbols(major_ver, minor_ver, pError);
 }
 
 // Define "slow" fallback implementation for Py version <= 3.9
 int _PyIter_Check(PyObject* o) {
   return PyObject_HasAttrString(o, "__next__");
 }
 
+int _PyObject_GetOptionalAttrString(PyObject* obj, const char* attr_name, PyObject** result) {
+  *result = PyObject_GetAttrString(obj, attr_name);
+  if (*result == NULL) {
+    if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+      PyErr_Clear();
+      return 0;
+    }
+    return -1;
+  }
+  return 1;
+}
 
-bool LibPython::loadSymbols(bool python3, std::string* pError)
+
+bool LibPython::loadSymbols(int python_major_ver, int python_minor_ver, std::string* pError)
 {
   bool is64bit = sizeof(size_t) >= 8;
 
   LOAD_PYTHON_SYMBOL(Py_InitializeEx)
   LOAD_PYTHON_SYMBOL(Py_Finalize)
   LOAD_PYTHON_SYMBOL(Py_IsInitialized)
-  LOAD_PYTHON_SYMBOL(Py_GetVersion)
+  LOAD_PYTHON_SYMBOL(Py_GetVersion)          // Deprecated in 3.13
   LOAD_PYTHON_SYMBOL(Py_AddPendingCall)
   LOAD_PYTHON_SYMBOL(Py_MakePendingCalls)
   LOAD_PYTHON_SYMBOL(PyErr_SetInterrupt)
@@ -209,6 +222,13 @@ bool LibPython::loadSymbols(bool python3, std::string* pError)
   LOAD_PYTHON_SYMBOL(PyObject_SetAttr)
   LOAD_PYTHON_SYMBOL(PyObject_GetAttrString)
   LOAD_PYTHON_SYMBOL(PyObject_HasAttrString)
+  if (python_major_ver >= 3 && python_minor_ver >= 13) {
+    LOAD_PYTHON_SYMBOL(PyObject_HasAttrStringWithError)
+    LOAD_PYTHON_SYMBOL(PyObject_GetOptionalAttrString)
+  } else {
+    LOAD_PYTHON_SYMBOL_AS(PyObject_HasAttrStringWithError, PyObject_HasAttrString)
+    PyObject_GetOptionalAttrString = &_PyObject_GetOptionalAttrString;
+  }
   LOAD_PYTHON_SYMBOL(PyObject_SetAttrString)
   LOAD_PYTHON_SYMBOL(PyObject_GetItem)
   LOAD_PYTHON_SYMBOL(PyObject_SetItem)
@@ -314,9 +334,9 @@ bool LibPython::loadSymbols(bool python3, std::string* pError)
   if (!loadSymbol(pLib_, names, (void**)&PyUnicode_AsEncodedString, pError) )
     return false;
 
-  if (python3) {
+  if (python_major_ver >= 3) {
     LOAD_PYTHON_SYMBOL(PyException_SetTraceback)
-    LOAD_PYTHON_SYMBOL(Py_GetProgramFullPath)
+    LOAD_PYTHON_SYMBOL(Py_GetProgramFullPath)   // Deprecated in 3.13
 
     // Debug versions of Python will provide PyModule_Create2TraceRefs,
     // while release versions will provide PyModule_Create
@@ -346,7 +366,7 @@ bool LibPython::loadSymbols(bool python3, std::string* pError)
     } else {
       LOAD_PYTHON_SYMBOL(Py_InitModule4)
     }
-    LOAD_PYTHON_SYMBOL_AS(Py_GetProgramFullPath, Py_GetProgramFullPath_v2)
+    LOAD_PYTHON_SYMBOL_AS(Py_GetProgramFullPath, Py_GetProgramFullPath_v2)  // Deprecated in 3.13
     LOAD_PYTHON_SYMBOL(PyString_AsStringAndSize)
     LOAD_PYTHON_SYMBOL(PyString_FromStringAndSize)
     LOAD_PYTHON_SYMBOL(PyString_FromString)

src/libpython.h

@@ -145,6 +145,7 @@ LIBPYTHON_EXTERN PyObject* Py_Tuple;
 LIBPYTHON_EXTERN PyObject* Py_Complex;
 LIBPYTHON_EXTERN PyObject* Py_ByteArray;
 LIBPYTHON_EXTERN PyObject* PyExc_KeyboardInterrupt;
+LIBPYTHON_EXTERN PyObject* PyExc_AttributeError;
 LIBPYTHON_EXTERN PyObject* PyExc_RuntimeError;
 LIBPYTHON_EXTERN PyObject* PyExc_ValueError;
 
@@ -259,6 +260,11 @@ LIBPYTHON_EXTERN PyObject* (*PyObject_GetAttrString)(PyObject*, const char *);
 LIBPYTHON_EXTERN int (*PyObject_HasAttrString)(PyObject*, const char *);
 LIBPYTHON_EXTERN int (*PyObject_SetAttrString)(PyObject*, const char *, PyObject*);
 
+// added in Python 3.13
+LIBPYTHON_EXTERN int (*PyObject_HasAttrStringWithError)(PyObject*, const char *);
+LIBPYTHON_EXTERN int (*PyObject_GetOptionalAttrString)(PyObject* obj, const char* attr_name, PyObject** result);
+
+
 LIBPYTHON_EXTERN PyObject* (*PyObject_GetItem)(PyObject*, PyObject*);
 LIBPYTHON_EXTERN int (*PyObject_SetItem)(PyObject*, PyObject*, PyObject*);
 LIBPYTHON_EXTERN int (*PyObject_DelItem)(PyObject*, PyObject*);
@@ -636,12 +642,12 @@ bool import_numpy_api(bool python3, std::string* pError);
 class SharedLibrary {
 
 public:
-  bool load(const std::string& libPath, bool python3, std::string* pError);
+  bool load(const std::string& libPath, int major_ver, int minor_ver, std::string* pError);
   bool unload(std::string* pError);
   virtual ~SharedLibrary() {}
 
 private:
-  virtual bool loadSymbols(bool python3, std::string* pError) = 0;
+  virtual bool loadSymbols(int major_ver, int minor_ver, std::string* pError) = 0;
 
 protected:
   SharedLibrary() : pLib_(NULL) {}
@@ -656,7 +662,7 @@ class LibPython : public SharedLibrary {
 private:
   LibPython() : SharedLibrary() {}
   friend SharedLibrary& libPython();
-  virtual bool loadSymbols(bool python3, std::string* pError);
+  virtual bool loadSymbols(int major_ver, int minor_ver, std::string* pError);
 };
 
 inline SharedLibrary& libPython() {