From 36112effe154c54ade294eb48677dd8876f5f658 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 11 Jul 2022 21:46:04 -0700 Subject: [PATCH 01/69] WIP: Build pandas with meson --- generate_pxi.py | 40 ++++++++++++ meson.build | 18 ++++++ pandas/_libs/meson.build | 107 ++++++++++++++++++++++++++++++++ pandas/_libs/tslibs/meson.build | 46 ++++++++++++++ pandas/meson.build | 21 +++++++ pyproject.toml | 3 +- 6 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 generate_pxi.py create mode 100644 meson.build create mode 100644 pandas/_libs/meson.build create mode 100644 pandas/_libs/tslibs/meson.build create mode 100644 pandas/meson.build diff --git a/generate_pxi.py b/generate_pxi.py new file mode 100644 index 0000000000000..d142a0e816d42 --- /dev/null +++ b/generate_pxi.py @@ -0,0 +1,40 @@ +import argparse +import os + +from Cython import Tempita + + +def process_tempita(pxifile, outfile): + if ( + os.path.exists(outfile) + and os.stat(pxifile).st_mtime < os.stat(outfile).st_mtime + ): + # if .pxi.in is not updated, no need to output .pxi + return + + with open(pxifile) as f: + tmpl = f.read() + pyxcontent = Tempita.sub(tmpl) + + with open(outfile, "w") as f: + f.write(pyxcontent) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("infile", type=str, help="Path to the input file") + parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory") + args = parser.parse_args() + + if not args.infile.endswith(".in"): + raise ValueError(f"Unexpected extension: {args.infile}") + + outdir_abs = os.path.join(os.getcwd(), args.outdir) + outfile = os.path.join( + outdir_abs, os.path.splitext(os.path.split(args.infile)[1])[0] + ) + + process_tempita(args.infile, outfile) + + +main() diff --git a/meson.build b/meson.build new file mode 100644 index 0000000000000..4a7d4d94b0624 --- /dev/null +++ b/meson.build @@ -0,0 +1,18 @@ +# This file is adapted from https://github.com/scipy/scipy/blob/main/meson.build +project( + 'pandas', + 'c', 'cpp', 'cython', + version: '1.5.0.dev0', + license: 'BSD-3', + meson_version: '>=0.60', + default_options: [ + 'buildtype=debugoptimized' + ] +) + +py_mod = import('python') +py = py_mod.find_installation('python3') +py_dep = py.dependency() +tempita = files('generate_pxi.py') + +subdir('pandas') diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build new file mode 100644 index 0000000000000..fdc1c1d2887ee --- /dev/null +++ b/pandas/_libs/meson.build @@ -0,0 +1,107 @@ +_algos_common_helper = custom_target('algos_common_helper_pxi', + output: 'algos_common_helper.pxi', + input: 'algos_common_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_algos_take_helper = custom_target('algos_take_helper_pxi', + output: 'algos_take_helper.pxi', + input: 'algos_take_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_khash_primitive_helper = custom_target('khash_primitive_helper_pxi', + output: 'khash_for_primitive_helper.pxi', + input: 'khash_for_primitive_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_algos_common_helper_dep = declare_dependency(sources: _algos_common_helper) +_algos_take_helper_dep = declare_dependency(sources: _algos_take_helper) +_khash_primitive_helper_dep = declare_dependency(sources: _khash_primitive_helper) +# TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically +# The reason we can't build the pyx files inplace and copy to build dir is because +# the generated pxi files cannot be written to the source directory. +# (Meson only supports out of tree builds) +cython_sources_list = [ + # List of cython sources e.g. .pyx, .pxd & __init__.py + # Does NOT include .pxi.in + '__init__.py', + 'algos.pxd', + 'algos.pyx', + 'arrays.pxd', + 'arrays.pyx', + 'dtypes.pxd', + 'groupby.pyx', + 'hashing.pyx', + 'hashtable.pxd', + 'hashtable.pyx', + 'index.pyx', + 'indexing.pyx', + 'internals.pyx', + 'interval.pyx', + 'join.pyx', + 'khash.pxd', + 'lib.pyx', + 'lib.pxd', + 'missing.pxd', + 'missing.pyx', + 'ops.pyx', + 'ops_dispatch.pyx', + 'parsers.pyx', + 'properties.pyx', + 'reduction.pyx', + 'reshape.pyx', + 'sparse.pyx', + 'testing.pyx', + 'tslib.pyx', + 'util.pxd', + 'writers.pyx' +] +cython_sources = {} + +foreach source: cython_sources_list + source_pyx = configure_file( + input: source, + output: source, + copy: true + ) + cython_sources += {source: source_pyx} +endforeach + +subdir('tslibs') + +py.extension_module( + 'algos', + [_algos_common_helper, _algos_take_helper, _khash_primitive_helper, cython_sources['algos.pyx']], + include_directories: [inc_np, klib_include], + dependencies: [py_dep, _algos_common_helper_dep, _algos_take_helper_dep, _khash_primitive_helper_dep], + subdir: 'pandas/_libs' +) + +py.extension_module( + 'arrays', + cython_sources['arrays.pyx'], + include_directories: [inc_np], + dependencies: py_dep, + subdir: 'pandas/_libs' +) + +py.extension_module( + 'groupby', + cython_sources['groupby.pyx'], + include_directories: [inc_np], + dependencies: py_dep, + subdir: 'pandas/_libs' +) + +py.extension_module( + 'hashing', + cython_sources['hashing.pyx'], + include_directories: [inc_np], + dependencies: py_dep, + subdir: 'pandas/_libs' +) diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build new file mode 100644 index 0000000000000..d0030b09688d7 --- /dev/null +++ b/pandas/_libs/tslibs/meson.build @@ -0,0 +1,46 @@ +# TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically +tslibs_cython_sources_list = [ + # List of cython sources e.g. .pyx, .pxd & __init__.py + # Does NOT include .pxi.in + '__init__.py', + 'base.pxd', + 'base.pyx', + 'ccalendar.pxd', + 'ccalendar.pyx', + 'conversion.pxd', + 'conversion.pyx', + 'dtypes.pyx', + 'dtypes.pxd', + 'fields.pyx', + 'nattype.pyx', + 'nattype.pxd', + 'np_datetime.pxd', + 'np_datetime.pyx', + 'offsets.pxd', + 'offsets.pyx', + 'parsing.pxd', + 'parsing.pyx', + 'period.pyx', + 'period.pxd', + 'strptime.pyx', + 'timedeltas.pxd', + 'timedeltas.pyx', + 'timestamps.pyx', + 'timestamps.pxd', + 'timezones.pyx', + 'timezones.pxd', + 'tzconversion.pxd', + 'tzconversion.pyx', + 'util.pxd', + 'vectorized.pyx', +] +tslibs_cython_sources = {} + +foreach source: tslibs_cython_sources_list + source_pyx = configure_file( + input: source, + output: source, + copy: true + ) + tslibs_cython_sources += {source: source_pyx} +endforeach diff --git a/pandas/meson.build b/pandas/meson.build new file mode 100644 index 0000000000000..61de13e50f60e --- /dev/null +++ b/pandas/meson.build @@ -0,0 +1,21 @@ +incdir_numpy = run_command(py, + [ + '-c', + 'import os; os.chdir(".."); import numpy; print(numpy.get_include())' + ], + check: true +).stdout().strip() + +inc_np = include_directories(incdir_numpy) +klib_include = include_directories('_libs/src/klib') +configure_file( + input: '__init__.py', + output: '__init__.py', + copy: true +) +subdir('_libs') + +#install_subdir('_config', install_dir: meson.current_build_dir() + 'pandas/_config') +#py.install_sources('__init__.py', +# pure: false, +# subdir: 'pandas/__init__.py') diff --git a/pyproject.toml b/pyproject.toml index 0e2e41fba461c..1080d03e6a215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ # Minimum requirements for the build system to execute. # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ + "meson-python", "setuptools>=51.0.0", "wheel", "Cython>=0.29.24,<3", # Note: sync with setup.py, environment.yml and asv.conf.json @@ -9,7 +10,7 @@ requires = [ ] # uncomment to enable pep517 after versioneer problem is fixed. # https://github.com/python-versioneer/python-versioneer/issues/193 -# build-backend = "setuptools.build_meta" +build-backend = "mesonpy" [tool.black] target-version = ['py38', 'py39'] From 6219ad05ce63de56184c5f6332fb9acc7e3e5b16 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 14 Jul 2022 12:19:50 -0700 Subject: [PATCH 02/69] update --- .gitignore | 1 + pandas/_libs/meson.build | 12 ++++++++---- pandas/meson.build | 6 +++--- pyproject.toml | 3 +-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 07b1f056d511b..324e978e01e19 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ *.py[ocd] *.so .build_cache_dir +.mesonpy-native-file.ini MANIFEST # Python files # diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index fdc1c1d2887ee..a29f88902bc1c 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -79,7 +79,8 @@ py.extension_module( [_algos_common_helper, _algos_take_helper, _khash_primitive_helper, cython_sources['algos.pyx']], include_directories: [inc_np, klib_include], dependencies: [py_dep, _algos_common_helper_dep, _algos_take_helper_dep, _khash_primitive_helper_dep], - subdir: 'pandas/_libs' + subdir: 'pandas/_libs', + install: true ) py.extension_module( @@ -87,7 +88,8 @@ py.extension_module( cython_sources['arrays.pyx'], include_directories: [inc_np], dependencies: py_dep, - subdir: 'pandas/_libs' + subdir: 'pandas/_libs', + install: true ) py.extension_module( @@ -95,7 +97,8 @@ py.extension_module( cython_sources['groupby.pyx'], include_directories: [inc_np], dependencies: py_dep, - subdir: 'pandas/_libs' + subdir: 'pandas/_libs', + install: true ) py.extension_module( @@ -103,5 +106,6 @@ py.extension_module( cython_sources['hashing.pyx'], include_directories: [inc_np], dependencies: py_dep, - subdir: 'pandas/_libs' + subdir: 'pandas/_libs', + install: true ) diff --git a/pandas/meson.build b/pandas/meson.build index 61de13e50f60e..9fcf8711feb26 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -16,6 +16,6 @@ configure_file( subdir('_libs') #install_subdir('_config', install_dir: meson.current_build_dir() + 'pandas/_config') -#py.install_sources('__init__.py', -# pure: false, -# subdir: 'pandas/__init__.py') +py.install_sources('__init__.py', + pure: false, + subdir: 'pandas/__init__.py') diff --git a/pyproject.toml b/pyproject.toml index 1080d03e6a215..639bd4baab9c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,7 @@ # Minimum requirements for the build system to execute. # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ - "meson-python", - "setuptools>=51.0.0", + "git+https://github.com/FFY00/meson-python.git@main", "wheel", "Cython>=0.29.24,<3", # Note: sync with setup.py, environment.yml and asv.conf.json "oldest-supported-numpy>=0.10" From 33a0a7cea25851a71cc0834228631f93958157dd Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 14 Jul 2022 15:05:14 -0700 Subject: [PATCH 03/69] [skip ci] compile more & test something --- pandas/_libs/meson.build | 67 ++++++++++++++++++++++++++++++++++++++++ pandas/meson.build | 10 ++++-- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index a29f88902bc1c..3c76075b60b44 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -19,9 +19,49 @@ _khash_primitive_helper = custom_target('khash_primitive_helper_pxi', py, tempita, '@INPUT@', '-o', '@OUTDIR@' ] ) +_hashtable_class_helper = custom_target('hashtable_class_helper_pxi', + output: 'hashtable_class_helper.pxi', + input: 'hashtable_class_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_hashtable_func_helper = custom_target('hashtable_func_helper_pxi', + output: 'hashtable_func_helper.pxi', + input: 'hashtable_func_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_index_class_helper = custom_target('index_class_helper_pxi', + output: 'index_class_helper.pxi', + input: 'index_class_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_sparse_op_helper = custom_target('sparse_op_helper_pxi', + output: 'sparse_op_helper.pxi', + input: 'sparse_op_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_intervaltree_helper = custom_target('intervaltree_helper_pxi', + output: 'intervaltree.pxi', + input: 'intervaltree.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) _algos_common_helper_dep = declare_dependency(sources: _algos_common_helper) _algos_take_helper_dep = declare_dependency(sources: _algos_take_helper) _khash_primitive_helper_dep = declare_dependency(sources: _khash_primitive_helper) +_hashtable_class_helper_dep = declare_dependency(sources: _hashtable_class_helper) +_hashtable_func_helper_dep = declare_dependency(sources: _hashtable_func_helper) +_index_class_helper_dep = declare_dependency(sources: _index_class_helper) +_sparse_op_helper_dep = declare_dependency(sources: _sparse_op_helper) +_intervaltree_helper = declare_dependency(sources: _intervaltree_helper) # TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically # The reason we can't build the pyx files inplace and copy to build dir is because # the generated pxi files cannot be written to the source directory. @@ -109,3 +149,30 @@ py.extension_module( subdir: 'pandas/_libs', install: true ) + +py.extension_module( + 'hashtable', + [cython_sources['hashtable.pyx'], _khash_primitive_helper, _hashtable_class_helper, _hashtable_func_helper], + include_directories: [inc_np, klib_include], + dependencies: [py_dep, _khash_primitive_helper_dep, _hashtable_class_helper_dep, _hashtable_func_helper_dep], + subdir: 'pandas/_libs', + install: true +) + +#py.extension_module( +# 'index', +# [cython_sources['index.pyx'], _index_class_helper], +# include_directories: [inc_np, klib_include], +# dependencies: [py_dep, _index_class_helper_dep], +# subdir: 'pandas/_libs', +# install: true +#) + +py.extension_module( + 'indexing', + ['indexing.pyx'], + include_directories: [inc_np], + dependencies: py_dep, + subdir: 'pandas/_libs', + install: true +) diff --git a/pandas/meson.build b/pandas/meson.build index 9fcf8711feb26..05ffde8e95630 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -8,14 +8,18 @@ incdir_numpy = run_command(py, inc_np = include_directories(incdir_numpy) klib_include = include_directories('_libs/src/klib') +inc_datetime = include_directories('_libs/tslibs/src/datetime/') + configure_file( input: '__init__.py', output: '__init__.py', copy: true ) subdir('_libs') - -#install_subdir('_config', install_dir: meson.current_build_dir() + 'pandas/_config') +# TODO: this works but needs a patch in meson-python +# meson-python only looks in the meson-info/install_plan.json not in meson-info/installed.json +# However, subdirs are not listed in the install_plan only in installed.json +install_subdir('_config', install_dir: get_option('python.platlibdir') / 'pandas/_config') py.install_sources('__init__.py', pure: false, - subdir: 'pandas/__init__.py') + subdir: 'pandas') From 43ba9954909d262db80cb1d7f04291754507639d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 15 Jul 2022 21:12:54 -0700 Subject: [PATCH 04/69] compile more --- meson.build | 3 +- pandas/_libs/meson.build | 119 +++++++++++++++++++++++++- pandas/_libs/tslibs/meson.build | 147 ++++++++++++++++++++++++++++++++ pandas/meson.build | 2 +- 4 files changed, 268 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index 4a7d4d94b0624..8d39e4d2428c7 100644 --- a/meson.build +++ b/meson.build @@ -6,7 +6,8 @@ project( license: 'BSD-3', meson_version: '>=0.60', default_options: [ - 'buildtype=debugoptimized' + 'buildtype=debugoptimized', + 'c_std=c99' ] ) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 3c76075b60b44..5286191c02a7b 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -61,7 +61,7 @@ _hashtable_class_helper_dep = declare_dependency(sources: _hashtable_class_helpe _hashtable_func_helper_dep = declare_dependency(sources: _hashtable_func_helper) _index_class_helper_dep = declare_dependency(sources: _index_class_helper) _sparse_op_helper_dep = declare_dependency(sources: _sparse_op_helper) -_intervaltree_helper = declare_dependency(sources: _intervaltree_helper) +_intervaltree_helper_dep = declare_dependency(sources: _intervaltree_helper) # TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically # The reason we can't build the pyx files inplace and copy to build dir is because # the generated pxi files cannot be written to the source directory. @@ -112,6 +112,15 @@ foreach source: cython_sources_list cython_sources += {source: source_pyx} endforeach +#py.extension_module( +# 'parsing', +# ['tslibs/parsing.pyx'], +# include_directories: [inc_np, klib_include, 'src/parser', 'src', 'tslibs/src/datetime'], +# dependencies: [py_dep], +# subdir: 'pandas/_libs/tslibs', +# install: true +#) + subdir('tslibs') py.extension_module( @@ -176,3 +185,111 @@ py.extension_module( subdir: 'pandas/_libs', install: true ) + +py.extension_module( + 'internals', + ['internals.pyx'], + include_directories: [inc_np], + dependencies: py_dep, + subdir: 'pandas/_libs', + install: true +) + +#py.extension_module( +# 'interval', +# [cython_sources['interval.pyx'], _intervaltree_helper], +# include_directories: [inc_np, klib_include], +# dependencies: [py_dep, _intervaltree_helper_dep], +# subdir: 'pandas/_libs', +# install: true +#) + +py.extension_module( + 'join', + [cython_sources['join.pyx'], _khash_primitive_helper], + include_directories: [inc_np, klib_include], + dependencies: [py_dep, _khash_primitive_helper_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'lib', + ['lib.pyx', 'src/parser/tokenizer.c'], + include_directories: [inc_np, klib_include, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +#py.extension_module( +# 'parsers', +# [cython_sources['parsers.pyx'], _khash_primitive_helper, 'src/parser/tokenizer.c', 'src/parser/io.c'], +# include_directories: [inc_np, klib_include, 'src/parser', 'src'], +# dependencies: [py_dep, _khash_primitive_helper_dep], +# subdir: 'pandas/_libs', +# install: true +#) + +py.extension_module( + 'reduction', + ['reduction.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'ops', + ['ops.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'ops_dispatch', + ['ops_dispatch.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'properties', + ['properties.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'reshape', + ['reshape.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'sparse', + [cython_sources['sparse.pyx'], _sparse_op_helper], + include_directories: [inc_np], + dependencies: [py_dep, _sparse_op_helper_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'tslib', + ['tslib.pyx', 'tslibs/src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index d0030b09688d7..f36f6aed3cd44 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -44,3 +44,150 @@ foreach source: tslibs_cython_sources_list ) tslibs_cython_sources += {source: source_pyx} endforeach + +# TODO: build a shared library for np_datetime and np_datetime_strings +# and use lto? + +py.extension_module( + 'base', + ['base.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'ccalendar', + ['ccalendar.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'dtypes', + ['dtypes.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'conversion', + ['conversion.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'fields', + ['fields.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'nattype', + ['nattype.pyx'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'np_datetime', + ['np_datetime.pyx', 'src/datetime/np_datetime.c', 'src/datetime/np_datetime_strings.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'offsets', + ['offsets.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'parsing', + ['parsing.pyx', '../src/parser/tokenizer.c'], + include_directories: [inc_np, klib_include], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'period', + ['period.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'strptime', + ['strptime.pyx'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'timedeltas', + ['timedeltas.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'timestamps', + ['timestamps.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'timezones', + ['timezones.pyx'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'tzconversion', + ['tzconversion.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) + +py.extension_module( + 'vectorized', + ['tzconversion.pyx', 'src/datetime/np_datetime.c'], + include_directories: [inc_np, inc_datetime], + dependencies: [py_dep], + subdir: 'pandas/_libs/tslibs', + install: true +) diff --git a/pandas/meson.build b/pandas/meson.build index 05ffde8e95630..83d47937b8ce7 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -8,7 +8,7 @@ incdir_numpy = run_command(py, inc_np = include_directories(incdir_numpy) klib_include = include_directories('_libs/src/klib') -inc_datetime = include_directories('_libs/tslibs/src/datetime/') +inc_datetime = include_directories('_libs/tslibs') configure_file( input: '__init__.py', From 15e0385355f46d2a4b2107a3d0ad6adf6f21786d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 18 Jul 2022 07:27:55 -0700 Subject: [PATCH 05/69] build all extensions --- pandas/_libs/meson.build | 93 +++++++++++++++++++++------------ pandas/_libs/window/meson.build | 18 +++++++ pandas/io/sas/meson.build | 8 +++ pandas/meson.build | 1 + 4 files changed, 87 insertions(+), 33 deletions(-) create mode 100644 pandas/_libs/window/meson.build create mode 100644 pandas/io/sas/meson.build diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 5286191c02a7b..0a72e755d3f60 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -112,15 +112,6 @@ foreach source: cython_sources_list cython_sources += {source: source_pyx} endforeach -#py.extension_module( -# 'parsing', -# ['tslibs/parsing.pyx'], -# include_directories: [inc_np, klib_include, 'src/parser', 'src', 'tslibs/src/datetime'], -# dependencies: [py_dep], -# subdir: 'pandas/_libs/tslibs', -# install: true -#) - subdir('tslibs') py.extension_module( @@ -168,14 +159,14 @@ py.extension_module( install: true ) -#py.extension_module( -# 'index', -# [cython_sources['index.pyx'], _index_class_helper], -# include_directories: [inc_np, klib_include], -# dependencies: [py_dep, _index_class_helper_dep], -# subdir: 'pandas/_libs', -# install: true -#) +py.extension_module( + 'index', + [cython_sources['index.pyx'], _index_class_helper], + include_directories: [inc_np, klib_include, 'tslibs'], + dependencies: [py_dep, _index_class_helper_dep], + subdir: 'pandas/_libs', + install: true +) py.extension_module( 'indexing', @@ -195,14 +186,14 @@ py.extension_module( install: true ) -#py.extension_module( -# 'interval', -# [cython_sources['interval.pyx'], _intervaltree_helper], -# include_directories: [inc_np, klib_include], -# dependencies: [py_dep, _intervaltree_helper_dep], -# subdir: 'pandas/_libs', -# install: true -#) +py.extension_module( + 'interval', + [cython_sources['interval.pyx'], _intervaltree_helper], + include_directories: [inc_np, klib_include, 'tslibs'], + dependencies: [py_dep, _intervaltree_helper_dep], + subdir: 'pandas/_libs', + install: true +) py.extension_module( 'join', @@ -222,14 +213,30 @@ py.extension_module( install: true ) -#py.extension_module( -# 'parsers', -# [cython_sources['parsers.pyx'], _khash_primitive_helper, 'src/parser/tokenizer.c', 'src/parser/io.c'], -# include_directories: [inc_np, klib_include, 'src/parser', 'src'], -# dependencies: [py_dep, _khash_primitive_helper_dep], -# subdir: 'pandas/_libs', -# install: true -#) +py.extension_module( + 'parsers', + [cython_sources['parsers.pyx'], 'src/parser/tokenizer.c', 'src/parser/io.c'], + include_directories: [inc_np, klib_include, 'src'], + dependencies: [py_dep, _khash_primitive_helper_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'json', + ['src/ujson/python/ujson.c', + 'src/ujson/python/objToJSON.c', + 'src/ujson/python/date_conversions.c', + 'src/ujson/python/JSONtoObj.c', + 'src/ujson/lib/ultrajsonenc.c', + 'src/ujson/lib/ultrajsondec.c', + 'tslibs/src/datetime/np_datetime.c', + 'tslibs/src/datetime/np_datetime_strings.c'], + include_directories: [inc_np, inc_datetime, 'src/ujson/lib', 'src/ujson/python'], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) py.extension_module( 'reduction', @@ -293,3 +300,23 @@ py.extension_module( subdir: 'pandas/_libs', install: true ) + +py.extension_module( + 'testing', + ['testing.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +py.extension_module( + 'writers', + ['writers.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs', + install: true +) + +subdir('window') diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build new file mode 100644 index 0000000000000..7d7c34a57c6a6 --- /dev/null +++ b/pandas/_libs/window/meson.build @@ -0,0 +1,18 @@ +py.extension_module( + 'aggregations', + ['aggregations.pyx'], + include_directories: [inc_np, '../src'], + dependencies: [py_dep], + subdir: 'pandas/_libs/window', + override_options : ['cython_language=cpp'], + install: true +) + +py.extension_module( + 'indexers', + ['indexers.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs/window', + install: true +) diff --git a/pandas/io/sas/meson.build b/pandas/io/sas/meson.build new file mode 100644 index 0000000000000..d5dad5c2f1ec7 --- /dev/null +++ b/pandas/io/sas/meson.build @@ -0,0 +1,8 @@ +py.extension_module( + '_sas', + ['sas.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/io/sas', + install: true +) diff --git a/pandas/meson.build b/pandas/meson.build index 83d47937b8ce7..b3bd4f719aef8 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -16,6 +16,7 @@ configure_file( copy: true ) subdir('_libs') +subdir('io/sas') # TODO: this works but needs a patch in meson-python # meson-python only looks in the meson-info/install_plan.json not in meson-info/installed.json # However, subdirs are not listed in the install_plan only in installed.json From 0eeaa7df6644bb050b67194b8191be25725df907 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:59:47 -0700 Subject: [PATCH 06/69] clean a little --- pandas/_libs/tslibs/meson.build | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index f36f6aed3cd44..f423a452621f4 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -1,48 +1,30 @@ # TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically -tslibs_cython_sources_list = [ +tslibs_pxd_sources_list = [ # List of cython sources e.g. .pyx, .pxd & __init__.py # Does NOT include .pxi.in '__init__.py', 'base.pxd', - 'base.pyx', 'ccalendar.pxd', - 'ccalendar.pyx', 'conversion.pxd', - 'conversion.pyx', - 'dtypes.pyx', 'dtypes.pxd', - 'fields.pyx', - 'nattype.pyx', 'nattype.pxd', 'np_datetime.pxd', - 'np_datetime.pyx', 'offsets.pxd', - 'offsets.pyx', 'parsing.pxd', - 'parsing.pyx', - 'period.pyx', 'period.pxd', - 'strptime.pyx', 'timedeltas.pxd', - 'timedeltas.pyx', - 'timestamps.pyx', 'timestamps.pxd', - 'timezones.pyx', 'timezones.pxd', 'tzconversion.pxd', - 'tzconversion.pyx', 'util.pxd', - 'vectorized.pyx', ] -tslibs_cython_sources = {} -foreach source: tslibs_cython_sources_list - source_pyx = configure_file( +foreach source: tslibs_pxd_sources_list + source_pxd = configure_file( input: source, output: source, copy: true ) - tslibs_cython_sources += {source: source_pyx} endforeach # TODO: build a shared library for np_datetime and np_datetime_strings From 7c2c132dbf05db035dcd0407ad8548f7dccb6c7e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:42:06 -0700 Subject: [PATCH 07/69] clean more --- pandas/_libs/meson.build | 21 +++++++-------------- pandas/_libs/tslibs/meson.build | 4 ++++ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 0a72e755d3f60..21ed8d44588aa 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -73,10 +73,7 @@ cython_sources_list = [ 'algos.pxd', 'algos.pyx', 'arrays.pxd', - 'arrays.pyx', 'dtypes.pxd', - 'groupby.pyx', - 'hashing.pyx', 'hashtable.pxd', 'hashtable.pyx', 'index.pyx', @@ -85,21 +82,13 @@ cython_sources_list = [ 'interval.pyx', 'join.pyx', 'khash.pxd', - 'lib.pyx', 'lib.pxd', 'missing.pxd', - 'missing.pyx', - 'ops.pyx', - 'ops_dispatch.pyx', 'parsers.pyx', - 'properties.pyx', - 'reduction.pyx', - 'reshape.pyx', 'sparse.pyx', 'testing.pyx', 'tslib.pyx', 'util.pxd', - 'writers.pyx' ] cython_sources = {} @@ -125,7 +114,7 @@ py.extension_module( py.extension_module( 'arrays', - cython_sources['arrays.pyx'], + 'arrays.pyx', include_directories: [inc_np], dependencies: py_dep, subdir: 'pandas/_libs', @@ -134,7 +123,7 @@ py.extension_module( py.extension_module( 'groupby', - cython_sources['groupby.pyx'], + 'groupby.pyx', include_directories: [inc_np], dependencies: py_dep, subdir: 'pandas/_libs', @@ -143,7 +132,7 @@ py.extension_module( py.extension_module( 'hashing', - cython_sources['hashing.pyx'], + 'hashing.pyx', include_directories: [inc_np], dependencies: py_dep, subdir: 'pandas/_libs', @@ -319,4 +308,8 @@ py.extension_module( install: true ) +py.install_sources('__init__.py', + pure: false, + subdir: 'pandas/_libs') + subdir('window') diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index f423a452621f4..0591ff4517ac1 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -173,3 +173,7 @@ py.extension_module( subdir: 'pandas/_libs/tslibs', install: true ) + +py.install_sources('__init__.py', + pure: false, + subdir: 'pandas/_libs/tslibs') From 3df0963e1ddfb7894c937282d87a2d0b9156a024 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 24 Jul 2022 11:48:45 -0700 Subject: [PATCH 08/69] super mega cleanup --- generate_pxi.py | 7 - meson.build | 2 +- pandas/_libs/meson.build | 270 ++++++++------------------------ pandas/_libs/tslibs/meson.build | 188 +++++----------------- 4 files changed, 107 insertions(+), 360 deletions(-) diff --git a/generate_pxi.py b/generate_pxi.py index d142a0e816d42..3462b97aefcbf 100644 --- a/generate_pxi.py +++ b/generate_pxi.py @@ -5,13 +5,6 @@ def process_tempita(pxifile, outfile): - if ( - os.path.exists(outfile) - and os.stat(pxifile).st_mtime < os.stat(outfile).st_mtime - ): - # if .pxi.in is not updated, no need to output .pxi - return - with open(pxifile) as f: tmpl = f.read() pyxcontent = Tempita.sub(tmpl) diff --git a/meson.build b/meson.build index 8d39e4d2428c7..f82ca443a7213 100644 --- a/meson.build +++ b/meson.build @@ -4,7 +4,7 @@ project( 'c', 'cpp', 'cython', version: '1.5.0.dev0', license: 'BSD-3', - meson_version: '>=0.60', + meson_version: '>=0.63', default_options: [ 'buildtype=debugoptimized', 'c_std=c99' diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 21ed8d44588aa..f092871923e4d 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -54,8 +54,9 @@ _intervaltree_helper = custom_target('intervaltree_helper_pxi', py, tempita, '@INPUT@', '-o', '@OUTDIR@' ] ) -_algos_common_helper_dep = declare_dependency(sources: _algos_common_helper) -_algos_take_helper_dep = declare_dependency(sources: _algos_take_helper) +#_algos_common_helper_dep = declare_dependency(sources: _algos_common_helper) +#_algos_take_helper_dep = declare_dependency(sources: _algos_take_helper) +_algos_helper_dep = declare_dependency(sources: [_algos_common_helper, _algos_take_helper]) _khash_primitive_helper_dep = declare_dependency(sources: _khash_primitive_helper) _hashtable_class_helper_dep = declare_dependency(sources: _hashtable_class_helper) _hashtable_func_helper_dep = declare_dependency(sources: _hashtable_func_helper) @@ -103,210 +104,67 @@ endforeach subdir('tslibs') -py.extension_module( - 'algos', - [_algos_common_helper, _algos_take_helper, _khash_primitive_helper, cython_sources['algos.pyx']], - include_directories: [inc_np, klib_include], - dependencies: [py_dep, _algos_common_helper_dep, _algos_take_helper_dep, _khash_primitive_helper_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'arrays', - 'arrays.pyx', - include_directories: [inc_np], - dependencies: py_dep, - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'groupby', - 'groupby.pyx', - include_directories: [inc_np], - dependencies: py_dep, - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'hashing', - 'hashing.pyx', - include_directories: [inc_np], - dependencies: py_dep, - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'hashtable', - [cython_sources['hashtable.pyx'], _khash_primitive_helper, _hashtable_class_helper, _hashtable_func_helper], - include_directories: [inc_np, klib_include], - dependencies: [py_dep, _khash_primitive_helper_dep, _hashtable_class_helper_dep, _hashtable_func_helper_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'index', - [cython_sources['index.pyx'], _index_class_helper], - include_directories: [inc_np, klib_include, 'tslibs'], - dependencies: [py_dep, _index_class_helper_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'indexing', - ['indexing.pyx'], - include_directories: [inc_np], - dependencies: py_dep, - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'internals', - ['internals.pyx'], - include_directories: [inc_np], - dependencies: py_dep, - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'interval', - [cython_sources['interval.pyx'], _intervaltree_helper], - include_directories: [inc_np, klib_include, 'tslibs'], - dependencies: [py_dep, _intervaltree_helper_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'join', - [cython_sources['join.pyx'], _khash_primitive_helper], - include_directories: [inc_np, klib_include], - dependencies: [py_dep, _khash_primitive_helper_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'lib', - ['lib.pyx', 'src/parser/tokenizer.c'], - include_directories: [inc_np, klib_include, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'parsers', - [cython_sources['parsers.pyx'], 'src/parser/tokenizer.c', 'src/parser/io.c'], - include_directories: [inc_np, klib_include, 'src'], - dependencies: [py_dep, _khash_primitive_helper_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'json', - ['src/ujson/python/ujson.c', - 'src/ujson/python/objToJSON.c', - 'src/ujson/python/date_conversions.c', - 'src/ujson/python/JSONtoObj.c', - 'src/ujson/lib/ultrajsonenc.c', - 'src/ujson/lib/ultrajsondec.c', - 'tslibs/src/datetime/np_datetime.c', - 'tslibs/src/datetime/np_datetime_strings.c'], - include_directories: [inc_np, inc_datetime, 'src/ujson/lib', 'src/ujson/python'], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'reduction', - ['reduction.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'ops', - ['ops.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'ops_dispatch', - ['ops_dispatch.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'properties', - ['properties.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'reshape', - ['reshape.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'sparse', - [cython_sources['sparse.pyx'], _sparse_op_helper], - include_directories: [inc_np], - dependencies: [py_dep, _sparse_op_helper_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'tslib', - ['tslib.pyx', 'tslibs/src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'testing', - ['testing.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) - -py.extension_module( - 'writers', - ['writers.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs', - install: true -) +libs_sources = { + # Dict of extension name -> dict of {sources, include_dirs, and deps} + # numpy include dir is implicitly included + 'algos': {'sources': [_algos_common_helper, _algos_take_helper, _khash_primitive_helper, cython_sources['algos.pyx']], + 'include_dirs': klib_include, + 'deps': [_algos_helper_dep, _khash_primitive_helper_dep]}, + 'arrays': {'sources': ['arrays.pyx']}, + 'groupby': {'sources': ['groupby.pyx']}, + 'hashing': {'sources': ['hashing.pyx']}, + 'hashtable': {'sources': [cython_sources['hashtable.pyx'], _khash_primitive_helper, _hashtable_class_helper, _hashtable_func_helper], + 'include_dirs': klib_include, + 'deps': [_khash_primitive_helper_dep, _hashtable_class_helper_dep, _hashtable_func_helper_dep]}, + 'index': {'sources': [cython_sources['index.pyx'], _index_class_helper], + 'include_dirs': [klib_include, 'tslibs'], + 'deps': _index_class_helper_dep}, + 'indexing': {'sources': ['indexing.pyx']}, + 'internals': {'sources': ['internals.pyx']}, + 'interval': {'sources': [cython_sources['interval.pyx'], _intervaltree_helper], + 'include_dirs': [klib_include, 'tslibs'], + 'deps': _intervaltree_helper_dep}, + 'join': {'sources': [cython_sources['join.pyx'], _khash_primitive_helper], + 'include_dirs': klib_include, + 'deps': _khash_primitive_helper_dep}, + 'lib': {'sources': ['lib.pyx', 'src/parser/tokenizer.c'], + 'include_dirs': [klib_include, inc_datetime]}, + 'parsers': {'sources': [cython_sources['parsers.pyx'], 'src/parser/tokenizer.c', 'src/parser/io.c'], + 'include_dirs': [klib_include, 'src'], + 'deps': _khash_primitive_helper_dep}, + 'json': {'sources': ['src/ujson/python/ujson.c', + 'src/ujson/python/objToJSON.c', + 'src/ujson/python/date_conversions.c', + 'src/ujson/python/JSONtoObj.c', + 'src/ujson/lib/ultrajsonenc.c', + 'src/ujson/lib/ultrajsondec.c', + 'tslibs/src/datetime/np_datetime.c', + 'tslibs/src/datetime/np_datetime_strings.c'], + 'include_dirs': [inc_datetime, 'src/ujson/lib', 'src/ujson/python']}, + 'reduction': {'sources': ['reduction.pyx']}, + 'ops': {'sources': ['ops.pyx']}, + 'ops_dispatch': {'sources': ['ops_dispatch.pyx']}, + 'properties': {'sources': ['properties.pyx']}, + 'reshape': {'sources': ['reshape.pyx']}, + 'sparse': {'sources': [cython_sources['sparse.pyx'], _sparse_op_helper], + 'deps': _sparse_op_helper_dep}, + 'tslib': {'sources': ['tslib.pyx', 'tslibs/src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'testing': {'sources': ['testing.pyx']}, + 'writers': {'sources': ['writers.pyx']} +} + + +foreach ext_name, ext_dict : libs_sources + py.extension_module( + ext_name, + ext_dict.get('sources'), + include_directories: [inc_np] + ext_dict.get('include_dirs', ''), + dependencies: ext_dict.get('deps', ''), + subdir: 'pandas/_libs/tslibs', + install: true + ) +endforeach py.install_sources('__init__.py', pure: false, diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index 0591ff4517ac1..4899df476c8b4 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -27,152 +27,48 @@ foreach source: tslibs_pxd_sources_list ) endforeach -# TODO: build a shared library for np_datetime and np_datetime_strings -# and use lto? - -py.extension_module( - 'base', - ['base.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'ccalendar', - ['ccalendar.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'dtypes', - ['dtypes.pyx'], - include_directories: [inc_np], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'conversion', - ['conversion.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'fields', - ['fields.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'nattype', - ['nattype.pyx'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'np_datetime', - ['np_datetime.pyx', 'src/datetime/np_datetime.c', 'src/datetime/np_datetime_strings.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'offsets', - ['offsets.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'parsing', - ['parsing.pyx', '../src/parser/tokenizer.c'], - include_directories: [inc_np, klib_include], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'period', - ['period.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'strptime', - ['strptime.pyx'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'timedeltas', - ['timedeltas.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'timestamps', - ['timestamps.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'timezones', - ['timezones.pyx'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'tzconversion', - ['tzconversion.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) - -py.extension_module( - 'vectorized', - ['tzconversion.pyx', 'src/datetime/np_datetime.c'], - include_directories: [inc_np, inc_datetime], - dependencies: [py_dep], - subdir: 'pandas/_libs/tslibs', - install: true -) +tslibs_sources = { + # Dict of extension name -> dict of {sources, include_dirs, and deps} + # numpy include dir is implicitly included + 'base': {'sources': ['base.pyx']}, + 'ccalendar': {'sources': ['ccalendar.pyx']}, + 'dtypes': {'sources': ['dtypes.pyx']}, + 'conversion': {'sources': ['conversion.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'fields': {'sources': ['fields.pyx', 'src/datetime/np_datetime.c']}, + 'nattype': {'sources': ['nattype.pyx']}, + 'np_datetime': {'sources': ['np_datetime.pyx', 'src/datetime/np_datetime.c', 'src/datetime/np_datetime_strings.c'], + 'include_dirs': inc_datetime}, + 'offsets': {'sources': ['offsets.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'parsing': {'sources': ['parsing.pyx', '../src/parser/tokenizer.c'], + 'include_dirs': klib_include}, + 'period': {'sources': ['period.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'strptime': {'sources': ['strptime.pyx'], + 'include_dirs': inc_datetime}, + 'timedeltas': {'sources': ['timedeltas.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'timestamps': {'sources': ['timestamps.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'timezones': {'sources': ['timezones.pyx'], + 'include_dirs': inc_datetime}, + 'tzconversion': {'sources': ['tzconversion.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'vectorized': {'sources': ['vectorized.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime} +} + +foreach ext_name, ext_dict : tslibs_sources + py.extension_module( + ext_name, + ext_dict.get('sources'), + include_directories: [inc_np] + ext_dict.get('include_dirs', ''), + dependencies: ext_dict.get('deps', ''), + subdir: 'pandas/_libs/tslibs', + install: true + ) +endforeach py.install_sources('__init__.py', pure: false, From e70bfc69af4b274024783eed8fe2e9d7efb277f3 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 25 Jul 2022 14:04:21 -0700 Subject: [PATCH 09/69] remove deps usage & maybe workaround bug --- pandas/_libs/meson.build | 46 +++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index f092871923e4d..10aaa7e7c4349 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -1,13 +1,20 @@ -_algos_common_helper = custom_target('algos_common_helper_pxi', - output: 'algos_common_helper.pxi', - input: 'algos_common_helper.pxi.in', - command: [ - py, tempita, '@INPUT@', '-o', '@OUTDIR@' - ] -) _algos_take_helper = custom_target('algos_take_helper_pxi', output: 'algos_take_helper.pxi', input: 'algos_take_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ], + # TODO: remove these two below lines + # Weird bug in meson that only repros on my potato computer + # (possibly b/c of low number of threads)? + # The first custom_target is never built for some reason + # so algos.pyx will error out later in the build. + build_by_default: true, + build_always_stale: true +) +_algos_common_helper = custom_target('algos_common_helper_pxi', + output: 'algos_common_helper.pxi', + input: 'algos_common_helper.pxi.in', command: [ py, tempita, '@INPUT@', '-o', '@OUTDIR@' ] @@ -54,15 +61,7 @@ _intervaltree_helper = custom_target('intervaltree_helper_pxi', py, tempita, '@INPUT@', '-o', '@OUTDIR@' ] ) -#_algos_common_helper_dep = declare_dependency(sources: _algos_common_helper) -#_algos_take_helper_dep = declare_dependency(sources: _algos_take_helper) -_algos_helper_dep = declare_dependency(sources: [_algos_common_helper, _algos_take_helper]) _khash_primitive_helper_dep = declare_dependency(sources: _khash_primitive_helper) -_hashtable_class_helper_dep = declare_dependency(sources: _hashtable_class_helper) -_hashtable_func_helper_dep = declare_dependency(sources: _hashtable_func_helper) -_index_class_helper_dep = declare_dependency(sources: _index_class_helper) -_sparse_op_helper_dep = declare_dependency(sources: _sparse_op_helper) -_intervaltree_helper_dep = declare_dependency(sources: _intervaltree_helper) # TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically # The reason we can't build the pyx files inplace and copy to build dir is because # the generated pxi files cannot be written to the source directory. @@ -107,23 +106,19 @@ subdir('tslibs') libs_sources = { # Dict of extension name -> dict of {sources, include_dirs, and deps} # numpy include dir is implicitly included - 'algos': {'sources': [_algos_common_helper, _algos_take_helper, _khash_primitive_helper, cython_sources['algos.pyx']], - 'include_dirs': klib_include, - 'deps': [_algos_helper_dep, _khash_primitive_helper_dep]}, + 'algos': {'sources': [cython_sources['algos.pyx'], _algos_common_helper, _algos_take_helper, _khash_primitive_helper], + 'include_dirs': klib_include}, 'arrays': {'sources': ['arrays.pyx']}, 'groupby': {'sources': ['groupby.pyx']}, 'hashing': {'sources': ['hashing.pyx']}, 'hashtable': {'sources': [cython_sources['hashtable.pyx'], _khash_primitive_helper, _hashtable_class_helper, _hashtable_func_helper], - 'include_dirs': klib_include, - 'deps': [_khash_primitive_helper_dep, _hashtable_class_helper_dep, _hashtable_func_helper_dep]}, + 'include_dirs': klib_include}, 'index': {'sources': [cython_sources['index.pyx'], _index_class_helper], - 'include_dirs': [klib_include, 'tslibs'], - 'deps': _index_class_helper_dep}, + 'include_dirs': [klib_include, 'tslibs']}, 'indexing': {'sources': ['indexing.pyx']}, 'internals': {'sources': ['internals.pyx']}, 'interval': {'sources': [cython_sources['interval.pyx'], _intervaltree_helper], - 'include_dirs': [klib_include, 'tslibs'], - 'deps': _intervaltree_helper_dep}, + 'include_dirs': [klib_include, 'tslibs']}, 'join': {'sources': [cython_sources['join.pyx'], _khash_primitive_helper], 'include_dirs': klib_include, 'deps': _khash_primitive_helper_dep}, @@ -146,8 +141,7 @@ libs_sources = { 'ops_dispatch': {'sources': ['ops_dispatch.pyx']}, 'properties': {'sources': ['properties.pyx']}, 'reshape': {'sources': ['reshape.pyx']}, - 'sparse': {'sources': [cython_sources['sparse.pyx'], _sparse_op_helper], - 'deps': _sparse_op_helper_dep}, + 'sparse': {'sources': [cython_sources['sparse.pyx'], _sparse_op_helper]}, 'tslib': {'sources': ['tslib.pyx', 'tslibs/src/datetime/np_datetime.c'], 'include_dirs': inc_datetime}, 'testing': {'sources': ['testing.pyx']}, From 950f0a69745387f43e35b2ff11dd2fdab3f5ab88 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 25 Jul 2022 14:41:17 -0700 Subject: [PATCH 10/69] fix install paths --- pandas/_libs/meson.build | 2 +- pandas/io/meson.build | 36 ++++++++++++++++++++++++++++++++++++ pandas/meson.build | 35 ++++++++++++++++++++++++++++++----- 3 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 pandas/io/meson.build diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 10aaa7e7c4349..f6aa933c96c7b 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -155,7 +155,7 @@ foreach ext_name, ext_dict : libs_sources ext_dict.get('sources'), include_directories: [inc_np] + ext_dict.get('include_dirs', ''), dependencies: ext_dict.get('deps', ''), - subdir: 'pandas/_libs/tslibs', + subdir: 'pandas/_libs', install: true ) endforeach diff --git a/pandas/io/meson.build b/pandas/io/meson.build new file mode 100644 index 0000000000000..51621aafb5de6 --- /dev/null +++ b/pandas/io/meson.build @@ -0,0 +1,36 @@ +subdirs_list = [ + # exclude sas, since it contains extension modules + # and has its own meson.build + 'clipboard', + 'excel', + 'formats', + 'json', + 'parsers' +] +foreach subdir: subdirs_list + install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas/io' / subdir) +endforeach +top_level_py_list = [ + '__init__.py', + 'api.py', + 'clipboards.py', + 'common.py', + 'date_converters.py', + 'feather_format.py', + 'gbq.py', + 'html.py', + 'orc.py', + 'parquet.py', + 'pickle.py', + 'pytables.py', + 'spss.py', + 'sql.py', + 'stata.py', + 'xml.py' +] +foreach file: top_level_py_list + py.install_sources(file, + pure: false, + subdir: 'pandas') +endforeach +subdir('sas') diff --git a/pandas/meson.build b/pandas/meson.build index b3bd4f719aef8..cd0c8518584b2 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -16,11 +16,36 @@ configure_file( copy: true ) subdir('_libs') -subdir('io/sas') +subdir('io') # TODO: this works but needs a patch in meson-python # meson-python only looks in the meson-info/install_plan.json not in meson-info/installed.json # However, subdirs are not listed in the install_plan only in installed.json -install_subdir('_config', install_dir: get_option('python.platlibdir') / 'pandas/_config') -py.install_sources('__init__.py', - pure: false, - subdir: 'pandas') +subdirs_list = [ + '_config', + '_libs', + '_testing', + 'api', + 'arrays', + 'compat', + 'core', + 'errors', + 'plotting', + 'tests', + 'tseries', + 'util' +] +foreach subdir: subdirs_list + install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas' / subdir) +endforeach +top_level_py_list = [ + '__init__.py', + '_typing.py', + '_version.py', + 'conftest.py', + 'testing.py' +] +foreach file: top_level_py_list + py.install_sources(file, + pure: false, + subdir: 'pandas') +endforeach From 7e401dc31ca155fcc7b3128ee7a7b43989c71da3 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 25 Jul 2022 15:54:06 -0700 Subject: [PATCH 11/69] fix all oopsies --- pandas/_libs/meson.build | 2 ++ pandas/io/meson.build | 4 ++-- pandas/io/sas/meson.build | 12 ++++++++++++ pandas/meson.build | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index f6aa933c96c7b..659b337898a4e 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -124,6 +124,8 @@ libs_sources = { 'deps': _khash_primitive_helper_dep}, 'lib': {'sources': ['lib.pyx', 'src/parser/tokenizer.c'], 'include_dirs': [klib_include, inc_datetime]}, + 'missing': {'sources': ['missing.pyx'], + 'include_dirs': [inc_datetime]}, 'parsers': {'sources': [cython_sources['parsers.pyx'], 'src/parser/tokenizer.c', 'src/parser/io.c'], 'include_dirs': [klib_include, 'src'], 'deps': _khash_primitive_helper_dep}, diff --git a/pandas/io/meson.build b/pandas/io/meson.build index 51621aafb5de6..5e7e68dacec49 100644 --- a/pandas/io/meson.build +++ b/pandas/io/meson.build @@ -8,7 +8,7 @@ subdirs_list = [ 'parsers' ] foreach subdir: subdirs_list - install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas/io' / subdir) + install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas/io') endforeach top_level_py_list = [ '__init__.py', @@ -31,6 +31,6 @@ top_level_py_list = [ foreach file: top_level_py_list py.install_sources(file, pure: false, - subdir: 'pandas') + subdir: 'pandas/io') endforeach subdir('sas') diff --git a/pandas/io/sas/meson.build b/pandas/io/sas/meson.build index d5dad5c2f1ec7..9a8f86d9d2858 100644 --- a/pandas/io/sas/meson.build +++ b/pandas/io/sas/meson.build @@ -6,3 +6,15 @@ py.extension_module( subdir: 'pandas/io/sas', install: true ) +top_level_py_list = [ + '__init__.py', + 'sas7bdat.py', + 'sas_constants.py', + 'sas_xport.py', + 'sasreader.py' +] +foreach file: top_level_py_list + py.install_sources(file, + pure: false, + subdir: 'pandas/io/sas') +endforeach diff --git a/pandas/meson.build b/pandas/meson.build index cd0c8518584b2..0ada1d20b9d01 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -35,7 +35,7 @@ subdirs_list = [ 'util' ] foreach subdir: subdirs_list - install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas' / subdir) + install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas') endforeach top_level_py_list = [ '__init__.py', From d074fc14cee9de1bea91cadad32f9e1feed24da6 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 1 Sep 2022 11:13:55 -0400 Subject: [PATCH 12/69] build pandas on CI with meson --- .github/actions/build_pandas/action.yml | 13 ++++++------- pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 23bb988ef4d73..de92545118c97 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -12,11 +12,10 @@ runs: - name: Build Pandas run: | - python setup.py build_ext -j $N_JOBS - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index + python -m pip install . shell: bash -el {0} - env: - # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 - # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct - N_JOBS: 1 - #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }} +# env: +# # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 +# # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct +# N_JOBS: 1 +# #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }} diff --git a/pyproject.toml b/pyproject.toml index c5a36104513ad..d313f53b04395 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ # Minimum requirements for the build system to execute. # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ - "git+https://github.com/FFY00/meson-python.git@main", + "meson-python", "wheel", "Cython>=0.29.32,<3", # Note: sync with setup.py, environment.yml and asv.conf.json "oldest-supported-numpy>=0.10" From e0214a51c3c7b885cac266aff8da47157519170e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 2 Sep 2022 11:11:34 -0400 Subject: [PATCH 13/69] try something --- .github/actions/build_pandas/action.yml | 9 ++++++++- ci/deps/actions-310-numpydev.yaml | 1 + ci/deps/actions-310.yaml | 1 + ci/deps/actions-38-downstream_compat.yaml | 1 + ci/deps/actions-38-minimum_versions.yaml | 1 + ci/deps/actions-38.yaml | 1 + ci/deps/actions-39.yaml | 1 + ci/deps/actions-pypy-38.yaml | 1 + ci/deps/circle-38-arm64.yaml | 1 + pyproject.toml | 2 -- 10 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index de92545118c97..78fe137018ae1 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -12,7 +12,14 @@ runs: - name: Build Pandas run: | - python -m pip install . + # TODO: uncomment this after meson/meson-python bugs patched + #python -m pip install . -v + install_dir=$(python -c "import sysconfig; print(sysconfig.get_path('platlib'))") + meson setup builddir + cd builddir + meson configure '-Dprefix=$install_dir' + meson compile + meson install shell: bash -el {0} # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index ef20c2aa889b9..e7b3bfe9f7c9f 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -6,6 +6,7 @@ dependencies: # tools - pytest>=6.0 + - meson - pytest-cov - pytest-xdist>=1.31 - hypothesis>=5.5.3 diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index da3578e7191eb..bc12ede73f5d2 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -6,6 +6,7 @@ dependencies: # test dependencies - cython>=0.29.32 + - meson - pytest>=6.0 - pytest-cov - pytest-xdist>=1.31 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 29ad2669afbd2..3102aec682831 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -6,6 +6,7 @@ dependencies: - python=3.8 # test dependencies + - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index fd23080c2ab04..44999c0a87cba 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -7,6 +7,7 @@ dependencies: - python=3.8.0 # test dependencies + - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index b478b7c900425..9c200f41caefa 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -5,6 +5,7 @@ dependencies: - python=3.8 # test dependencies + - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index a12f36ba84cca..24f00a84a012f 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -6,6 +6,7 @@ dependencies: # test dependencies - cython>=0.29.32 + - meson - pytest>=6.0 - pytest-cov - pytest-xdist>=1.31 diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index e06b992acc191..6b50f402f3591 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -8,6 +8,7 @@ dependencies: - python=3.8[build=*_pypy] # TODO: use this once pypy3.8 is available # tools + - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 2b65ece881df7..bbf5bd587620b 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -5,6 +5,7 @@ dependencies: - python=3.8 # test dependencies + - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov diff --git a/pyproject.toml b/pyproject.toml index d313f53b04395..743aa4d2c26aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,6 @@ requires = [ "Cython>=0.29.32,<3", # Note: sync with setup.py, environment.yml and asv.conf.json "oldest-supported-numpy>=0.10" ] -# uncomment to enable pep517 after versioneer problem is fixed. -# https://github.com/python-versioneer/python-versioneer/issues/193 build-backend = "mesonpy" [tool.black] From 2751aee69d30cf04d9bfc0e49e7ca71cffc49f1d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 2 Sep 2022 11:25:12 -0400 Subject: [PATCH 14/69] try something --- .github/actions/build_pandas/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 78fe137018ae1..c2c4c0fa36850 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -17,7 +17,7 @@ runs: install_dir=$(python -c "import sysconfig; print(sysconfig.get_path('platlib'))") meson setup builddir cd builddir - meson configure '-Dprefix=$install_dir' + meson configure -Dprefix=$install_dir meson compile meson install shell: bash -el {0} From 5e04d4efae82e772223efdbfdacbddb5a8f8b70f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 19 Sep 2022 16:45:05 -0400 Subject: [PATCH 15/69] try building with pip again --- .github/actions/build_pandas/action.yml | 8 +------- ci/deps/actions-310-numpydev.yaml | 2 +- ci/deps/actions-310.yaml | 3 ++- ci/deps/actions-38-downstream_compat.yaml | 3 ++- ci/deps/actions-38-minimum_versions.yaml | 3 ++- ci/deps/actions-38.yaml | 3 ++- ci/deps/actions-39.yaml | 3 ++- ci/deps/actions-pypy-38.yaml | 3 ++- ci/deps/circle-38-arm64.yaml | 3 ++- environment.yml | 1 - meson.build | 2 +- requirements-dev.txt | 2 -- scripts/validate_min_versions_in_sync.py | 6 +++++- 13 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index c2c4c0fa36850..e66a8a1a16f5c 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,13 +13,7 @@ runs: - name: Build Pandas run: | # TODO: uncomment this after meson/meson-python bugs patched - #python -m pip install . -v - install_dir=$(python -c "import sysconfig; print(sysconfig.get_path('platlib'))") - meson setup builddir - cd builddir - meson configure -Dprefix=$install_dir - meson compile - meson install + python -m pip install . --no-build-isolation shell: bash -el {0} # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index e7b3bfe9f7c9f..b6bdfbacfe8ab 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -6,7 +6,6 @@ dependencies: # tools - pytest>=6.0 - - meson - pytest-cov - pytest-xdist>=1.31 - hypothesis>=5.5.3 @@ -18,6 +17,7 @@ dependencies: - pip - pip: - "cython" + - "git+https://github.com/mesonbuild/meson.git@master" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index bc12ede73f5d2..69014058520a5 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -6,7 +6,6 @@ dependencies: # test dependencies - cython>=0.29.32 - - meson - pytest>=6.0 - pytest-cov - pytest-xdist>=1.31 @@ -54,3 +53,5 @@ dependencies: - xlsxwriter - xlwt - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 3102aec682831..6f73bc7f81374 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -6,7 +6,6 @@ dependencies: - python=3.8 # test dependencies - - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov @@ -70,3 +69,5 @@ dependencies: - pyyaml - py - pytorch + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 44999c0a87cba..c1b813c3e9b66 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -7,7 +7,6 @@ dependencies: - python=3.8.0 # test dependencies - - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov @@ -56,3 +55,5 @@ dependencies: - xlsxwriter=1.4.3 - xlwt=1.3.0 - zstandard=0.15.2 + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 9c200f41caefa..fe1caae861cec 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -5,7 +5,6 @@ dependencies: - python=3.8 # test dependencies - - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov @@ -53,3 +52,5 @@ dependencies: - xlsxwriter - xlwt - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 24f00a84a012f..924bb99e35dee 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -6,7 +6,6 @@ dependencies: # test dependencies - cython>=0.29.32 - - meson - pytest>=6.0 - pytest-cov - pytest-xdist>=1.31 @@ -54,3 +53,5 @@ dependencies: - xlsxwriter - xlwt - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index 6b50f402f3591..a59fa2cec893e 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -8,7 +8,6 @@ dependencies: - python=3.8[build=*_pypy] # TODO: use this once pypy3.8 is available # tools - - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov @@ -20,3 +19,5 @@ dependencies: - numpy - python-dateutil - pytz + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index bbf5bd587620b..d4a9fdca42577 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -5,7 +5,6 @@ dependencies: - python=3.8 # test dependencies - - meson - cython>=0.29.32 - pytest>=6.0 - pytest-cov @@ -54,3 +53,5 @@ dependencies: - xlsxwriter - xlwt - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" diff --git a/environment.yml b/environment.yml index ee6982fc897a4..ebcac6e5f9639 100644 --- a/environment.yml +++ b/environment.yml @@ -108,7 +108,6 @@ dependencies: - types-python-dateutil - types-PyMySQL - types-pytz - - types-setuptools # documentation (jupyter notebooks) - nbconvert>=6.4.5 diff --git a/meson.build b/meson.build index f82ca443a7213..3b321ba1e27e5 100644 --- a/meson.build +++ b/meson.build @@ -12,7 +12,7 @@ project( ) py_mod = import('python') -py = py_mod.find_installation('python3') +py = py_mod.find_installation('python') py_dep = py.dependency() tempita = files('generate_pxi.py') diff --git a/requirements-dev.txt b/requirements-dev.txt index 682d61a6a15b2..900b697ca6dae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -85,7 +85,6 @@ sphinx-copybutton types-python-dateutil types-PyMySQL types-pytz -types-setuptools nbconvert>=6.4.5 nbsphinx pandoc @@ -101,4 +100,3 @@ requests jupyterlab >=3.4,<4 jupyterlite==0.1.0b10 sphinx-toggleprompt -setuptools>=51.0.0 diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index cb6a204094bf5..b48b1eaeb2719 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -58,7 +58,11 @@ def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, elif "# optional dependencies" in line: seen_optional = True elif seen_required and line.strip(): - package, version = line.strip().split("=") + try: + package, version = line.strip().split("=") + except ValueError: + # pip dependencies, just skip + continue package = package[2:] if package in EXCLUDE_DEPS: continue From 6edad44327d8e39fb3aa910a63db75468d4a5769 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 19 Sep 2022 16:52:25 -0400 Subject: [PATCH 16/69] install meson-python too --- ci/deps/actions-310-numpydev.yaml | 1 + ci/deps/actions-310.yaml | 1 + ci/deps/actions-38-downstream_compat.yaml | 1 + ci/deps/actions-38-minimum_versions.yaml | 1 + ci/deps/actions-38.yaml | 1 + ci/deps/actions-39.yaml | 1 + ci/deps/actions-pypy-38.yaml | 1 + ci/deps/circle-38-arm64.yaml | 1 + 8 files changed, 8 insertions(+) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index b6bdfbacfe8ab..93170f884ba4c 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -18,6 +18,7 @@ dependencies: - pip: - "cython" - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 69014058520a5..7d3bd81a7580a 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -55,3 +55,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 6f73bc7f81374..a46ed3aae4c2e 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -71,3 +71,4 @@ dependencies: - pytorch - pip: - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index c1b813c3e9b66..46d1fcea40cf4 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -57,3 +57,4 @@ dependencies: - zstandard=0.15.2 - pip: - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index fe1caae861cec..b3f61b71706b1 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -54,3 +54,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 924bb99e35dee..25f01e0ee5d4c 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -55,3 +55,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index a59fa2cec893e..af925ad65b2e4 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -21,3 +21,4 @@ dependencies: - pytz - pip: - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index d4a9fdca42577..336775c2cb1ff 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -55,3 +55,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" From 8f0adf2531983eec84b768dad4f8de927572f8b1 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 19 Sep 2022 17:20:01 -0400 Subject: [PATCH 17/69] use my meson-python fork --- ci/deps/actions-310-numpydev.yaml | 2 +- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38-minimum_versions.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/actions-pypy-38.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 93170f884ba4c..fc88c82d1eee1 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -18,7 +18,7 @@ dependencies: - pip: - "cython" - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 7d3bd81a7580a..db359b5cd09ba 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -55,4 +55,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index a46ed3aae4c2e..8f40a0d69253f 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -71,4 +71,4 @@ dependencies: - pytorch - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 46d1fcea40cf4..2bd9fcc3da336 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -57,4 +57,4 @@ dependencies: - zstandard=0.15.2 - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index b3f61b71706b1..0df681a92e1c8 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -54,4 +54,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 25f01e0ee5d4c..db712a4d12174 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -55,4 +55,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index af925ad65b2e4..dae063afe7b00 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -21,4 +21,4 @@ dependencies: - pytz - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 336775c2cb1ff..e5da9e39ab7e7 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -55,4 +55,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" From 47f21d7fe60a80bf5918d38b368381091d599aa6 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 20 Sep 2022 17:50:54 -0400 Subject: [PATCH 18/69] build in verbose mode --- .github/actions/build_pandas/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index e66a8a1a16f5c..5495c8410c949 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -12,8 +12,7 @@ runs: - name: Build Pandas run: | - # TODO: uncomment this after meson/meson-python bugs patched - python -m pip install . --no-build-isolation + python -m pip install . --no-build-isolation -v shell: bash -el {0} # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 From 132a689e797b00d88ac1d1556658e5b8027713f7 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 21 Sep 2022 10:48:57 -0400 Subject: [PATCH 19/69] update --- .github/actions/build_pandas/action.yml | 1 - ci/run_tests.sh | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 5495c8410c949..cc8c650e6f058 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,7 +13,6 @@ runs: - name: Build Pandas run: | python -m pip install . --no-build-isolation -v - shell: bash -el {0} # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 # # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct diff --git a/ci/run_tests.sh b/ci/run_tests.sh index e6de5caf955fc..51ff00f074dd8 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,5 +1,10 @@ #!/bin/bash -e +# Meson can't build inplace, we need to cd out of the pandas directory +# to test +cd .. +PYTEST_TARGET="pandas/$PYTEST_TARGET" + # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set) # https://github.com/pytest-dev/pytest/issues/920 # https://github.com/pytest-dev/pytest/issues/1075 @@ -33,7 +38,7 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" -if [[ "$PANDAS_DATA_MANAGER" != "array" && "$PYTEST_TARGET" == "pandas" ]]; then +if [[ "$PANDAS_DATA_MANAGER" != "array" && "$PYTEST_TARGET" == "pandas/pandas" ]]; then # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array # If we're targeting specific files, e.g. test_downstream.py, don't run. PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" From 89c4b6b978979bdd6f3279685db55d19e8ea791a Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 21 Sep 2022 10:55:45 -0400 Subject: [PATCH 20/69] add required shell --- .github/actions/build_pandas/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index cc8c650e6f058..977983377daf2 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,6 +13,7 @@ runs: - name: Build Pandas run: | python -m pip install . --no-build-isolation -v + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash -el {0}' }} # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 # # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct From 5f0a17534137f0cfb526d47b2f04305c81ca6218 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 22 Sep 2022 09:11:28 -0400 Subject: [PATCH 21/69] fix msvc detection? --- .github/actions/build_pandas/action.yml | 10 +++++++++- meson.build | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 977983377daf2..e78542e1ed8fd 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -11,9 +11,17 @@ runs: shell: bash -el {0} - name: Build Pandas + if : ${{ runner.os != 'Windows' }} run: | python -m pip install . --no-build-isolation -v - shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash -el {0}' }} + shell: bash -el {0} + + - name: Build Pandas (Windows) + if: ${{ runner.os == 'Windows' }} + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build" + python -m pip install . --no-build-isolation -v + shell: cmd.exe # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 # # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct diff --git a/meson.build b/meson.build index 3b321ba1e27e5..88ed7d15887d4 100644 --- a/meson.build +++ b/meson.build @@ -2,11 +2,11 @@ project( 'pandas', 'c', 'cpp', 'cython', - version: '1.5.0.dev0', + version: '1.6.0.dev0', license: 'BSD-3', meson_version: '>=0.63', default_options: [ - 'buildtype=debugoptimized', + 'buildtype=release', 'c_std=c99' ] ) From 48b02d425f4be53a1929e802134853db0bfbd133 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 22 Sep 2022 10:13:34 -0400 Subject: [PATCH 22/69] try something --- .github/actions/build_pandas/action.yml | 2 +- ci/run_tests.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index e78542e1ed8fd..ab02f97ffe8cb 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -21,7 +21,7 @@ runs: run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build" python -m pip install . --no-build-isolation -v - shell: cmd.exe + shell: cmd # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 # # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 51ff00f074dd8..27a89f3fb9e89 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,9 +1,5 @@ #!/bin/bash -e -# Meson can't build inplace, we need to cd out of the pandas directory -# to test -cd .. -PYTEST_TARGET="pandas/$PYTEST_TARGET" # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set) # https://github.com/pytest-dev/pytest/issues/920 @@ -29,7 +25,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile --import-mode=append $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 9b83eff600a1f9aca988f9f009b31ca003a83125 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 22 Sep 2022 10:30:38 -0400 Subject: [PATCH 23/69] maybe fix windows? --- .github/actions/build_pandas/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index ab02f97ffe8cb..b026dc1e5cc21 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -19,9 +19,9 @@ runs: - name: Build Pandas (Windows) if: ${{ runner.os == 'Windows' }} run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build" + call "C:\Program Files\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" python -m pip install . --no-build-isolation -v - shell: cmd + shell: cmd /C call {0} # env: # # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 # # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct From 039123fe63c05afa1f36fe24a242abd0077664ec Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 22 Sep 2022 11:00:37 -0400 Subject: [PATCH 24/69] maybe fix? --- .github/actions/build_pandas/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index b026dc1e5cc21..66c4e377c09b9 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -13,7 +13,7 @@ runs: - name: Build Pandas if : ${{ runner.os != 'Windows' }} run: | - python -m pip install . --no-build-isolation -v + pip install . --no-build-isolation -v shell: bash -el {0} - name: Build Pandas (Windows) From eb41668aa0b3a27a4243b7209d4f43d843e40aef Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 24 Sep 2022 08:34:14 -0400 Subject: [PATCH 25/69] disable caching for now --- .github/actions/setup-conda/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-conda/action.yml b/.github/actions/setup-conda/action.yml index 002d0020c2df1..fce0cf4054ba2 100644 --- a/.github/actions/setup-conda/action.yml +++ b/.github/actions/setup-conda/action.yml @@ -32,5 +32,5 @@ runs: channels: conda-forge channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }} condarc-file: ci/condarc.yml - cache-env: true - cache-downloads: true + cache-env: false + cache-downloads: false From 94ccae2e41fa157f0684d9f26932ad8b055f0ffd Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 24 Sep 2022 11:14:26 -0400 Subject: [PATCH 26/69] try to fix windows again --- .github/actions/build_pandas/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 66c4e377c09b9..649b90d81b79b 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -19,7 +19,9 @@ runs: - name: Build Pandas (Windows) if: ${{ runner.os == 'Windows' }} run: | - call "C:\Program Files\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" + conda init cmd.exe + refreshenv + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" python -m pip install . --no-build-isolation -v shell: cmd /C call {0} # env: From 4c0d93c354ff7d9ad49283cfcf76c0a65718a6a5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 24 Sep 2022 14:37:50 -0400 Subject: [PATCH 27/69] maybe fix? --- .github/actions/build_pandas/action.yml | 3 +-- ci/run_tests.sh | 2 +- pandas/_libs/tslibs/meson.build | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 649b90d81b79b..45406ff7c537f 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -19,8 +19,7 @@ runs: - name: Build Pandas (Windows) if: ${{ runner.os == 'Windows' }} run: | - conda init cmd.exe - refreshenv + call micromamba activate myenv call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" python -m pip install . --no-build-isolation -v shell: cmd /C call {0} diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 27a89f3fb9e89..0bee718affc7f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -25,7 +25,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile --import-mode=append $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile --import-mode=importlib $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index 4899df476c8b4..cf25d7025a6cd 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -45,13 +45,13 @@ tslibs_sources = { 'include_dirs': klib_include}, 'period': {'sources': ['period.pyx', 'src/datetime/np_datetime.c'], 'include_dirs': inc_datetime}, - 'strptime': {'sources': ['strptime.pyx'], + 'strptime': {'sources': ['strptime.pyx', 'src/datetime/np_datetime.c'], 'include_dirs': inc_datetime}, 'timedeltas': {'sources': ['timedeltas.pyx', 'src/datetime/np_datetime.c'], 'include_dirs': inc_datetime}, 'timestamps': {'sources': ['timestamps.pyx', 'src/datetime/np_datetime.c'], 'include_dirs': inc_datetime}, - 'timezones': {'sources': ['timezones.pyx'], + 'timezones': {'sources': ['timezones.pyx', 'src/datetime/np_datetime.c'], 'include_dirs': inc_datetime}, 'tzconversion': {'sources': ['tzconversion.pyx', 'src/datetime/np_datetime.c'], 'include_dirs': inc_datetime}, From 6efabb1931ae71edd952e47053ff9eb870a0b755 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 24 Sep 2022 16:15:50 -0400 Subject: [PATCH 28/69] hardcode version, fix windows? --- .github/actions/build_pandas/action.yml | 2 +- pandas/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 45406ff7c537f..1515957cac82d 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -19,7 +19,7 @@ runs: - name: Build Pandas (Windows) if: ${{ runner.os == 'Windows' }} run: | - call micromamba activate myenv + call micromamba activate test call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" python -m pip install . --no-build-isolation -v shell: cmd /C call {0} diff --git a/pandas/__init__.py b/pandas/__init__.py index 5016bde000c3b..7bee916ed45a5 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -179,7 +179,9 @@ from pandas._version import get_versions v = get_versions() -__version__ = v.get("closest-tag", v["version"]) +# __version__ = v.get("closest-tag", v["version"]) +# TODO: Don't hardcode this and actually figure out how to do this +__version__ = "1.6.0" __git_version__ = v.get("full-revisionid") del get_versions, v From 883fca7634e30aff001534444b4890223241b676 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 24 Sep 2022 17:11:16 -0400 Subject: [PATCH 29/69] use my fork of meson as well --- ci/deps/actions-310-numpydev.yaml | 2 +- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38-minimum_versions.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/actions-pypy-38.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index fc88c82d1eee1..3bba6c3e16a29 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -17,7 +17,7 @@ dependencies: - pip - pip: - "cython" - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index db359b5cd09ba..f368af36f67ea 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -54,5 +54,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 8f40a0d69253f..06dc15135c16a 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -70,5 +70,5 @@ dependencies: - py - pytorch - pip: - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 2bd9fcc3da336..02b4588f05aa7 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -56,5 +56,5 @@ dependencies: - xlwt=1.3.0 - zstandard=0.15.2 - pip: - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 0df681a92e1c8..3d3bc400f9b89 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -53,5 +53,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index db712a4d12174..a54ad5cf8494c 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -54,5 +54,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index dae063afe7b00..c18d6e45dc657 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -20,5 +20,5 @@ dependencies: - python-dateutil - pytz - pip: - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index e5da9e39ab7e7..9fb069dae34c7 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -54,5 +54,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" From b49f4c52c0ec2d707d23a00dedb9ae5735081b0b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 24 Sep 2022 20:15:05 -0400 Subject: [PATCH 30/69] maybe fix sas module export name? --- pandas/io/sas/meson.build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/io/sas/meson.build b/pandas/io/sas/meson.build index 9a8f86d9d2858..3ad2c1b332959 100644 --- a/pandas/io/sas/meson.build +++ b/pandas/io/sas/meson.build @@ -3,6 +3,9 @@ py.extension_module( ['sas.pyx'], include_directories: [inc_np], dependencies: [py_dep], + # The file is named sas.pyx but we want the + # extension module to be named _sas + cython_args: ['--module-name=pandas.io.sas._sas'], subdir: 'pandas/io/sas', install: true ) From da09ba00187789038ee03bcff7b890a26e3c0b1d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 3 Oct 2022 10:27:37 -0400 Subject: [PATCH 31/69] keep using versioneer --- generate_version.py | 28 ++++++++++++++++++++++++++++ meson.build | 11 +++++++++++ pandas/__init__.py | 17 +++++++++-------- 3 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 generate_version.py diff --git a/generate_version.py b/generate_version.py new file mode 100644 index 0000000000000..64d96142f1daa --- /dev/null +++ b/generate_version.py @@ -0,0 +1,28 @@ +import argparse + +import versioneer + + +def write_version_info(path): + with open(path, "w") as file: + file.write(f"__version__={versioneer.get_version()}\n") + file.write(f'__git__version__={versioneer.get_versions()["full-revisionid"]}\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-o", "--outfile", type=str, help="Path to write version info to" + ) + args = parser.parse_args() + + if not args.outfile.endswith(".py"): + raise ValueError( + f"Output file must be a Python file. " + f"Got: {args.outfile} as filename instead" + ) + + write_version_info(args.outfile) + + +main() diff --git a/meson.build b/meson.build index 88ed7d15887d4..6f40bb6529618 100644 --- a/meson.build +++ b/meson.build @@ -15,5 +15,16 @@ py_mod = import('python') py = py_mod.find_installation('python') py_dep = py.dependency() tempita = files('generate_pxi.py') +versioneer = files('generate_version.py') +custom_target('write_version_file', + output: '_version_meson.py', + command: [ + py, versioneer, '-o', '@OUTPUT@' + ], + build_by_default: true, + build_always_stale: true, + install: true, + install_dir: py.get_install_dir(pure: false) / 'pandas' +) subdir('pandas') diff --git a/pandas/__init__.py b/pandas/__init__.py index 7bee916ed45a5..a469c6f2e7230 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -176,14 +176,15 @@ from pandas.util._tester import test # use the closest tagged version if possible -from pandas._version import get_versions - -v = get_versions() -# __version__ = v.get("closest-tag", v["version"]) -# TODO: Don't hardcode this and actually figure out how to do this -__version__ = "1.6.0" -__git_version__ = v.get("full-revisionid") -del get_versions, v +try: + from pandas._version_meson import __version__, __git_version__ +except ImportError: + from pandas._version import get_versions + + v = get_versions() + __version__ = v.get("closest-tag", v["version"]) + __git_version__ = v.get("full-revisionid") + del get_versions, v # GH 27101 __deprecated_num_index_names = ["Float64Index", "Int64Index", "UInt64Index"] From 5b4dddcb72c4a911f2109d698f7ca253fdb6eda0 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 3 Oct 2022 10:41:54 -0400 Subject: [PATCH 32/69] fix version string --- generate_version.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/generate_version.py b/generate_version.py index 64d96142f1daa..8c4f5649a27fd 100644 --- a/generate_version.py +++ b/generate_version.py @@ -5,8 +5,10 @@ def write_version_info(path): with open(path, "w") as file: - file.write(f"__version__={versioneer.get_version()}\n") - file.write(f'__git__version__={versioneer.get_versions()["full-revisionid"]}\n') + file.write(f'__version__="{versioneer.get_version()}"\n') + file.write( + f'__git__version__="{versioneer.get_versions()["full-revisionid"]}"\n' + ) def main(): From 25178d5decde2688223d1f685f68f78e4cd1f548 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 3 Oct 2022 13:38:41 -0400 Subject: [PATCH 33/69] hopefully get versionning working, other than the sdist case --- generate_version.py | 2 +- meson.build | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/generate_version.py b/generate_version.py index 8c4f5649a27fd..b9377628de480 100644 --- a/generate_version.py +++ b/generate_version.py @@ -7,7 +7,7 @@ def write_version_info(path): with open(path, "w") as file: file.write(f'__version__="{versioneer.get_version()}"\n') file.write( - f'__git__version__="{versioneer.get_versions()["full-revisionid"]}"\n' + f'__git_version__="{versioneer.get_versions()["full-revisionid"]}"\n' ) diff --git a/meson.build b/meson.build index 6f40bb6529618..7d6ee95286b33 100644 --- a/meson.build +++ b/meson.build @@ -17,6 +17,7 @@ py_dep = py.dependency() tempita = files('generate_pxi.py') versioneer = files('generate_version.py') +# TODO: Think about how this would work for an sdist custom_target('write_version_file', output: '_version_meson.py', command: [ @@ -27,4 +28,5 @@ custom_target('write_version_file', install: true, install_dir: py.get_install_dir(pure: false) / 'pandas' ) + subdir('pandas') From a15586a4865d7e5683d122b756ff61a3c3d1bf91 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 4 Oct 2022 16:09:49 -0400 Subject: [PATCH 34/69] fix show_versions --- pandas/util/_print_versions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pandas/util/_print_versions.py b/pandas/util/_print_versions.py index 91d518d1ab496..20d63d66a0279 100644 --- a/pandas/util/_print_versions.py +++ b/pandas/util/_print_versions.py @@ -21,10 +21,15 @@ def _get_commit_hash() -> str | None: Use vendored versioneer code to get git hash, which handles git worktree correctly. """ - from pandas._version import get_versions + try: + from pandas._version_meson import __git_version__ - versions = get_versions() - return versions["full-revisionid"] + return __git_version__ + except ImportError: + from pandas._version import get_versions + + versions = get_versions() + return versions["full-revisionid"] def _get_sys_info() -> dict[str, JSONSerializable]: From 3c5ae9439299fa9ae54d5c7c49e40ef1b4cdadc4 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 13 Oct 2022 20:56:40 -0400 Subject: [PATCH 35/69] green? --- meson.build | 2 +- pandas/__init__.py | 5 ++++- pandas/tests/api/test_api.py | 8 +++++++- pandas/tests/io/test_compression.py | 12 ++++++++++-- pandas/tests/plotting/test_converter.py | 13 +++++++++++-- pandas/tests/test_downstream.py | 23 ++++++++++++++++++++--- 6 files changed, 53 insertions(+), 10 deletions(-) diff --git a/meson.build b/meson.build index 7d6ee95286b33..50169b1510349 100644 --- a/meson.build +++ b/meson.build @@ -19,7 +19,7 @@ versioneer = files('generate_version.py') # TODO: Think about how this would work for an sdist custom_target('write_version_file', - output: '_version_meson.py', + output: '__version_meson.py', command: [ py, versioneer, '-o', '@OUTPUT@' ], diff --git a/pandas/__init__.py b/pandas/__init__.py index a469c6f2e7230..27c0c022d8857 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -176,8 +176,11 @@ from pandas.util._tester import test # use the closest tagged version if possible +__built_with_meson = False try: - from pandas._version_meson import __version__, __git_version__ + from pandas.__version_meson import __version__, __git_version__ + + __built_with_meson = True except ImportError: from pandas._version import get_versions diff --git a/pandas/tests/api/test_api.py b/pandas/tests/api/test_api.py index c2db9698d0537..19e652ee30e57 100644 --- a/pandas/tests/api/test_api.py +++ b/pandas/tests/api/test_api.py @@ -316,8 +316,14 @@ def test_util_testing_deprecated_direct(self): assert "pandas.util.testing is deprecated" in str(m[0].message) assert "pandas.testing instead" in str(m[0].message) - def test_util_in_top_level(self): + def test_util_in_top_level(self, monkeypatch): # in a subprocess to avoid import caching issues + + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd.__built_with_meson: + monkeypatch.chdir("..") + out = subprocess.check_output( [ sys.executable, diff --git a/pandas/tests/io/test_compression.py b/pandas/tests/io/test_compression.py index 125d078ff39b1..0292bdae473e1 100644 --- a/pandas/tests/io/test_compression.py +++ b/pandas/tests/io/test_compression.py @@ -201,9 +201,13 @@ def test_gzip_reproducibility_file_object(): assert output == buffer.getvalue() -def test_with_missing_lzma(): +def test_with_missing_lzma(monkeypatch): """Tests if import pandas works when lzma is not present.""" # https://github.com/pandas-dev/pandas/issues/27575 + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd.__built_with_meson: + monkeypatch.chdir("..") code = textwrap.dedent( """\ import sys @@ -214,10 +218,14 @@ def test_with_missing_lzma(): subprocess.check_output([sys.executable, "-c", code], stderr=subprocess.PIPE) -def test_with_missing_lzma_runtime(): +def test_with_missing_lzma_runtime(monkeypatch): """Tests if RuntimeError is hit when calling lzma without having the module available. """ + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd.__built_with_meson: + monkeypatch.chdir("..") code = textwrap.dedent( """ import sys diff --git a/pandas/tests/plotting/test_converter.py b/pandas/tests/plotting/test_converter.py index 3ec8f4bd71c2b..7485bde0cecba 100644 --- a/pandas/tests/plotting/test_converter.py +++ b/pandas/tests/plotting/test_converter.py @@ -18,6 +18,7 @@ PeriodIndex, Series, Timestamp, + __built_with_meson, arrays, date_range, ) @@ -45,8 +46,12 @@ dates = pytest.importorskip("matplotlib.dates") -def test_registry_mpl_resets(): +def test_registry_mpl_resets(monkeypatch): # Check that Matplotlib converters are properly reset (see issue #27481) + # Can't import pandas from the test directory since its not + # built inplace with meson + if __built_with_meson: + monkeypatch.chdir("..") code = ( "import matplotlib.units as units; " "import matplotlib.dates as mdates; " @@ -65,7 +70,11 @@ def test_timtetonum_accepts_unicode(): class TestRegistration: - def test_dont_register_by_default(self): + def test_dont_register_by_default(self, monkeypatch): + # Can't import pandas from the test directory since its not + # built inplace with meson + if __built_with_meson: + monkeypatch.chdir("..") # Run in subprocess to ensure a clean state code = ( "import matplotlib.units; " diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 119ffd8cfd5a1..46c316e34a444 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -133,13 +133,21 @@ def test_xarray_cftimeindex_nearest(): assert result == expected -def test_oo_optimizable(): +def test_oo_optimizable(monkeypatch): # GH 21071 + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd.__built_with_meson: + monkeypatch.chdir("..") subprocess.check_call([sys.executable, "-OO", "-c", "import pandas"]) -def test_oo_optimized_datetime_index_unpickle(): +def test_oo_optimized_datetime_index_unpickle(monkeypatch): # GH 42866 + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd.__built_with_meson: + monkeypatch.chdir("..") subprocess.check_call( [ sys.executable, @@ -270,7 +278,11 @@ def test_yaml_dump(df): tm.assert_frame_equal(df, loaded2) -def test_missing_required_dependency(): +def test_missing_required_dependency(monkeypatch): + # TODO: This test is basically disabled until we have + # editable installs in meson-python. Re-enable this when + # that happens. + # GH 23868 # To ensure proper isolation, we pass these flags # -S : disable site-packages @@ -283,6 +295,11 @@ def test_missing_required_dependency(): # We skip this test if pandas is installed as a site package. We first # import the package normally and check the path to the module before # executing the test which imports pandas with site packages disabled. + + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd.__built_with_meson: + monkeypatch.chdir("..") call = [pyexe, "-c", "import pandas;print(pandas.__file__)"] output = subprocess.check_output(call).decode() if "site-packages" in output: From c01409c2c3dbda7e88d97445dfa46dd5bdea5255 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 14 Oct 2022 07:45:20 -0400 Subject: [PATCH 36/69] try to fix coverage --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 0bee718affc7f..e6c9be39c17f9 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -14,7 +14,7 @@ if [[ "not network" == *"$PATTERN"* ]]; then fi if [[ "$COVERAGE" == "true" ]]; then - COVERAGE="-s --cov=pandas --cov-report=xml --cov-append" + COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=setup.cfg" else COVERAGE="" # We need to reset this for COVERAGE="false" case fi From a286e66113c7de377da032715033858df09a8622 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 14 Oct 2022 16:39:36 -0400 Subject: [PATCH 37/69] maybe fix? --- pandas/__init__.py | 9 ++++++--- pandas/tests/api/test_api.py | 6 +++--- pandas/tests/plotting/test_converter.py | 6 +++--- pandas/tests/test_downstream.py | 6 +++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pandas/__init__.py b/pandas/__init__.py index 27c0c022d8857..24c40370dbc15 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -176,11 +176,11 @@ from pandas.util._tester import test # use the closest tagged version if possible -__built_with_meson = False +_built_with_meson = False try: - from pandas.__version_meson import __version__, __git_version__ + from pandas._version_meson import __version__, __git_version__ - __built_with_meson = True + _built_with_meson = True except ImportError: from pandas._version import get_versions @@ -429,4 +429,7 @@ def __getattr__(name): "unique", "value_counts", "wide_to_long", + # TODO: remove this when meson becomes the default + # build system + "_built_with_meson", ] diff --git a/pandas/tests/api/test_api.py b/pandas/tests/api/test_api.py index 19e652ee30e57..858347f411254 100644 --- a/pandas/tests/api/test_api.py +++ b/pandas/tests/api/test_api.py @@ -29,7 +29,7 @@ def check(self, namespace, expected, ignored=None): class TestPDApi(Base): # these are optionally imported based on testing # & need to be ignored - ignored = ["tests", "locale", "conftest"] + ignored = ["tests", "locale", "conftest", "_version_meson"] # top-level sub-packages public_lib = [ @@ -43,7 +43,7 @@ class TestPDApi(Base): "io", "tseries", ] - private_lib = ["compat", "core", "pandas", "util"] + private_lib = ["compat", "core", "pandas", "util", "_built_with_meson"] # these are already deprecated; awaiting removal deprecated_modules: list[str] = ["np", "datetime"] @@ -321,7 +321,7 @@ def test_util_in_top_level(self, monkeypatch): # Can't import pandas from the test directory since its not # built inplace with meson - if pd.__built_with_meson: + if pd._built_with_meson: monkeypatch.chdir("..") out = subprocess.check_output( diff --git a/pandas/tests/plotting/test_converter.py b/pandas/tests/plotting/test_converter.py index 7485bde0cecba..f8ad051635e4c 100644 --- a/pandas/tests/plotting/test_converter.py +++ b/pandas/tests/plotting/test_converter.py @@ -18,7 +18,7 @@ PeriodIndex, Series, Timestamp, - __built_with_meson, + _built_with_meson, arrays, date_range, ) @@ -50,7 +50,7 @@ def test_registry_mpl_resets(monkeypatch): # Check that Matplotlib converters are properly reset (see issue #27481) # Can't import pandas from the test directory since its not # built inplace with meson - if __built_with_meson: + if _built_with_meson: monkeypatch.chdir("..") code = ( "import matplotlib.units as units; " @@ -73,7 +73,7 @@ class TestRegistration: def test_dont_register_by_default(self, monkeypatch): # Can't import pandas from the test directory since its not # built inplace with meson - if __built_with_meson: + if _built_with_meson: monkeypatch.chdir("..") # Run in subprocess to ensure a clean state code = ( diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 46c316e34a444..30f9b2aad5f9a 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -137,7 +137,7 @@ def test_oo_optimizable(monkeypatch): # GH 21071 # Can't import pandas from the test directory since its not # built inplace with meson - if pd.__built_with_meson: + if pd._built_with_meson: monkeypatch.chdir("..") subprocess.check_call([sys.executable, "-OO", "-c", "import pandas"]) @@ -146,7 +146,7 @@ def test_oo_optimized_datetime_index_unpickle(monkeypatch): # GH 42866 # Can't import pandas from the test directory since its not # built inplace with meson - if pd.__built_with_meson: + if pd._built_with_meson: monkeypatch.chdir("..") subprocess.check_call( [ @@ -298,7 +298,7 @@ def test_missing_required_dependency(monkeypatch): # Can't import pandas from the test directory since its not # built inplace with meson - if pd.__built_with_meson: + if pd._built_with_meson: monkeypatch.chdir("..") call = [pyexe, "-c", "import pandas;print(pandas.__file__)"] output = subprocess.check_output(call).decode() From e61ef35ef8579c20a72f274887b2377b12866b9e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 15 Oct 2022 07:37:54 -0400 Subject: [PATCH 38/69] fix version generation for sdists --- generate_version.py | 4 ++++ meson.build | 27 ++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/generate_version.py b/generate_version.py index b9377628de480..fbc78ab12429a 100644 --- a/generate_version.py +++ b/generate_version.py @@ -1,9 +1,13 @@ import argparse +import os import versioneer def write_version_info(path): + if os.environ.get("MESON_DIST_ROOT"): + # raise ValueError("dist root is", os.environ.get("MESON_DIST_ROOT")) + path = os.path.join(os.environ.get("MESON_DIST_ROOT"), path) with open(path, "w") as file: file.write(f'__version__="{versioneer.get_version()}"\n') file.write( diff --git a/meson.build b/meson.build index 50169b1510349..7eab7d492d2a8 100644 --- a/meson.build +++ b/meson.build @@ -12,21 +12,26 @@ project( ) py_mod = import('python') +fs = import('fs') py = py_mod.find_installation('python') py_dep = py.dependency() tempita = files('generate_pxi.py') versioneer = files('generate_version.py') -# TODO: Think about how this would work for an sdist -custom_target('write_version_file', - output: '__version_meson.py', - command: [ - py, versioneer, '-o', '@OUTPUT@' - ], - build_by_default: true, - build_always_stale: true, - install: true, - install_dir: py.get_install_dir(pure: false) / 'pandas' -) +if fs.exists('_version_meson.py') + py.install_sources('_version_meson.py', subdir: 'pandas') +else + custom_target('write_version_file', + output: '_version_meson.py', + command: [ + py, versioneer, '-o', '@OUTPUT@' + ], + build_by_default: true, + build_always_stale: true, + install: true, + install_dir: py.get_install_dir(pure: false) / 'pandas' + ) + meson.add_dist_script(py, versioneer, '-o', '_version_meson.py') +endif subdir('pandas') From 588527f428b803e6a87313772e20111ae8f55dbe Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 15 Oct 2022 09:17:49 -0400 Subject: [PATCH 39/69] Finally fix? --- pandas/__init__.py | 3 --- pandas/tests/api/test_api.py | 3 ++- pandas/tests/io/test_compression.py | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/__init__.py b/pandas/__init__.py index 24c40370dbc15..8ca970b2d28de 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -429,7 +429,4 @@ def __getattr__(name): "unique", "value_counts", "wide_to_long", - # TODO: remove this when meson becomes the default - # build system - "_built_with_meson", ] diff --git a/pandas/tests/api/test_api.py b/pandas/tests/api/test_api.py index 858347f411254..2e5d9cbe1a93f 100644 --- a/pandas/tests/api/test_api.py +++ b/pandas/tests/api/test_api.py @@ -195,8 +195,9 @@ class TestPDApi(Base): "_is_numpy_dev", "_testing", "_typing", - "_version", ] + if not pd._built_with_meson: + private_modules.append("_version") def test_api(self): diff --git a/pandas/tests/io/test_compression.py b/pandas/tests/io/test_compression.py index 0292bdae473e1..253277a22366b 100644 --- a/pandas/tests/io/test_compression.py +++ b/pandas/tests/io/test_compression.py @@ -206,7 +206,7 @@ def test_with_missing_lzma(monkeypatch): # https://github.com/pandas-dev/pandas/issues/27575 # Can't import pandas from the test directory since its not # built inplace with meson - if pd.__built_with_meson: + if pd._built_with_meson: monkeypatch.chdir("..") code = textwrap.dedent( """\ @@ -224,7 +224,7 @@ def test_with_missing_lzma_runtime(monkeypatch): """ # Can't import pandas from the test directory since its not # built inplace with meson - if pd.__built_with_meson: + if pd._built_with_meson: monkeypatch.chdir("..") code = textwrap.dedent( """ From d4da54b27d70971240b9a986ccaaeb44e06c606d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 18 Oct 2022 16:29:41 -0400 Subject: [PATCH 40/69] Update for sas byteswap module --- pandas/io/sas/meson.build | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pandas/io/sas/meson.build b/pandas/io/sas/meson.build index 3ad2c1b332959..172db6334734f 100644 --- a/pandas/io/sas/meson.build +++ b/pandas/io/sas/meson.build @@ -9,6 +9,17 @@ py.extension_module( subdir: 'pandas/io/sas', install: true ) +py.extension_module( + '_byteswap', + ['byteswap.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + # The file is named byteswap.pyx but we want the + # extension module to be named _byteswap + cython_args: ['--module-name=pandas.io.sas._byteswap'], + subdir: 'pandas/io/sas', + install: true +) top_level_py_list = [ '__init__.py', 'sas7bdat.py', From 15e2cd0f3274dcc6271b67a8757371c4d99a9e1b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 19 Oct 2022 13:49:27 -0400 Subject: [PATCH 41/69] Pull upstreamed changes --- .github/workflows/32-bit-linux.yml | 5 +++-- .github/workflows/python-dev.yml | 10 ++++++---- ci/deps/actions-310-numpydev.yaml | 4 ++-- ci/deps/actions-310.yaml | 4 ++-- ci/deps/actions-38-downstream_compat.yaml | 4 ++-- ci/deps/actions-38-minimum_versions.yaml | 4 ++-- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 4 ++-- ci/deps/actions-pypy-38.yaml | 4 ++-- ci/deps/circle-38-arm64.yaml | 4 ++-- environment.yml | 2 ++ requirements-dev.txt | 2 ++ 12 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index 8c9f0b594f321..971a52364e9d4 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -41,8 +41,9 @@ jobs: . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ - python setup.py build_ext -q -j1 && \ - python -m pip install --no-build-isolation --no-use-pep517 -e . && \ + pip install "git+https://github.com/mesonbuild/meson.git@master" && \ + pip install "git+https://github.com/FFY00/meson-python.git@main" && \ + python -m pip install --no-build-isolation -v . && \ python -m pip list && \ export PANDAS_CI=1 && \ pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index b725f6812bc3b..d214f3a26db27 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -77,14 +77,16 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy python -m pip install git+https://github.com/nedbat/coveragepy.git - python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python -m pip install python-dateutil pytz cython + # TODO: update when upstream releases fixes + python -m pip install "git+https://github.com/mesonbuild/meson.git@master" + python -m pip install "git+https://github.com/FFY00/meson-python.git@main" + python -m pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python -m pip list - # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs - name: Build Pandas run: | - python setup.py build_ext -q -j1 - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index + python -m pip install -v . --no-build-isolation --no-index - name: Build Version run: | diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 3bba6c3e16a29..93170f884ba4c 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -17,8 +17,8 @@ dependencies: - pip - pip: - "cython" - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index f368af36f67ea..7d3bd81a7580a 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -54,5 +54,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 06dc15135c16a..a46ed3aae4c2e 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -70,5 +70,5 @@ dependencies: - py - pytorch - pip: - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 02b4588f05aa7..46d1fcea40cf4 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -56,5 +56,5 @@ dependencies: - xlwt=1.3.0 - zstandard=0.15.2 - pip: - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 3d3bc400f9b89..0e6f3986048b8 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -54,4 +54,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index a54ad5cf8494c..25f01e0ee5d4c 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -54,5 +54,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index c18d6e45dc657..af925ad65b2e4 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -20,5 +20,5 @@ dependencies: - python-dateutil - pytz - pip: - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 9fb069dae34c7..336775c2cb1ff 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -54,5 +54,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" - - "git+https://github.com/lithomas1/meson-python.git@fix-install-subdirs" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/environment.yml b/environment.yml index 1c3073c3c7966..8ff4f84176a2c 100644 --- a/environment.yml +++ b/environment.yml @@ -130,3 +130,5 @@ dependencies: - pip: - jupyterlite==0.1.0b12 - sphinx-toggleprompt + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/requirements-dev.txt b/requirements-dev.txt index 8d3126cea4c5e..3a7742509515f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -100,3 +100,5 @@ requests jupyterlab >=3.4,<4 jupyterlite==0.1.0b12 sphinx-toggleprompt +git+https://github.com/mesonbuild/meson.git@master +git+https://github.com/FFY00/meson-python.git@main From 3983e1c1f178d324cafd17129ba7a711b2114042 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:43:46 -0400 Subject: [PATCH 42/69] Fix everything else? --- .github/workflows/32-bit-linux.yml | 2 +- .github/workflows/python-dev.yml | 3 ++- ci/deps/actions-38.yaml | 2 +- doc/make.py | 23 ++++++++++++++++++++--- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index 971a52364e9d4..26beba2637f06 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -46,7 +46,7 @@ jobs: python -m pip install --no-build-isolation -v . && \ python -m pip list && \ export PANDAS_CI=1 && \ - pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" + pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml --import-mode=importlib" - name: Publish test results for Python 3.8-32 bit full Linux uses: actions/upload-artifact@v3 diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index d214f3a26db27..fc5ed3abf5f73 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -90,7 +90,8 @@ jobs: - name: Build Version run: | - python -c "import pandas; pandas.show_versions();" + # Can't import pandas from the source directory + cd .. && python -c "import pandas; pandas.show_versions();" - name: Test uses: ./.github/actions/run-tests diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 0e6f3986048b8..b3f61b71706b1 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -53,5 +53,5 @@ dependencies: - xlwt - zstandard - pip: - - "git+https://github.com/lithomas1/meson.git@fix-ghost-pdb-target" + - "git+https://github.com/mesonbuild/meson.git@master" - "git+https://github.com/FFY00/meson-python.git@main" diff --git a/doc/make.py b/doc/make.py index c758c7fc84bbb..35abe228f7a85 100755 --- a/doc/make.py +++ b/doc/make.py @@ -354,9 +354,26 @@ def main(): # external libraries (namely Sphinx) to compile this module and resolve # the import of `python_path` correctly. The latter is used to resolve # the import within the module, injecting it into the global namespace - os.environ["PYTHONPATH"] = args.python_path - sys.path.insert(0, args.python_path) - globals()["pandas"] = importlib.import_module("pandas") + + # Check for pandas built by meson + loaded_meson_pandas = False + try: + import pandas as pd + + if not pd._built_with_meson: + # This is just a random pandas, so use the editable install of pandas + loaded_meson_pandas = False + del sys.module["pandas"] + del pd + else: + loaded_meson_pandas = True + except ImportError: + pass + + if not loaded_meson_pandas: + os.environ["PYTHONPATH"] = args.python_path + sys.path.insert(0, args.python_path) + globals()["pandas"] = importlib.import_module("pandas") # Set the matplotlib backend to the non-interactive Agg backend for all # child processes. From f3a941902e0e90cf79b9043c1bab8c769df2fc97 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 20 Oct 2022 07:23:38 -0400 Subject: [PATCH 43/69] fix circleci and maybe fix docs --- .circleci/setup_env.sh | 6 +----- doc/make.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.circleci/setup_env.sh b/.circleci/setup_env.sh index 52a8cab1cd2de..dc0ba20c190f5 100755 --- a/.circleci/setup_env.sh +++ b/.circleci/setup_env.sh @@ -54,11 +54,7 @@ if pip list | grep -q ^pandas; then pip uninstall -y pandas || true fi -echo "Build extensions" -# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs -python setup.py build_ext -q -j1 - echo "Install pandas" -python -m pip install --no-build-isolation --no-use-pep517 -e . +python -m pip install --no-build-isolation -v . echo "done" diff --git a/doc/make.py b/doc/make.py index 35abe228f7a85..d9c458734a254 100755 --- a/doc/make.py +++ b/doc/make.py @@ -362,11 +362,11 @@ def main(): if not pd._built_with_meson: # This is just a random pandas, so use the editable install of pandas - loaded_meson_pandas = False - del sys.module["pandas"] + del sys.modules["pandas"] del pd else: loaded_meson_pandas = True + globals()["pandas"] = pd except ImportError: pass From bfcb9d04c4bb67a74afd50cf1f279c30f0525ba8 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 22 Oct 2022 13:09:03 -0400 Subject: [PATCH 44/69] Update --- meson.build | 2 ++ pandas/_libs/meson.build | 6 +----- pandas/meson.build | 11 +++-------- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/meson.build b/meson.build index 7eab7d492d2a8..bc5bcf5b845f6 100644 --- a/meson.build +++ b/meson.build @@ -4,6 +4,8 @@ project( 'c', 'cpp', 'cython', version: '1.6.0.dev0', license: 'BSD-3', + # TODO: bump when meson 0.64.0 comes out, + # we are relying on 0.64.0 features meson_version: '>=0.63', default_options: [ 'buildtype=release', diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index 659b337898a4e..b0012fb76e57c 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -93,11 +93,7 @@ cython_sources_list = [ cython_sources = {} foreach source: cython_sources_list - source_pyx = configure_file( - input: source, - output: source, - copy: true - ) + source_pyx = fs.copyfile(source) cython_sources += {source: source_pyx} endforeach diff --git a/pandas/meson.build b/pandas/meson.build index 0ada1d20b9d01..8ffa524570815 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -10,16 +10,11 @@ inc_np = include_directories(incdir_numpy) klib_include = include_directories('_libs/src/klib') inc_datetime = include_directories('_libs/tslibs') -configure_file( - input: '__init__.py', - output: '__init__.py', - copy: true -) +fs.copyfile('__init__.py') + subdir('_libs') subdir('io') -# TODO: this works but needs a patch in meson-python -# meson-python only looks in the meson-info/install_plan.json not in meson-info/installed.json -# However, subdirs are not listed in the install_plan only in installed.json + subdirs_list = [ '_config', '_libs', From a62ee23e0b65938ea902eee8d85c5a395e291c75 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 2 Nov 2022 06:46:00 -0400 Subject: [PATCH 45/69] adjust build to account for deprecations --- pandas/io/meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/meson.build b/pandas/io/meson.build index 5e7e68dacec49..04d7585ebc274 100644 --- a/pandas/io/meson.build +++ b/pandas/io/meson.build @@ -15,7 +15,6 @@ top_level_py_list = [ 'api.py', 'clipboards.py', 'common.py', - 'date_converters.py', 'feather_format.py', 'gbq.py', 'html.py', From 0397b46bb1c08b4afddfdd256cf8b76d25ed41ab Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 2 Nov 2022 09:00:04 -0400 Subject: [PATCH 46/69] try to fix docs --- doc/make.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/make.py b/doc/make.py index d9c458734a254..7d199948aa5c3 100755 --- a/doc/make.py +++ b/doc/make.py @@ -27,6 +27,7 @@ SOURCE_PATH = os.path.join(DOC_PATH, "source") BUILD_PATH = os.path.join(DOC_PATH, "build") REDIRECTS_FILE = os.path.join(DOC_PATH, "redirects.csv") +pandas_built_with_meson = False class DocBuilder: @@ -145,7 +146,11 @@ def _sphinx_build(self, kind: str): SOURCE_PATH, os.path.join(BUILD_PATH, kind), ] - return subprocess.call(cmd) + if pandas_built_with_meson: + cwd = DOC_PATH + else: + cwd = None + return subprocess.call(cmd, cwd=cwd) def _open_browser(self, single_doc_html): """ @@ -291,6 +296,8 @@ def zip_html(self): def main(): + global pandas_built_with_meson + cmds = [method for method in dir(DocBuilder) if not method.startswith("_")] joined = ",".join(cmds) @@ -356,7 +363,6 @@ def main(): # the import within the module, injecting it into the global namespace # Check for pandas built by meson - loaded_meson_pandas = False try: import pandas as pd @@ -365,12 +371,12 @@ def main(): del sys.modules["pandas"] del pd else: - loaded_meson_pandas = True + pandas_built_with_meson = True globals()["pandas"] = pd except ImportError: pass - if not loaded_meson_pandas: + if not pandas_built_with_meson: os.environ["PYTHONPATH"] = args.python_path sys.path.insert(0, args.python_path) globals()["pandas"] = importlib.import_module("pandas") From cdf560596dd5d79478a1a42fc0dc9a9e162e876d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 2 Nov 2022 09:00:46 -0400 Subject: [PATCH 47/69] update meson-python repo --- .github/actions/build_pandas/action.yml | 5 ----- .github/actions/setup-conda/action.yml | 4 ++-- .github/workflows/32-bit-linux.yml | 2 +- ci/deps/actions-310-numpydev.yaml | 2 +- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38-minimum_versions.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/actions-pypy-38.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- 11 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 1515957cac82d..5acc5ca572128 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -23,8 +23,3 @@ runs: call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" python -m pip install . --no-build-isolation -v shell: cmd /C call {0} -# env: -# # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 -# # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct -# N_JOBS: 1 -# #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }} diff --git a/.github/actions/setup-conda/action.yml b/.github/actions/setup-conda/action.yml index fce0cf4054ba2..002d0020c2df1 100644 --- a/.github/actions/setup-conda/action.yml +++ b/.github/actions/setup-conda/action.yml @@ -32,5 +32,5 @@ runs: channels: conda-forge channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }} condarc-file: ci/condarc.yml - cache-env: false - cache-downloads: false + cache-env: true + cache-downloads: true diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index 26beba2637f06..b1c9fd4c609c5 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -42,7 +42,7 @@ jobs: python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ pip install "git+https://github.com/mesonbuild/meson.git@master" && \ - pip install "git+https://github.com/FFY00/meson-python.git@main" && \ + pip install "git+https://github.com/mesonbuild/meson-python.git@main" && \ python -m pip install --no-build-isolation -v . && \ python -m pip list && \ export PANDAS_CI=1 && \ diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 93170f884ba4c..4de419b6d961c 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -18,7 +18,7 @@ dependencies: - pip: - "cython" - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 1730cbc833731..15afa7b340e18 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -54,4 +54,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 08cdff99b2fa4..7ae162ec1216d 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -70,4 +70,4 @@ dependencies: - pytorch - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index b4b95aa8fe2f5..f4f63d14c4419 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -56,4 +56,4 @@ dependencies: - zstandard=0.15.2 - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index f943af151af5e..1b64641beb3db 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -53,4 +53,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 1c2c007b633c7..f8c1c8dd82bb6 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -54,4 +54,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index af925ad65b2e4..f9c51bca85b95 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -21,4 +21,4 @@ dependencies: - pytz - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 30e9be97436f8..3e8f89e6e2417 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -54,4 +54,4 @@ dependencies: - zstandard - pip: - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" From 4ce0be46bcc3c4497a93ac817c78d7441784bdaa Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 2 Nov 2022 10:07:01 -0400 Subject: [PATCH 48/69] debug docs --- doc/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/make.py b/doc/make.py index 7d199948aa5c3..9581dce6e4a4e 100755 --- a/doc/make.py +++ b/doc/make.py @@ -137,7 +137,7 @@ def _sphinx_build(self, kind: str): if self.num_jobs: cmd += ["-j", self.num_jobs] if self.warnings_are_errors: - cmd += ["-W", "--keep-going"] + cmd += ["-W", "--keep-going", "T"] if self.verbosity: cmd.append(f"-{'v' * self.verbosity}") cmd += [ From 7ca8ef9b9bbd2b9a3acfec6d5df77d778c0a7fa9 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 2 Nov 2022 10:35:28 -0400 Subject: [PATCH 49/69] typoed --- doc/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/make.py b/doc/make.py index 9581dce6e4a4e..1dc07805a01b1 100755 --- a/doc/make.py +++ b/doc/make.py @@ -137,7 +137,7 @@ def _sphinx_build(self, kind: str): if self.num_jobs: cmd += ["-j", self.num_jobs] if self.warnings_are_errors: - cmd += ["-W", "--keep-going", "T"] + cmd += ["-W", "--keep-going", "-T"] if self.verbosity: cmd.append(f"-{'v' * self.verbosity}") cmd += [ From 3ae895486d2abdbf8f4c015c4e348b52d7ee6dc0 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:30:15 -0400 Subject: [PATCH 50/69] debug more --- doc/make.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/make.py b/doc/make.py index 1dc07805a01b1..9fa36e2806e63 100755 --- a/doc/make.py +++ b/doc/make.py @@ -135,9 +135,9 @@ def _sphinx_build(self, kind: str): cmd = ["sphinx-build", "-b", kind] if self.num_jobs: - cmd += ["-j", self.num_jobs] + cmd += ["-j", "1"] # self.num_jobs] if self.warnings_are_errors: - cmd += ["-W", "--keep-going", "-T"] + cmd += ["-W", "--keep-going", "-T", "-v"] if self.verbosity: cmd.append(f"-{'v' * self.verbosity}") cmd += [ From d7ddd871c5380f2ab0fcf6080e6dd6124ff59331 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 3 Nov 2022 10:39:41 -0400 Subject: [PATCH 51/69] Try cd'ing into the doc directory --- .github/workflows/docbuild-and-upload.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index cfb4966847721..e4d4a3222ec6f 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -51,10 +51,10 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors + run: cd doc && make.py --warnings-are-errors - name: Build documentation zip - run: doc/make.py zip_html + run: cd doc && make.py zip_html - name: Build the interactive terminal run: | From 19157f0718dbfe639f7bd01e196f0f64a95b1180 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 3 Nov 2022 10:52:35 -0400 Subject: [PATCH 52/69] adjust --- .github/workflows/docbuild-and-upload.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index e4d4a3222ec6f..4fb70cf89e698 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -51,10 +51,10 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && make.py --warnings-are-errors + run: cd doc && python make.py --warnings-are-errors - name: Build documentation zip - run: cd doc && make.py zip_html + run: cd doc && python make.py zip_html - name: Build the interactive terminal run: | From 6f97335727cfe85cdb22ad3a42507cefa470299b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 3 Nov 2022 11:02:26 -0400 Subject: [PATCH 53/69] more verbosity --- doc/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/make.py b/doc/make.py index 9fa36e2806e63..fe3506a0f6ab6 100755 --- a/doc/make.py +++ b/doc/make.py @@ -137,7 +137,7 @@ def _sphinx_build(self, kind: str): if self.num_jobs: cmd += ["-j", "1"] # self.num_jobs] if self.warnings_are_errors: - cmd += ["-W", "--keep-going", "-T", "-v"] + cmd += ["-W", "--keep-going", "-T", "-vvv"] if self.verbosity: cmd.append(f"-{'v' * self.verbosity}") cmd += [ From a3a4eb548a0dca4f71f39d695602e92dd7f9e557 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 5 Nov 2022 07:10:37 -0400 Subject: [PATCH 54/69] update and take another stab at fixing the docs --- doc/make.py | 35 ++++++----------------------------- meson.build | 2 +- pandas/_libs/meson.build | 12 ++++-------- 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/doc/make.py b/doc/make.py index 9fa36e2806e63..84f2179491c33 100755 --- a/doc/make.py +++ b/doc/make.py @@ -27,7 +27,6 @@ SOURCE_PATH = os.path.join(DOC_PATH, "source") BUILD_PATH = os.path.join(DOC_PATH, "build") REDIRECTS_FILE = os.path.join(DOC_PATH, "redirects.csv") -pandas_built_with_meson = False class DocBuilder: @@ -135,9 +134,9 @@ def _sphinx_build(self, kind: str): cmd = ["sphinx-build", "-b", kind] if self.num_jobs: - cmd += ["-j", "1"] # self.num_jobs] + cmd += ["-j", self.num_jobs] if self.warnings_are_errors: - cmd += ["-W", "--keep-going", "-T", "-v"] + cmd += ["-W", "--keep-going"] if self.verbosity: cmd.append(f"-{'v' * self.verbosity}") cmd += [ @@ -146,11 +145,7 @@ def _sphinx_build(self, kind: str): SOURCE_PATH, os.path.join(BUILD_PATH, kind), ] - if pandas_built_with_meson: - cwd = DOC_PATH - else: - cwd = None - return subprocess.call(cmd, cwd=cwd) + return subprocess.call(cmd) def _open_browser(self, single_doc_html): """ @@ -296,8 +291,6 @@ def zip_html(self): def main(): - global pandas_built_with_meson - cmds = [method for method in dir(DocBuilder) if not method.startswith("_")] joined = ",".join(cmds) @@ -361,25 +354,9 @@ def main(): # external libraries (namely Sphinx) to compile this module and resolve # the import of `python_path` correctly. The latter is used to resolve # the import within the module, injecting it into the global namespace - - # Check for pandas built by meson - try: - import pandas as pd - - if not pd._built_with_meson: - # This is just a random pandas, so use the editable install of pandas - del sys.modules["pandas"] - del pd - else: - pandas_built_with_meson = True - globals()["pandas"] = pd - except ImportError: - pass - - if not pandas_built_with_meson: - os.environ["PYTHONPATH"] = args.python_path - sys.path.insert(0, args.python_path) - globals()["pandas"] = importlib.import_module("pandas") + # os.environ["PYTHONPATH"] = args.python_path + # sys.path.insert(0, args.python_path) + globals()["pandas"] = importlib.import_module("pandas") # Set the matplotlib backend to the non-interactive Agg backend for all # child processes. diff --git a/meson.build b/meson.build index bc5bcf5b845f6..4ee878ab2e827 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'pandas', 'c', 'cpp', 'cython', - version: '1.6.0.dev0', + version: '2.0.0.dev0', license: 'BSD-3', # TODO: bump when meson 0.64.0 comes out, # we are relying on 0.64.0 features diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index b0012fb76e57c..d4e3819357da4 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -3,14 +3,7 @@ _algos_take_helper = custom_target('algos_take_helper_pxi', input: 'algos_take_helper.pxi.in', command: [ py, tempita, '@INPUT@', '-o', '@OUTDIR@' - ], - # TODO: remove these two below lines - # Weird bug in meson that only repros on my potato computer - # (possibly b/c of low number of threads)? - # The first custom_target is never built for some reason - # so algos.pyx will error out later in the build. - build_by_default: true, - build_always_stale: true + ] ) _algos_common_helper = custom_target('algos_common_helper_pxi', output: 'algos_common_helper.pxi', @@ -91,10 +84,12 @@ cython_sources_list = [ 'util.pxd', ] cython_sources = {} +cython_sources_tgts = [] foreach source: cython_sources_list source_pyx = fs.copyfile(source) cython_sources += {source: source_pyx} + cython_sources_tgts += source_pyx endforeach subdir('tslibs') @@ -103,6 +98,7 @@ libs_sources = { # Dict of extension name -> dict of {sources, include_dirs, and deps} # numpy include dir is implicitly included 'algos': {'sources': [cython_sources['algos.pyx'], _algos_common_helper, _algos_take_helper, _khash_primitive_helper], + 'deps': declare_dependency(sources: [cython_sources['khash.pxd'], cython_sources['util.pxd'],_algos_common_helper, _algos_take_helper, _khash_primitive_helper]), 'include_dirs': klib_include}, 'arrays': {'sources': ['arrays.pyx']}, 'groupby': {'sources': ['groupby.pyx']}, From b7e6624a2d905aabf24b1b9493996889ba41da57 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 6 Nov 2022 10:56:51 -0500 Subject: [PATCH 55/69] try not installing conda-forge c/cxx-compiler --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 5e49cc761bad6..7f2b43b3ba660 100644 --- a/environment.yml +++ b/environment.yml @@ -79,8 +79,8 @@ dependencies: - asv # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. - - c-compiler - - cxx-compiler + #- c-compiler + #- cxx-compiler # code checks - black=22.3.0 From 6cbdfb8be7bc56681bb0eb3d100838afc1e0b345 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 7 Nov 2022 21:24:25 -0500 Subject: [PATCH 56/69] try to get minimal doc build that doesn't segfault --- environment.yml | 160 ++++++++++++++++++++++++------------------------ 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/environment.yml b/environment.yml index 7f2b43b3ba660..f0685d0e2dab8 100644 --- a/environment.yml +++ b/environment.yml @@ -21,78 +21,78 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 - - blosc - - brotlipy - - bottleneck - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs - - jinja2 - - lxml - - matplotlib - - numba>=0.53.1 - - numexpr>=2.8.0 # pin for "Run checks on imported code" job - - openpyxl - - odfpy - - pandas-gbq - - psycopg2 - - pyarrow - - pymysql - - pyreadstat - - pytables - - python-snappy - - pyxlsb - - s3fs>=2021.08.0 - - scipy - - sqlalchemy - - tabulate - - tzdata>=2022a - - xarray - - xlrd - - xlsxwriter - - zstandard - - # downstream packages - - aiobotocore<2.0.0 # GH#44311 pinned to fix docbuild - - botocore - - cftime - - dask - - ipython - - geopandas-base - - seaborn - - scikit-learn - - statsmodels - - coverage - - pandas-datareader - - pyyaml - - py - - pytorch - - # local testing dependencies - - moto - - flask - - # benchmarks - - asv - - # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. - #- c-compiler - #- cxx-compiler - - # code checks - - black=22.3.0 - - cpplint - - flake8=5.0.4 - - flake8-bugbear=22.7.1 # used by flake8, find likely bugs - - isort>=5.2.1 # check that imports are in the right order - - mypy=0.981 - - pre-commit>=2.15.0 - - pycodestyle # used by flake8 - - pyupgrade - +# - beautifulsoup4 +# - blosc +# - brotlipy +# - bottleneck +# - fastparquet +# - fsspec +# - html5lib +# - hypothesis +# - gcsfs +# - jinja2 +# - lxml +# - matplotlib +# - numba>=0.53.1 +# - numexpr>=2.8.0 # pin for "Run checks on imported code" job +# - openpyxl +# - odfpy +# - pandas-gbq +# - psycopg2 +# - pyarrow +# - pymysql +# - pyreadstat +# - pytables +# - python-snappy +# - pyxlsb +# - s3fs>=2021.08.0 +# - scipy +# - sqlalchemy +# - tabulate +# - tzdata>=2022a +# - xarray +# - xlrd +# - xlsxwriter +# - zstandard +# +# # downstream packages +# - aiobotocore<2.0.0 # GH#44311 pinned to fix docbuild +# - botocore +# - cftime +# - dask +# - ipython +# - geopandas-base +# - seaborn +# - scikit-learn +# - statsmodels +# - coverage +# - pandas-datareader +# - pyyaml +# - py +# - pytorch +# +# # local testing dependencies +# - moto +# - flask +# +# # benchmarks +# - asv +# +# # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. +# - c-compiler +# - cxx-compiler +# +# # code checks +# - black=22.3.0 +# - cpplint +# - flake8=5.0.4 +# - flake8-bugbear=22.7.1 # used by flake8, find likely bugs +# - isort>=5.2.1 # check that imports are in the right order +# - mypy=0.981 +# - pre-commit>=2.15.0 +# - pycodestyle # used by flake8 +# - pyupgrade +# # documentation - gitpython # obtain contributors from git for whatsnew - gitdb @@ -116,14 +116,14 @@ dependencies: - nbformat - notebook>=6.0.3 - ipykernel - - # web - - jinja2 # in optional dependencies, but documented here as needed - - markdown - - feedparser - - pyyaml - - requests - +# +# # web +# - jinja2 # in optional dependencies, but documented here as needed +# - markdown +# - feedparser +# - pyyaml +# - requests +# # build the interactive terminal - jupyterlab >=3.4,<4 - pip: From a4c33898de8d0803a4aa97999d68f3ec7c52408d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 07:28:47 -0500 Subject: [PATCH 57/69] add back web deps --- environment.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/environment.yml b/environment.yml index f0685d0e2dab8..d5f51026baab2 100644 --- a/environment.yml +++ b/environment.yml @@ -116,14 +116,14 @@ dependencies: - nbformat - notebook>=6.0.3 - ipykernel -# -# # web -# - jinja2 # in optional dependencies, but documented here as needed -# - markdown -# - feedparser -# - pyyaml -# - requests -# + + # web + - jinja2 # in optional dependencies, but documented here as needed + - markdown + - feedparser + - pyyaml + - requests + # build the interactive terminal - jupyterlab >=3.4,<4 - pip: From 05d2f533d3b550303c04f0ae175bf3db14fcda78 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 17:09:35 -0500 Subject: [PATCH 58/69] add matplotlib back --- .github/workflows/docbuild-and-upload.yml | 5 +++++ environment.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 45306c65c36ee..4d7719f8056f1 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -45,6 +45,11 @@ jobs: - name: Build Pandas uses: ./.github/actions/build_pandas + - name: Build Version + run: | + # Can't import pandas from the source directory + cd .. && python -c "import pandas; pandas.show_versions();" + - name: Build website run: python web/pandas_web.py web/pandas --target-path=web/build diff --git a/environment.yml b/environment.yml index d5f51026baab2..a84ac1386f6b9 100644 --- a/environment.yml +++ b/environment.yml @@ -32,7 +32,7 @@ dependencies: # - gcsfs # - jinja2 # - lxml -# - matplotlib + - matplotlib # - numba>=0.53.1 # - numexpr>=2.8.0 # pin for "Run checks on imported code" job # - openpyxl From 26435dd35267334d523a677756dfc528fe91ea00 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 17:21:54 -0500 Subject: [PATCH 59/69] bring back more packages --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index a84ac1386f6b9..9cba5175ef497 100644 --- a/environment.yml +++ b/environment.yml @@ -35,7 +35,7 @@ dependencies: - matplotlib # - numba>=0.53.1 # - numexpr>=2.8.0 # pin for "Run checks on imported code" job -# - openpyxl + - openpyxl # - odfpy # - pandas-gbq # - psycopg2 @@ -46,7 +46,7 @@ dependencies: # - python-snappy # - pyxlsb # - s3fs>=2021.08.0 -# - scipy + - scipy # - sqlalchemy # - tabulate # - tzdata>=2022a From 7a2bb0caaadbe57142dfff6a35a5a539b757a311 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 17:50:41 -0500 Subject: [PATCH 60/69] Debug more --- .github/workflows/docbuild-and-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 4d7719f8056f1..5d2e903c0654e 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && python make.py --warnings-are-errors + run: cd doc && python make.py --warnings-are-errors -vvv --num-jobs 1 - name: Build documentation zip run: cd doc && python make.py zip_html From a16e98989a4a86a460825853256730205915a7d2 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 18:21:46 -0500 Subject: [PATCH 61/69] Silence warnings and debug more --- .github/workflows/docbuild-and-upload.yml | 2 +- meson.build | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 5d2e903c0654e..afd2ac93a965e 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && python make.py --warnings-are-errors -vvv --num-jobs 1 + run: cd doc && python make.py --warnings-are-errors -vv --num-jobs 1 - name: Build documentation zip run: cd doc && python make.py zip_html diff --git a/meson.build b/meson.build index 4ee878ab2e827..09f9bfefe1f22 100644 --- a/meson.build +++ b/meson.build @@ -13,6 +13,9 @@ project( ] ) +add_global_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'c') +add_global_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') + py_mod = import('python') fs = import('fs') py = py_mod.find_installation('python') From 71bcd582482a1114c7f35952e16e5a6d7687759b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:21:45 -0500 Subject: [PATCH 62/69] try to catch the segfault --- .github/workflows/docbuild-and-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index afd2ac93a965e..54648c3dcfa37 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && python make.py --warnings-are-errors -vv --num-jobs 1 + run: cd doc && PYTHONFAULTHANDLER=1 python make.py --num-jobs 1 -vv #--warnings-are-errors - name: Build documentation zip run: cd doc && python make.py zip_html From a948860a98266b86772ac01295fbae92e0e07520 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:54:13 -0500 Subject: [PATCH 63/69] catchsegv and debug build --- .github/workflows/docbuild-and-upload.yml | 2 +- meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 54648c3dcfa37..7112cf623c044 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && PYTHONFAULTHANDLER=1 python make.py --num-jobs 1 -vv #--warnings-are-errors + run: cd doc && catchsegv PYTHONFAULTHANDLER=1 python make.py --num-jobs 1 -vv #--warnings-are-errors - name: Build documentation zip run: cd doc && python make.py zip_html diff --git a/meson.build b/meson.build index 09f9bfefe1f22..0d042277cda69 100644 --- a/meson.build +++ b/meson.build @@ -8,7 +8,7 @@ project( # we are relying on 0.64.0 features meson_version: '>=0.63', default_options: [ - 'buildtype=release', + 'buildtype=debug', 'c_std=c99' ] ) From dec531721560664230d4d0f58b5f58d6f81d5409 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 21:12:04 -0500 Subject: [PATCH 64/69] use catchsegv properly --- .github/workflows/docbuild-and-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 7112cf623c044..f6d843ac53f67 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && catchsegv PYTHONFAULTHANDLER=1 python make.py --num-jobs 1 -vv #--warnings-are-errors + run: cd doc && catchsegv python make.py --num-jobs 1 -vv #--warnings-are-errors - name: Build documentation zip run: cd doc && python make.py zip_html From aa57379fa93cff87c9cf5e61b7bc2547f59420c9 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 8 Nov 2022 21:46:38 -0500 Subject: [PATCH 65/69] try to fix strdup not found by lowering compiler strictness --- meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 0d042277cda69..db6d460438b88 100644 --- a/meson.build +++ b/meson.build @@ -9,7 +9,7 @@ project( meson_version: '>=0.63', default_options: [ 'buildtype=debug', - 'c_std=c99' + #'c_std=c99' ] ) From ecb6f6e011b926b10dfb1f424cd184d8e1660609 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 9 Nov 2022 06:40:37 -0500 Subject: [PATCH 66/69] Bring back everything but compilers --- environment.yml | 128 ++++++++++++++++++++++++------------------------ meson.build | 2 + 2 files changed, 66 insertions(+), 64 deletions(-) diff --git a/environment.yml b/environment.yml index 9cba5175ef497..aa7d2095c1a4c 100644 --- a/environment.yml +++ b/environment.yml @@ -21,78 +21,78 @@ dependencies: - pytz # optional dependencies -# - beautifulsoup4 -# - blosc -# - brotlipy -# - bottleneck -# - fastparquet -# - fsspec -# - html5lib -# - hypothesis -# - gcsfs -# - jinja2 -# - lxml + - beautifulsoup4 + - blosc + - brotlipy + - bottleneck + - fastparquet + - fsspec + - html5lib + - hypothesis + - gcsfs + - jinja2 + - lxml - matplotlib -# - numba>=0.53.1 -# - numexpr>=2.8.0 # pin for "Run checks on imported code" job + - numba>=0.53.1 + - numexpr>=2.8.0 # pin for "Run checks on imported code" job - openpyxl -# - odfpy -# - pandas-gbq -# - psycopg2 -# - pyarrow -# - pymysql -# - pyreadstat -# - pytables -# - python-snappy -# - pyxlsb -# - s3fs>=2021.08.0 + - odfpy + - pandas-gbq + - psycopg2 + - pyarrow + - pymysql + - pyreadstat + - pytables + - python-snappy + - pyxlsb + - s3fs>=2021.08.0 - scipy -# - sqlalchemy -# - tabulate -# - tzdata>=2022a -# - xarray -# - xlrd -# - xlsxwriter -# - zstandard -# -# # downstream packages -# - aiobotocore<2.0.0 # GH#44311 pinned to fix docbuild -# - botocore -# - cftime -# - dask -# - ipython -# - geopandas-base -# - seaborn -# - scikit-learn -# - statsmodels -# - coverage -# - pandas-datareader -# - pyyaml -# - py -# - pytorch -# -# # local testing dependencies -# - moto -# - flask -# -# # benchmarks -# - asv + - sqlalchemy + - tabulate + - tzdata>=2022a + - xarray + - xlrd + - xlsxwriter + - zstandard + + # downstream packages + - aiobotocore<2.0.0 # GH#44311 pinned to fix docbuild + - botocore + - cftime + - dask + - ipython + - geopandas-base + - seaborn + - scikit-learn + - statsmodels + - coverage + - pandas-datareader + - pyyaml + - py + - pytorch + + # local testing dependencies + - moto + - flask + + # benchmarks + - asv # # # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. # - c-compiler # - cxx-compiler # -# # code checks -# - black=22.3.0 -# - cpplint -# - flake8=5.0.4 -# - flake8-bugbear=22.7.1 # used by flake8, find likely bugs -# - isort>=5.2.1 # check that imports are in the right order -# - mypy=0.981 -# - pre-commit>=2.15.0 -# - pycodestyle # used by flake8 -# - pyupgrade -# + # code checks + - black=22.3.0 + - cpplint + - flake8=5.0.4 + - flake8-bugbear=22.7.1 # used by flake8, find likely bugs + - isort>=5.2.1 # check that imports are in the right order + - mypy=0.981 + - pre-commit>=2.15.0 + - pycodestyle # used by flake8 + - pyupgrade + # documentation - gitpython # obtain contributors from git for whatsnew - gitdb diff --git a/meson.build b/meson.build index db6d460438b88..00aa2985ea8e2 100644 --- a/meson.build +++ b/meson.build @@ -9,6 +9,8 @@ project( meson_version: '>=0.63', default_options: [ 'buildtype=debug', + # TODO: We are using POSIX functions(strdup in ujson), so we can't compile + # with strict C99 :( #'c_std=c99' ] ) From 4d0a15bca62afe626b90eaa12d3cbdf18a37a9bf Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 9 Nov 2022 08:44:28 -0500 Subject: [PATCH 67/69] Going for green --- .github/workflows/32-bit-linux.yml | 2 +- .github/workflows/docbuild-and-upload.yml | 2 +- .github/workflows/python-dev.yml | 15 ++++++++++++++- environment.yml | 12 ++++++------ meson.build | 6 +++++- requirements-dev.txt | 2 +- 6 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index 77a1f73a88346..073776e3158df 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -41,9 +41,9 @@ jobs: pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ pip install "git+https://github.com/mesonbuild/meson.git@master" && \ pip install "git+https://github.com/mesonbuild/meson-python.git@main" && \ + export PANDAS_CI=1 && \ python -m pip install --no-build-isolation -v . && \ python -m pip list && \ - export PANDAS_CI=1 && \ pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml --import-mode=importlib" - name: Publish test results for Python 3.8-32 bit full Linux diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index f6d843ac53f67..2fc25b096558d 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && catchsegv python make.py --num-jobs 1 -vv #--warnings-are-errors + run: cd doc && catchsegv python make.py --num-jobs 1 --warnings-are-errors - name: Build documentation zip run: cd doc && python make.py zip_html diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c45fa7f91b428..dface111d7310 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -82,9 +82,22 @@ jobs: python -m pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python -m pip list + # Sigh, someone (numpy?) is depending on mingw, which pandas doesn't compile with. + # Also, meson doesn't detect visual c++ unless cl.exe is in path. + # TODO: File a bug with meson about this. - name: Build Pandas + if : ${{ runner.os != 'Windows' }} run: | - python -m pip install -v . --no-build-isolation --no-index + pip install . --no-build-isolation -v + shell: bash -el {0} + + - name: Build Pandas (Windows) + if: ${{ runner.os == 'Windows' }} + run: | + call micromamba activate test + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + python -m pip install . --no-build-isolation -v + shell: cmd /C call {0} - name: Build Version run: | diff --git a/environment.yml b/environment.yml index aa7d2095c1a4c..887d7e0a1426e 100644 --- a/environment.yml +++ b/environment.yml @@ -77,11 +77,11 @@ dependencies: # benchmarks - asv -# -# # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. -# - c-compiler -# - cxx-compiler -# + + # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms. + - c-compiler + - cxx-compiler + # code checks - black=22.3.0 - cpplint @@ -130,4 +130,4 @@ dependencies: - jupyterlite==0.1.0b12 - sphinx-toggleprompt - "git+https://github.com/mesonbuild/meson.git@master" - - "git+https://github.com/FFY00/meson-python.git@main" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/meson.build b/meson.build index 00aa2985ea8e2..92e14d57b3cf4 100644 --- a/meson.build +++ b/meson.build @@ -8,7 +8,11 @@ project( # we are relying on 0.64.0 features meson_version: '>=0.63', default_options: [ - 'buildtype=debug', + # TODO: investigate, does meson try to compile against debug Python + # when buildtype = debug, this seems to be causing problems on CI + # where provided Python is not compiled in debug mode + 'buildtype=release', + 'werror=true', # TODO: We are using POSIX functions(strdup in ujson), so we can't compile # with strict C99 :( #'c_std=c99' diff --git a/requirements-dev.txt b/requirements-dev.txt index 0510ed4df3778..63a11df6a8248 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -100,4 +100,4 @@ jupyterlab >=3.4,<4 jupyterlite==0.1.0b12 sphinx-toggleprompt git+https://github.com/mesonbuild/meson.git@master -git+https://github.com/FFY00/meson-python.git@main +git+https://github.com/mesonbuild/meson-python.git@main From c4c3f9b29bf64611fd452f340c81790f02e7c5c9 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:13:00 -0500 Subject: [PATCH 68/69] Try for green sans 32-bit --- .github/workflows/python-dev.yml | 2 +- environment.yml | 1 + meson.build | 3 ++- requirements-dev.txt | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index dface111d7310..5d975e81fb730 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -88,7 +88,7 @@ jobs: - name: Build Pandas if : ${{ runner.os != 'Windows' }} run: | - pip install . --no-build-isolation -v + python -m pip install . --no-build-isolation -v shell: bash -el {0} - name: Build Pandas (Windows) diff --git a/environment.yml b/environment.yml index 887d7e0a1426e..d002cb16d59f5 100644 --- a/environment.yml +++ b/environment.yml @@ -107,6 +107,7 @@ dependencies: - types-python-dateutil - types-PyMySQL - types-pytz + - types-setuptools # documentation (jupyter notebooks) - nbconvert>=6.4.5 diff --git a/meson.build b/meson.build index 92e14d57b3cf4..43b6621410c7e 100644 --- a/meson.build +++ b/meson.build @@ -12,7 +12,8 @@ project( # when buildtype = debug, this seems to be causing problems on CI # where provided Python is not compiled in debug mode 'buildtype=release', - 'werror=true', + # TODO: turn on werror when ready + #'werror=true', # TODO: We are using POSIX functions(strdup in ujson), so we can't compile # with strict C99 :( #'c_std=c99' diff --git a/requirements-dev.txt b/requirements-dev.txt index 63a11df6a8248..fe464827da419 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -84,6 +84,7 @@ sphinx-copybutton types-python-dateutil types-PyMySQL types-pytz +types-setuptools nbconvert>=6.4.5 nbsphinx pandoc From 335ccf80e590baf339917e3a111125a20c34384b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:36:17 -0500 Subject: [PATCH 69/69] restore doc build and fix wrong python on macos --- .github/workflows/docbuild-and-upload.yml | 7 +------ .github/workflows/python-dev.yml | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 2fc25b096558d..45306c65c36ee 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -45,16 +45,11 @@ jobs: - name: Build Pandas uses: ./.github/actions/build_pandas - - name: Build Version - run: | - # Can't import pandas from the source directory - cd .. && python -c "import pandas; pandas.show_versions();" - - name: Build website run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: cd doc && catchsegv python make.py --num-jobs 1 --warnings-are-errors + run: cd doc && python make.py --warnings-are-errors - name: Build documentation zip run: cd doc && python make.py zip_html diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 5d975e81fb730..145e5e00d57ec 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -88,7 +88,7 @@ jobs: - name: Build Pandas if : ${{ runner.os != 'Windows' }} run: | - python -m pip install . --no-build-isolation -v + python3 -m pip install . --no-build-isolation -v shell: bash -el {0} - name: Build Pandas (Windows)