From 8ee710e1ec9d87369e3030e894f812a9aa9c1851 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Tue, 13 Aug 2024 08:58:23 -0500 Subject: [PATCH 01/21] fix: apply TBranch.array's veto of AsGrouped from 'ranges_or_baskets' to HasBranches.arrays and HasBranches.iterate (#1269) --- src/uproot/behaviors/TBranch.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/uproot/behaviors/TBranch.py b/src/uproot/behaviors/TBranch.py index 63f1e161a..688ae4de7 100644 --- a/src/uproot/behaviors/TBranch.py +++ b/src/uproot/behaviors/TBranch.py @@ -811,7 +811,10 @@ def get_from_cache(branchname, interpretation): checked = set() for _, context in expression_context: for branch in context["branches"]: - if branch.cache_key not in checked: + if branch.cache_key not in checked and not isinstance( + branchid_interpretation[branch.cache_key], + uproot.interpretation.grouped.AsGrouped, + ): checked.add(branch.cache_key) for ( basket_num, @@ -1035,7 +1038,10 @@ def iterate( checked = set() for _, context in expression_context: for branch in context["branches"]: - if branch.cache_key not in checked: + if branch.cache_key not in checked and not isinstance( + branchid_interpretation[branch.cache_key], + uproot.interpretation.grouped.AsGrouped, + ): checked.add(branch.cache_key) for ( basket_num, From ba3269b31cd791589bbd3f6d4634dbe606b9f997 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 09:40:00 -0500 Subject: [PATCH 02/21] chore(deps): bump actions/attest-build-provenance (#1268) Bumps the actions group with 1 update in the / directory: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance). Updates `actions/attest-build-provenance` from 1.3.2 to 1.4.1 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/bdd51370e0416ac948727f861e03c2f05d32d78e...310b0a4a3b0b78ef57ecda988ee04b132db73ef8) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b3b9d588c..897946774 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,7 +36,7 @@ jobs: run: ls -lha dist/ - name: Generate artifact attestation for sdist and wheel - uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + uses: actions/attest-build-provenance@310b0a4a3b0b78ef57ecda988ee04b132db73ef8 # v1.4.1 with: subject-path: dist/uproot-* From aa8b94f722982c6eb418f9f32273152039836fb2 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Thu, 15 Aug 2024 10:23:38 -0500 Subject: [PATCH 03/21] fix: explicit 'import awkward' needed to write NumPy strings (#1266) --- src/uproot/writing/_cascadetree.py | 19 ++++++++++++++----- .../test_1264_write_NumPy_array_of_strings.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 tests/test_1264_write_NumPy_array_of_strings.py diff --git a/src/uproot/writing/_cascadetree.py b/src/uproot/writing/_cascadetree.py index 9ecd2e87f..bf5491cec 100644 --- a/src/uproot/writing/_cascadetree.py +++ b/src/uproot/writing/_cascadetree.py @@ -663,11 +663,20 @@ def extend(self, file, sink, data): if datum["counter"] is None: if datum["dtype"] == ">U0": - lengths = numpy.asarray(awkward.num(branch_array.layout)) + awkward = uproot.extras.awkward() + + layout = awkward.to_layout(branch_array) + if isinstance( + layout, + (awkward.contents.ListArray, awkward.contents.RegularArray), + ): + layout = layout.to_ListOffsetArray64() + + lengths = numpy.asarray(awkward.num(layout)) which_big = lengths >= 255 lengths_extension_offsets = numpy.empty( - len(branch_array.layout) + 1, numpy.int64 + len(layout) + 1, numpy.int64 ) lengths_extension_offsets[0] = 0 numpy.cumsum(which_big * 4, out=lengths_extension_offsets[1:]) @@ -685,7 +694,7 @@ def extend(self, file, sink, data): [ lengths.reshape(-1, 1).astype("u1"), lengths_extension, - awkward.without_parameters(branch_array.layout), + awkward.without_parameters(layout), ], axis=1, ) @@ -693,8 +702,8 @@ def extend(self, file, sink, data): big_endian = numpy.asarray(awkward.flatten(leafc_data_awkward)) big_endian_offsets = ( lengths_extension_offsets - + numpy.asarray(branch_array.layout.offsets) - + numpy.arange(len(branch_array.layout.offsets)) + + numpy.asarray(layout.offsets) + + numpy.arange(len(layout.offsets)) ).astype(">i4", copy=True) tofill.append( ( diff --git a/tests/test_1264_write_NumPy_array_of_strings.py b/tests/test_1264_write_NumPy_array_of_strings.py new file mode 100644 index 000000000..0872bf417 --- /dev/null +++ b/tests/test_1264_write_NumPy_array_of_strings.py @@ -0,0 +1,18 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE + +import pytest +import uproot +import os +import numpy as np + + +def test(tmp_path): + newfile = os.path.join(tmp_path, "example.root") + + with uproot.recreate(newfile) as f: + f["t"] = {"x": np.array(["A", "B"]), "y": np.array([1, 2])} + f["t"].extend({"x": np.array(["A", "B"]), "y": np.array([1, 2])}) + + with uproot.open(newfile) as f: + assert f["t"]["x"].array().tolist() == ["A", "B", "A", "B"] + assert f["t"]["y"].array().tolist() == [1, 2, 1, 2] From 448f20e40573f01c9305c3e0017e6b0c66c2e506 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 23 Aug 2024 23:46:29 +0200 Subject: [PATCH 04/21] fix: TStreamerLoop code generation should make an array (#1277) * fix: TStreamerLoop code generation should make an array * fix: TStreamerLoop code generation should make an array --- src/uproot/streamers.py | 3 +- ..._1275_fix_TStreamerLoop_code_generation.py | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/test_1275_fix_TStreamerLoop_code_generation.py diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 4a4b7000d..7a6bdd0dc 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -1203,8 +1203,9 @@ def class_code( " if forth_obj is not None:", " raise CannotBeForth()", " cursor.skip(6)", + f" self._members[{self.name!r}] = numpy.empty(self.member({self.count_name!r}), dtype='O')", f" for tmp in range(self.member({self.count_name!r})):", - f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, af.add_to_path(forth_obj, context, {self.name!r}), file, self._file, self.concrete)", + f" self._members[{self.name!r}][tmp] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, af.add_to_path(forth_obj, context, {self.name!r}), file, self._file, self.concrete)", " if forth_obj is not None:", " if len(forth_obj.active_node.children) != 0:", f" forth_obj.active_node.children[-1].field_name = {self.name!r}", diff --git a/tests/test_1275_fix_TStreamerLoop_code_generation.py b/tests/test_1275_fix_TStreamerLoop_code_generation.py new file mode 100644 index 000000000..363f3a130 --- /dev/null +++ b/tests/test_1275_fix_TStreamerLoop_code_generation.py @@ -0,0 +1,30 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE + +import pytest +import uproot +import skhep_testdata + + +def test_ttime_custom(): + filename = skhep_testdata.data_path("uproot-issue-1275.root") + + with uproot.open(filename) as file: + spline = file["spline"] + + assert spline.member("fPoly")[0].member("fX") == 1.0 + assert spline.member("fPoly")[0].member("fY") == 1.0 + assert spline.member("fPoly")[0].member("fB") == 2.5 + assert spline.member("fPoly")[0].member("fC") == 0.0 + assert spline.member("fPoly")[0].member("fD") == 0.5 + + assert spline.member("fPoly")[1].member("fX") == 2.0 + assert spline.member("fPoly")[1].member("fY") == 4.0 + assert spline.member("fPoly")[1].member("fB") == 4.0 + assert spline.member("fPoly")[1].member("fC") == 1.5 + assert spline.member("fPoly")[1].member("fD") == -0.5 + + assert spline.member("fPoly")[2].member("fX") == 3.0 + assert spline.member("fPoly")[2].member("fY") == 9.0 + assert spline.member("fPoly")[2].member("fB") == 5.5 + assert spline.member("fPoly")[2].member("fC") == 1.0 + assert spline.member("fPoly")[2].member("fD") == 1.7142857142857144 From 12b473b50ced02354fd14f866d9e6a1208375f80 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Tue, 27 Aug 2024 17:20:18 +0200 Subject: [PATCH 05/21] fix: if first whole TBasket has inconclusive AwkwardForth (#1280) --- src/uproot/interpretation/objects.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index cca4e77b9..8404977db 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -512,6 +512,27 @@ def final_array( start = stop + # If *some* of the baskets are Awkward and *some* are not, + # convert the ones that are not, individually. + if any( + uproot._util.from_module(x, "awkward") for x in basket_arrays.values() + ) and isinstance( + library, + ( + uproot.interpretation.library.Awkward, + uproot.interpretation.library.Pandas, + ), + ): + awkward = uproot.extras.awkward() + for k, v in basket_arrays.items(): + if not uproot._util.from_module(v, "awkward"): + form = json.loads(self.awkward_form(branch.file).to_json()) + basket_arrays[k] = ( + uproot.interpretation.library._object_to_awkward_array( + awkward, form, v + ) + ) + if len(basket_arrays) == 0: output = numpy.array([], dtype=self.numpy_dtype) From adf831ebe2824eea669f7cb6e1046adbb3d17d93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 10:42:40 +0200 Subject: [PATCH 06/21] chore(deps): bump actions/attest-build-provenance in the actions group (#1279) Bumps the actions group with 1 update: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance). Updates `actions/attest-build-provenance` from 1.4.1 to 1.4.2 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/310b0a4a3b0b78ef57ecda988ee04b132db73ef8...6149ea5740be74af77f260b9db67e633f6b0a9a1) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 897946774..e4df62511 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,7 +36,7 @@ jobs: run: ls -lha dist/ - name: Generate artifact attestation for sdist and wheel - uses: actions/attest-build-provenance@310b0a4a3b0b78ef57ecda988ee04b132db73ef8 # v1.4.1 + uses: actions/attest-build-provenance@6149ea5740be74af77f260b9db67e633f6b0a9a1 # v1.4.2 with: subject-path: dist/uproot-* From 012df94ccace9147d12aae8710585318abacda3d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:29:38 +0000 Subject: [PATCH 07/21] chore: update pre-commit hooks (#1247) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pre-commit hooks updates: - [github.com/psf/black-pre-commit-mirror: 24.4.2 → 24.8.0](https://github.com/psf/black-pre-commit-mirror/compare/24.4.2...24.8.0) - [github.com/astral-sh/ruff-pre-commit: v0.5.0 → v0.6.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.0...v0.6.2) - [github.com/asottile/pyupgrade: v3.16.0 → v3.17.0](https://github.com/asottile/pyupgrade/compare/v3.16.0...v3.17.0) - [github.com/macisamuele/language-formatters-pre-commit-hooks: v2.13.0 → v2.14.0](https://github.com/macisamuele/language-formatters-pre-commit-hooks/compare/v2.13.0...v2.14.0) * follow ruff's advice --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jpivarski --- .pre-commit-config.yaml | 8 ++++---- src/uproot/behaviors/TH1.py | 2 +- src/uproot/interpretation/numerical.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d1fadeece..efe72b8de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,25 +18,25 @@ repos: - id: trailing-whitespace - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.0 + rev: v0.6.2 hooks: - id: ruff args: [--fix, --show-fixes] - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.17.0 hooks: - id: pyupgrade args: [--py38-plus] - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.13.0 + rev: v2.14.0 hooks: - id: pretty-format-toml args: [--autofix] diff --git a/src/uproot/behaviors/TH1.py b/src/uproot/behaviors/TH1.py index b9d24f7f1..3a256f325 100644 --- a/src/uproot/behaviors/TH1.py +++ b/src/uproot/behaviors/TH1.py @@ -71,7 +71,7 @@ def __eq__(self, other): Two histograms are equal if their axes are equal, their values are equal, and their variances are equal. """ - if type(self) != type(other): + if type(self) is not type(other): return False if self.axes != other.axes: return False diff --git a/src/uproot/interpretation/numerical.py b/src/uproot/interpretation/numerical.py index 241e35a67..1a978eeb9 100644 --- a/src/uproot/interpretation/numerical.py +++ b/src/uproot/interpretation/numerical.py @@ -518,7 +518,7 @@ def __repr__(self): def __eq__(self, other): return ( - type(self) == type(other) + type(self) is type(other) and self._low == other._low and self._high == other._high and self._num_bits == other._num_bits From 8c4d636e61e9810a1172a850e27ef5e7578ab2b8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:14:49 -0500 Subject: [PATCH 08/21] chore: update pre-commit hooks (#1284) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.2 → v0.6.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.2...v0.6.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index efe72b8de..36ba347fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.2 + rev: v0.6.4 hooks: - id: ruff args: [--fix, --show-fixes] From 3e2e08c5f1355ad75604f4daaf551132d288ff99 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Thu, 12 Sep 2024 13:32:13 -0400 Subject: [PATCH 09/21] Disable test that uses UNL file (#1287) --- tests/test_1146_split_ranges_for_large_files_over_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_1146_split_ranges_for_large_files_over_http.py b/tests/test_1146_split_ranges_for_large_files_over_http.py index a9c64be56..0c70e78d0 100644 --- a/tests/test_1146_split_ranges_for_large_files_over_http.py +++ b/tests/test_1146_split_ranges_for_large_files_over_http.py @@ -1,8 +1,12 @@ # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +import pytest import uproot +@pytest.mark.skip( + reason="The server started requiring authentication to access the file." +) def test_split_ranges_if_large_file_in_http(): fname = ( "https://xrootd-local.unl.edu:1094//store/user/AGC/nanoAOD/TT_TuneCUETP8M1_13TeV" From baec16d55232410521c35711ec4c0dcfea4d0ce7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Sep 2024 18:40:13 +0000 Subject: [PATCH 10/21] chore(deps): bump actions/attest-build-provenance in the actions group (#1286) Bumps the actions group with 1 update: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance). Updates `actions/attest-build-provenance` from 1.4.2 to 1.4.3 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/6149ea5740be74af77f260b9db67e633f6b0a9a1...1c608d11d69870c2092266b3f9a6f3abbf17002c) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jim Pivarski --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e4df62511..31583a7c3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,7 +36,7 @@ jobs: run: ls -lha dist/ - name: Generate artifact attestation for sdist and wheel - uses: actions/attest-build-provenance@6149ea5740be74af77f260b9db67e633f6b0a9a1 # v1.4.2 + uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3 with: subject-path: dist/uproot-* From 1d5a17a4f95ed5c2ce3cc479dd9b356803e49a03 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Thu, 12 Sep 2024 15:55:22 -0400 Subject: [PATCH 11/21] ci: add job to test with Emscripten (#1272) * Added test-emscripten dependencies * Added emscripten-build workflow * Ignore Pyodide files * Updated dependencies to run pyodide tests * Updated workflow to run Pyodide tests * Added simple Pyodide test * Moved Pyodide testing to a separate job * Removed timeout * Fixed Pyodide testing * Fixed test dependencies * Fixed dependencies * Patched a few functions to work with pyodide * Added more tests * Fixed tests * Fixed typo * Small patches to get things to work on pyodide * Added http tests * style: pre-commit fixes * Cleaned up and added writing test * Replaced wasm checks with global boolean --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/build-test.yml | 63 ++++++++++ .gitignore | 7 ++ pyproject.toml | 6 + src/uproot/_util.py | 3 + src/uproot/source/coalesce.py | 11 +- src/uproot/source/fsspec.py | 6 + src/uproot/source/http.py | 25 ++++ tests-wasm/__init__.py | 1 + tests-wasm/test_1272_basic_functionality.py | 124 ++++++++++++++++++++ tests-wasm/utils.py | 63 ++++++++++ tests/test_1191_rntuple_fixes.py | 2 +- 11 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 tests-wasm/__init__.py create mode 100644 tests-wasm/test_1272_basic_functionality.py create mode 100644 tests-wasm/utils.py diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6248c0dd5..c847e607d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -116,3 +116,66 @@ jobs: - name: Run pytest run: | python -m pytest -vv tests --reruns 10 --reruns-delay 30 --only-rerun "(?i)http|ssl|timeout|expired|connection|socket" + + pyodide-build: + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + PYODIDE_VERSION: 0.26.2 + PYODIDE_BUILD_VERSION: 0.28.0 + AWKWARD_VERSION: v2.6.4 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pyodide-build + run: python3 -m pip install pyodide-build==$PYODIDE_BUILD_VERSION + + - name: Determine EMSDK version + id: compute-emsdk-version + run: | + pyodide config list + # Save EMSDK version + EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) + echo "emsdk-version=$EMSCRIPTEN_VERSION" >> $GITHUB_OUTPUT + + - name: Install EMSDK + uses: mymindstorm/setup-emsdk@v14 + with: + version: ${{ steps.compute-emsdk-version.outputs.emsdk-version }} + + - name: Build the package + run: pyodide build + + - name: Build an awkward wheel compatible with the awkward-cpp version in pyodide + run: | + git clone --depth 1 --branch $AWKWARD_VERSION https://github.com/scikit-hep/awkward.git dependencies/awkward + pyodide build dependencies/awkward + rm -rf dependencies/ + + - name: Download Pyodide + uses: pyodide/pyodide-actions/download-pyodide@v1 + with: + version: ${{ env.PYODIDE_VERSION }} + to: pyodide-dist + + - name: Install browser + uses: pyodide/pyodide-actions/install-browser@v1 + with: + runner: selenium + browser: chrome + browser-version: latest + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: pip install .[test-pyodide] pyodide-py==$PYODIDE_VERSION + + - name: Run pytest + run: | + pytest -vv --dist-dir=./pyodide-dist/ --runner=selenium --runtime=chrome tests-wasm diff --git a/.gitignore b/.gitignore index 0aceed8b2..777961eb1 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,10 @@ dmypy.json # Pyre type checker .pyre/ + +# Local copies of skhep_testdata files +skhep_testdata/ + +# Pyodide +.pyodide* +dist-pyodide/ diff --git a/pyproject.toml b/pyproject.toml index 0a5b74418..08cda684e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,12 @@ test = [ "scikit-hep-testdata", "rangehttpserver" ] +test-pyodide = [ + "pytest>=6", + "pytest-pyodide", + "pytest-timeout", + "scikit-hep-testdata" +] xrootd = ["fsspec-xrootd"] [project.urls] diff --git a/src/uproot/_util.py b/src/uproot/_util.py index 11c1d5bc4..96e9e8d21 100644 --- a/src/uproot/_util.py +++ b/src/uproot/_util.py @@ -13,6 +13,7 @@ import numbers import os import re +import sys import warnings from collections.abc import Iterable from pathlib import Path @@ -27,6 +28,8 @@ import uproot.source.fsspec import uproot.source.object +wasm = sys.platform in ("emscripten", "wasi") + def tobytes(array): """ diff --git a/src/uproot/source/coalesce.py b/src/uproot/source/coalesce.py index 52cad9289..ab3ada9b7 100644 --- a/src/uproot/source/coalesce.py +++ b/src/uproot/source/coalesce.py @@ -33,6 +33,9 @@ def add_done_callback(self, callback, *, context=None): self._parent.add_done_callback(callback) def result(self, timeout=None): + if uproot._util.wasm: + # Pyodide futures don't support timeout + return self._parent.result()[self._s] return self._parent.result(timeout=timeout)[self._s] @@ -126,7 +129,13 @@ def coalesce_requests( def chunkify(req: RangeRequest): chunk = uproot.source.chunk.Chunk(source, req.start, req.stop, req.future) - req.future.add_done_callback(uproot.source.chunk.notifier(chunk, notifications)) + if uproot._util.wasm: + # Callbacks don't work in pyodide yet, so we call the notifier directly + uproot.source.chunk.notifier(chunk, notifications)() + else: + req.future.add_done_callback( + uproot.source.chunk.notifier(chunk, notifications) + ) return chunk return list(map(chunkify, all_requests)) diff --git a/src/uproot/source/fsspec.py b/src/uproot/source/fsspec.py index 588b6b31d..6f47890ba 100644 --- a/src/uproot/source/fsspec.py +++ b/src/uproot/source/fsspec.py @@ -164,6 +164,12 @@ def submit(request_ranges: list[tuple[int, int]]): self._fs.cat_ranges, paths=paths, starts=starts, ends=ends ) ) + if uproot._util.wasm: + # Threads can't be spawned in pyodide yet, so we run the function directly + # and return a future that is already resolved. + return uproot.source.futures.TrivialFuture( + self._fs.cat_ranges(paths=paths, starts=starts, ends=ends) + ) return self._executor.submit(coroutine) return coalesce_requests( diff --git a/src/uproot/source/http.py b/src/uproot/source/http.py index 2f0b64a03..e3fb1e617 100644 --- a/src/uproot/source/http.py +++ b/src/uproot/source/http.py @@ -239,6 +239,23 @@ def future(source: uproot.source.chunk.Source, start: int, stop: int): Returns a :doc:`uproot.source.futures.ResourceFuture` that calls :ref:`uproot.source.http.HTTPResource.get` with ``start`` and ``stop``. """ + # The default implementation doesn't work in Pyodide + if uproot._util.wasm: + + def task(resource): + import requests + + r = requests.get( + source._file_path, + headers=dict( + {"Range": f"bytes={start}-{stop - 1}"}, **source.auth_headers + ), + timeout=source.timeout, + ) + return r.content + + return uproot.source.futures.ResourceFuture(task) + connection = make_connection(source.parsed_url, source.timeout) connection.request( "GET", @@ -281,6 +298,14 @@ def multifuture( ``results`` and ``futures``. Subsequent attempts would immediately use the :ref:`uproot.source.http.HTTPSource.fallback`. """ + # The default implementation doesn't work in Pyodide + if uproot._util.wasm: + + def task(resource): + resource.handle_no_multipart(source, ranges, futures, results) + + return uproot.source.futures.ResourceFuture(task) + connection = make_connection(source.parsed_url, source.timeout) connection.request( diff --git a/tests-wasm/__init__.py b/tests-wasm/__init__.py new file mode 100644 index 000000000..f8c1f9969 --- /dev/null +++ b/tests-wasm/__init__.py @@ -0,0 +1 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE diff --git a/tests-wasm/test_1272_basic_functionality.py b/tests-wasm/test_1272_basic_functionality.py new file mode 100644 index 000000000..68d14e575 --- /dev/null +++ b/tests-wasm/test_1272_basic_functionality.py @@ -0,0 +1,124 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +from __future__ import annotations + +import pytest +from utils import run_test_in_pyodide + + +# Taken from test_0034_generic_objects_in_ttrees.py +@run_test_in_pyodide(test_file="uproot-HZZ-objects.root", packages=["pytest", "xxhash"]) +def test_read_ttree(selenium): + import pytest + + import uproot + + awkward = pytest.importorskip("awkward") + + with uproot.open("uproot-HZZ-objects.root")["events"] as tree: + result = tree["muonp4"].array(library="ak") + + assert ( + str(awkward.type(result)) + == "2421 * var * TLorentzVector[fP: TVector3[fX: float64, " + "fY: float64, fZ: float64], fE: float64]" + ) + + assert result[0, 0, "fE"] == 54.77949905395508 + assert result[0, 0, "fP", "fX"] == -52.89945602416992 + assert result[0, 0, "fP", "fY"] == -11.654671669006348 + assert result[0, 0, "fP", "fZ"] == -8.16079330444336 + + +# Taken from test_0406_write_a_tree.py +@run_test_in_pyodide() +def test_write_ttree(selenium): + import numpy as np + + import uproot + + newfile = "newfile.root" + + b1 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + b2 = [0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9] + + with uproot.recreate(newfile, compression=None) as fout: + tree = fout.mktree("t", {"b1": np.int32, "b2": np.float64}, "title") + + assert tree._cascading._basket_capacity == 10 + + for _ in range(5): + fout["t"].extend({"b1": b1, "b2": b2}) + + assert tree._cascading._basket_capacity == 10 + + for _ in range(10): + fout["t"].extend({"b1": b1, "b2": b2}) + + assert tree._cascading._basket_capacity == 100 + + for _ in range(90): + fout["t"].extend({"b1": b1, "b2": b2}) + + assert tree._cascading._basket_capacity == 1000 + + with uproot.open(newfile) as fin: + assert fin.keys() == ["t;1"] # same cycle number + t2 = fin["t"] + assert t2.num_entries == len(b1) * 105 + assert t2["b1"].array(library="np").tolist() == b1 * 105 + assert t2["b2"].array(library="np").tolist() == b2 * 105 + + +# Taken from test_1191_rntuple_fixes.py +@run_test_in_pyodide(test_file="test_ntuple_extension_columns.root") +def test_read_rntuple(selenium): + import uproot + + with uproot.open("test_ntuple_extension_columns.root") as f: + obj = f["EventData"] + + assert len(obj.column_records) > len(obj.header.column_records) + assert len(obj.column_records) == 936 + assert obj.column_records[903].first_ele_index == 36 + + arrays = obj.arrays() + + pbs = arrays[ + "HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux::fastDIPS20211215_pb" + ] + assert len(pbs) == 40 + assert all(len(a) == 0 for a in pbs[:36]) + assert next(i for i, a in enumerate(pbs) if len(a) != 0) == 36 + + jets = arrays["HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux:"] + assert len(jets.pt) == len(pbs) + + +# Taken from test_0088_read_with_http.py +@pytest.mark.network +@run_test_in_pyodide(packages=["requests"]) +def test_read_ttree_http(selenium): + import uproot + + with uproot.open( + "http://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root", + handler=uproot.source.http.HTTPSource, + ) as f: + data = f["tree"].arrays(["Y1", "Y2"], library="np") + assert len(data["Y1"]) == 100000 + assert len(data["Y2"]) == 100000 + + +# Taken from test_1191_rntuple_fixes.py +@pytest.mark.network +@run_test_in_pyodide(packages=["requests"]) +def test_read_rntuple_http(selenium): + import uproot + + with uproot.open( + "https://github.com/scikit-hep/scikit-hep-testdata/raw/main/src/skhep_testdata/data/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root", + handler=uproot.source.http.HTTPSource, + ) as f: + obj = f["Events"] + arrays = obj.arrays() + assert arrays["nMuon"].tolist() == [len(a) for a in arrays["Muon_pt"]] diff --git a/tests-wasm/utils.py b/tests-wasm/utils.py new file mode 100644 index 000000000..a703b13e2 --- /dev/null +++ b/tests-wasm/utils.py @@ -0,0 +1,63 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +from __future__ import annotations + +import os +import pathlib +import shutil +from functools import wraps + +import pytest +import skhep_testdata + +try: + import pytest_pyodide + from pytest_pyodide import run_in_pyodide + from pytest_pyodide.decorator import copy_files_to_pyodide +except ImportError: + pytest.skip("Pyodide is not available", allow_module_level=True) + +# Disable CORS so that we can fetch files for http tests +# Currently, this can only be done for Chrome +selenium_config = pytest_pyodide.config.get_global_config() +selenium_config.set_flags( + "chrome", + [ + *selenium_config.get_flags("chrome"), + "--disable-web-security", + "--disable-site-isolation-trials", + ], +) + + +# copy skhep_testdata files to testdata directory (needed for @copy_files_to_pyodide) +def ensure_testdata(filename): + if not pathlib.Path("skhep_testdata/" + filename).is_file(): + filepath = skhep_testdata.data_path(filename) + os.makedirs("skhep_testdata", exist_ok=True) + shutil.copyfile(filepath, "skhep_testdata/" + filename) + + +def run_test_in_pyodide(test_file=None, **kwargs): + def decorator(test_func): + @wraps(test_func) + def wrapper(selenium): + if test_file is not None: + ensure_testdata(test_file) + + @copy_files_to_pyodide( + file_list=[("dist", "dist")] + + ( + [] + if test_file is None + else [("skhep_testdata/" + test_file, test_file)] + ), + install_wheels=True, + ) + def inner_func(selenium): + run_in_pyodide(**kwargs)(test_func)(selenium) + + return inner_func(selenium) + + return wrapper + + return decorator diff --git a/tests/test_1191_rntuple_fixes.py b/tests/test_1191_rntuple_fixes.py index 7b64e45d2..a1e259310 100644 --- a/tests/test_1191_rntuple_fixes.py +++ b/tests/test_1191_rntuple_fixes.py @@ -41,7 +41,7 @@ def test_rntuple_cardinality(): def test_skip_recursively_empty_structs(): filename = skhep_testdata.data_path("DAOD_TRUTH3_RC2.root") with uproot.open(filename) as f: - obj = uproot.open(filename)["RNT:CollectionTree"] + obj = f["RNT:CollectionTree"] arrays = obj.arrays() jets = arrays["AntiKt4TruthDressedWZJetsAux:"] assert len(jets[0].pt) == 5 From dc19ce911caf45b3239d47aa61e1732f79417b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Ju=C5=A1kevi=C4=8Dius?= <71819123+giedrius2020@users.noreply.github.com> Date: Thu, 12 Sep 2024 22:54:02 +0200 Subject: [PATCH 12/21] fix: RNTuple ug-fixing offset array concatenation, adding filter_name (#1285) * Fixing cardinality cluster edges * style: pre-commit fixes * Ruff CI error fix * Pull request edit * style: pre-commit fixes * Testing newest ROOT version * style: pre-commit fixes * Update src/uproot/models/RNTuple.py Co-authored-by: Andres Rios Tascon * Update src/uproot/models/RNTuple.py Co-authored-by: Andres Rios Tascon * style: pre-commit fixes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Andres Rios Tascon Co-authored-by: Jim Pivarski --- src/uproot/models/RNTuple.py | 51 +++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/src/uproot/models/RNTuple.py b/src/uproot/models/RNTuple.py index f38e00a02..ee7bc2c1c 100644 --- a/src/uproot/models/RNTuple.py +++ b/src/uproot/models/RNTuple.py @@ -7,6 +7,7 @@ import struct from collections import defaultdict +from itertools import accumulate import numpy @@ -60,8 +61,20 @@ def _keys(self): keys.append(fr.field_name) return keys - def keys(self): - return self._keys + def keys( + self, + *, + filter_name=None, + filter_typename=None, + recursive=False, + full_paths=True, + # TODO: some arguments might be missing when compared with TTree. Solve when blocker is present in dask/coffea. + ): + if filter_name: + # Return keys from the filter_name list: + return [key for key in self._keys if key in filter_name] + else: + return self._keys def read_members(self, chunk, cursor, context, file): if uproot._awkwardforth.get_forth_obj(context) is not None: @@ -480,10 +493,28 @@ def read_pagedesc(self, destination, desc, dtype_str, dtype, nbits, split): # needed to chop off extra bits incase we used `unpackbits` destination[:] = content[:num_elements] - def read_col_pages(self, ncol, cluster_range, pad_missing_ele=False): - res = numpy.concatenate( - [self.read_col_page(ncol, i) for i in cluster_range], axis=0 - ) + def read_col_pages(self, ncol, cluster_range, dtype_byte, pad_missing_ele=False): + arrays = [self.read_col_page(ncol, i) for i in cluster_range] + + # Check if column stores offset values for jagged arrays (splitindex64) (applies to cardinality cols too): + if ( + dtype_byte == uproot.const.rntuple_col_type_to_num_dict["splitindex64"] + or dtype_byte == uproot.const.rntuple_col_type_to_num_dict["splitindex32"] + ): + # Extract the last offset values: + last_elements = [ + arr[-1] for arr in arrays[:-1] + ] # First value always zero, therefore skip first arr. + # Compute cumulative sum using itertools.accumulate: + last_offsets = list(accumulate(last_elements)) + # Add the offsets to each array + for i in range(1, len(arrays)): + arrays[i] += last_offsets[i - 1] + # Remove the first element from every sub-array except for the first one: + arrays = [arrays[0]] + [arr[1:] for arr in arrays[1:]] + + res = numpy.concatenate(arrays, axis=0) + if pad_missing_ele: first_ele_index = self.column_records[ncol].first_ele_index res = numpy.pad(res, (first_ele_index, 0)) @@ -530,8 +561,8 @@ def read_col_page(self, ncol, cluster_i): def arrays( self, - filter_names="*", - filter_typenames=None, + filter_name="*", + filter_typename=None, entry_start=0, entry_stop=None, decompression_executor=None, @@ -553,7 +584,7 @@ def arrays( ) form = self.to_akform().select_columns( - filter_names, prune_unions_and_records=False + filter_name, prune_unions_and_records=False ) # only read columns mentioned in the awkward form target_cols = [] @@ -563,9 +594,11 @@ def arrays( if "column" in key and "union" not in key: key_nr = int(key.split("-")[1]) dtype_byte = self.column_records[key_nr].type + content = self.read_col_pages( key_nr, range(start_cluster_idx, stop_cluster_idx), + dtype_byte=dtype_byte, pad_missing_ele=True, ) if "cardinality" in key: From 4a5c94a2ef6b6927c3f6f5dc3d10d41b431879f8 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Thu, 19 Sep 2024 13:29:32 -0500 Subject: [PATCH 13/21] docs: Uproot only supports Pythons 3.8 through 3.12 (#1293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Okay, so I'm leaving this as manual because the automated version doesn't show a range—it lists every Python version separately. This would make the badge too big, and I don't want the badges to use so much space. They're carefully arranged in two rows just to keep them from being overwhelming. ![](https://media.licdn.com/dms/image/v2/C5612AQGL9qoPMFZMYw/article-cover_image-shrink_600_2000/article-cover_image-shrink_600_2000/0/1520093949131?e=2147483647&v=beta&t=C8cPdu3LFOSwRe4VzeIHUUeg4rvRkuW9LIwQj6vVVAQ) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7460bdfa3..95d7e2835 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI version](https://badge.fury.io/py/uproot.svg)](https://pypi.org/project/uproot) [![Conda-Forge](https://img.shields.io/conda/vn/conda-forge/uproot)](https://github.com/conda-forge/uproot-feedstock) -[![Python 3.7‒3.12](https://img.shields.io/badge/python-3.7%E2%80%923.12-blue)](https://www.python.org) +[![Python 3.8‒3.12](https://img.shields.io/badge/python-3.8%E2%80%923.12-blue)](https://www.python.org) [![BSD-3 Clause License](https://img.shields.io/badge/license-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Continuous integration tests](https://github.com/scikit-hep/uproot5/actions/workflows/build-test.yml/badge.svg)](https://github.com/scikit-hep/uproot5/actions) From e3c1d6b3b5b853afdb96babeeaef8fa9fddccbce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:21:18 +0000 Subject: [PATCH 14/21] chore: update pre-commit hooks (#1290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.4 → v0.6.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.4...v0.6.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36ba347fc..dbad740d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.6.5 hooks: - id: ruff args: [--fix, --show-fixes] From 8cb3b238e8b0562d65644ef7519e05b80da1b4a6 Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:25:12 -0500 Subject: [PATCH 15/21] docs: add giedrius2020 as a contributor for code (#1295) * docs: update README.md [skip ci] * docs: update .all-contributorsrc [skip ci] --------- Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com> --- .all-contributorsrc | 9 +++++++++ README.md | 1 + 2 files changed, 10 insertions(+) diff --git a/.all-contributorsrc b/.all-contributorsrc index 5205fcd42..cb1917136 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -578,6 +578,15 @@ "contributions": [ "code" ] + }, + { + "login": "giedrius2020", + "name": "Giedrius Juškevičius", + "avatar_url": "https://avatars.githubusercontent.com/u/71819123?v=4", + "profile": "https://github.com/giedrius2020", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 95d7e2835..002c530e8 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ Thanks especially to the gracious help of Uproot contributors (including the [or Miles
Miles
💻 djw9497
djw9497
💻 Piotr Sobczyński
Piotr Sobczyński
💻 + Giedrius Juškevičius
Giedrius Juškevičius
💻 From b7dea1f409453ab52194806f83bf1ee83cc23cea Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Fri, 20 Sep 2024 14:47:32 -0400 Subject: [PATCH 16/21] Added xxhash dependency (#1289) Co-authored-by: Jim Pivarski --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 08cda684e..bf122b9d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ classifiers = [ dependencies = [ "awkward>=2.4.6", "cramjam>=2.5.0", + "xxhash", "numpy", "fsspec", "packaging", @@ -67,7 +68,6 @@ s3 = ["s3fs"] test = [ "isal", "deflate", - "xxhash", "minio", "aiohttp", "fsspec-xrootd", From 2ba0e4a904122966bb44871abd02cfe838e704f5 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Fri, 20 Sep 2024 15:15:49 -0400 Subject: [PATCH 17/21] Replaced Pyodide http test to make it more stable (#1296) --- tests-wasm/test_1272_basic_functionality.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests-wasm/test_1272_basic_functionality.py b/tests-wasm/test_1272_basic_functionality.py index 68d14e575..ed0495508 100644 --- a/tests-wasm/test_1272_basic_functionality.py +++ b/tests-wasm/test_1272_basic_functionality.py @@ -94,19 +94,25 @@ def test_read_rntuple(selenium): assert len(jets.pt) == len(pbs) -# Taken from test_0088_read_with_http.py +# Taken from test_0034_generic_objects_in_ttrees.py @pytest.mark.network @run_test_in_pyodide(packages=["requests"]) def test_read_ttree_http(selenium): + import awkward + import uproot with uproot.open( - "http://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root", + "https://github.com/scikit-hep/scikit-hep-testdata/raw/main/src/skhep_testdata/data/uproot-stl_containers.root", handler=uproot.source.http.HTTPSource, - ) as f: - data = f["tree"].arrays(["Y1", "Y2"], library="np") - assert len(data["Y1"]) == 100000 - assert len(data["Y2"]) == 100000 + )["tree"] as tree: + assert awkward.to_list(tree["vector_int32"].array(library="ak")) == [ + [1], + [1, 2], + [1, 2, 3], + [1, 2, 3, 4], + [1, 2, 3, 4, 5], + ] # Taken from test_1191_rntuple_fixes.py From f36f39c05018cc4609d0c6fa30756ad8c0555dfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 09:30:12 -0400 Subject: [PATCH 18/21] chore(deps): bump pyodide/pyodide-actions in the actions group (#1298) Bumps the actions group with 1 update: [pyodide/pyodide-actions](https://github.com/pyodide/pyodide-actions). Updates `pyodide/pyodide-actions` from 1 to 2 - [Commits](https://github.com/pyodide/pyodide-actions/compare/v1...v2) --- updated-dependencies: - dependency-name: pyodide/pyodide-actions dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c847e607d..9ce7ca368 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -160,13 +160,13 @@ jobs: rm -rf dependencies/ - name: Download Pyodide - uses: pyodide/pyodide-actions/download-pyodide@v1 + uses: pyodide/pyodide-actions/download-pyodide@v2 with: version: ${{ env.PYODIDE_VERSION }} to: pyodide-dist - name: Install browser - uses: pyodide/pyodide-actions/install-browser@v1 + uses: pyodide/pyodide-actions/install-browser@v2 with: runner: selenium browser: chrome From 6c802274177e00b293d57bec9ae37e7121635f79 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Wed, 25 Sep 2024 15:24:40 -0500 Subject: [PATCH 19/21] perf: desperate attempts to unlink output arrays --- src/uproot/behaviors/TBranch.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/uproot/behaviors/TBranch.py b/src/uproot/behaviors/TBranch.py index 688ae4de7..5d0945287 100644 --- a/src/uproot/behaviors/TBranch.py +++ b/src/uproot/behaviors/TBranch.py @@ -210,10 +210,14 @@ def iterate( arrays, report = item arrays = library.global_index(arrays, global_offset) report = report.to_global(global_offset) - yield arrays, report + popper = [arrays] + del arrays + yield popper.pop(), report + else: - arrays = library.global_index(item, global_offset) - yield arrays + popper = [library.global_index(item, global_offset)] + yield popper.pop() + except uproot.exceptions.KeyInFileError: if allow_missing: continue @@ -1111,6 +1115,9 @@ def iterate( ak_add_doc, ) + # no longer needed; save memory + del output + next_baskets = {} for branch, basket_num, basket in ranges_or_baskets: basket_entry_start, basket_entry_stop = basket.entry_start_stop @@ -1119,10 +1126,14 @@ def iterate( previous_baskets = next_baskets + # no longer needed; save memory + popper = [out] + del out + if report: - yield out, Report(self, sub_entry_start, sub_entry_stop) + yield popper.pop(), Report(self, sub_entry_start, sub_entry_stop) else: - yield out + yield popper.pop() def keys( self, From c0786a46230f79a68670b50f96cb43b9cb2823cc Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Wed, 25 Sep 2024 15:28:33 -0500 Subject: [PATCH 20/21] Revert "perf: desperate attempts to unlink output arrays" This reverts commit 6c802274177e00b293d57bec9ae37e7121635f79. --- src/uproot/behaviors/TBranch.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/uproot/behaviors/TBranch.py b/src/uproot/behaviors/TBranch.py index 5d0945287..688ae4de7 100644 --- a/src/uproot/behaviors/TBranch.py +++ b/src/uproot/behaviors/TBranch.py @@ -210,14 +210,10 @@ def iterate( arrays, report = item arrays = library.global_index(arrays, global_offset) report = report.to_global(global_offset) - popper = [arrays] - del arrays - yield popper.pop(), report - + yield arrays, report else: - popper = [library.global_index(item, global_offset)] - yield popper.pop() - + arrays = library.global_index(item, global_offset) + yield arrays except uproot.exceptions.KeyInFileError: if allow_missing: continue @@ -1115,9 +1111,6 @@ def iterate( ak_add_doc, ) - # no longer needed; save memory - del output - next_baskets = {} for branch, basket_num, basket in ranges_or_baskets: basket_entry_start, basket_entry_stop = basket.entry_start_stop @@ -1126,14 +1119,10 @@ def iterate( previous_baskets = next_baskets - # no longer needed; save memory - popper = [out] - del out - if report: - yield popper.pop(), Report(self, sub_entry_start, sub_entry_stop) + yield out, Report(self, sub_entry_start, sub_entry_stop) else: - yield popper.pop() + yield out def keys( self, From 4f980759ea9c3012c724b31a3939fd8fb9652cd1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:55:49 -0400 Subject: [PATCH 21/21] chore: update pre-commit hooks (#1300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.5 → v0.6.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.5...v0.6.7) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dbad740d7..e6892dd97 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.5 + rev: v0.6.7 hooks: - id: ruff args: [--fix, --show-fixes]