diff --git a/examples/pangeo-vorticity.ipynb b/examples/pangeo-vorticity.ipynb new file mode 100644 index 00000000..bd776418 --- /dev/null +++ b/examples/pangeo-vorticity.ipynb @@ -0,0 +1,1318 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f5a954d5", + "metadata": {}, + "source": [ + "# Pangeo Vorticity Workload\n", + "\n", + "This is a notebook for exploring a simplified version of the example in https://github.com/pangeo-data/distributed-array-examples/issues/1.\n", + "\n", + "In particular, it is the code from https://github.com/coiled/coiled-runtime/issues/174" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8d745a7b", + "metadata": {}, + "outputs": [], + "source": [ + "import cubed.array_api as xp\n", + "import cubed.random\n", + "from cubed.extensions.tqdm import TqdmProgressBar" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1bff01e3", + "metadata": {}, + "outputs": [], + "source": [ + "a = cubed.random.random((1000, 900, 800), chunks=100)\n", + "b = cubed.random.random((1000, 900, 800), chunks=100)\n", + "x = cubed.random.random((900, 800), chunks=100)\n", + "y = cubed.random.random((900, 800), chunks=100)\n", + "\n", + "result = a[1:] * x + b[1:] * y\n", + "result = xp.mean(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f8ca9389", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "num tasks: 5192\n", + "max required memory: 100.0 MB\n", + "\n", + "\n", + "array-017\n", + "\n", + "\n", + "array-017\n", + "__getitem__ \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-020\n", + "\n", + "\n", + "array-020\n", + "__getitem__ (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-017->array-020\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-018\n", + "\n", + "\n", + "array-018\n", + "__getitem__ \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-019\n", + "\n", + "\n", + "array-019\n", + "__getitem__ (rc)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-018->array-019\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-019->array-020\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-001\n", + "\n", + "\n", + "array-001\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-004\n", + "\n", + "\n", + "array-004\n", + "random (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-001->array-004\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-002\n", + "\n", + "\n", + "array-002\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-003\n", + "\n", + "\n", + "array-003\n", + "random (rc)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-002->array-003\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-003->array-004\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-004->array-020\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-021\n", + "\n", + "\n", + "array-021\n", + "__mul__ (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-020->array-021\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-009\n", + "\n", + "\n", + "array-009\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-012\n", + "\n", + "\n", + "array-012\n", + "random (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-009->array-012\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-010\n", + "\n", + "\n", + "array-010\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-011\n", + "\n", + "\n", + "array-011\n", + "random (rc)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-010->array-011\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-011->array-012\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-012->array-021\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-028\n", + "\n", + "\n", + "array-028\n", + "mean (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-021->array-028\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-022\n", + "\n", + "\n", + "array-022\n", + "__getitem__ \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-025\n", + "\n", + "\n", + "array-025\n", + "__getitem__ (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-022->array-025\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-023\n", + "\n", + "\n", + "array-023\n", + "__getitem__ \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-024\n", + "\n", + "\n", + "array-024\n", + "__getitem__ (rc)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-023->array-024\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-024->array-025\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-005\n", + "\n", + "\n", + "array-005\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-008\n", + "\n", + "\n", + "array-008\n", + "random (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-005->array-008\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-006\n", + "\n", + "\n", + "array-006\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-007\n", + "\n", + "\n", + "array-007\n", + "random (rc)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-006->array-007\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-007->array-008\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-008->array-025\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-026\n", + "\n", + "\n", + "array-026\n", + "__mul__ (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-025->array-026\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-013\n", + "\n", + "\n", + "array-013\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-016\n", + "\n", + "\n", + "array-016\n", + "random (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-013->array-016\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-014\n", + "\n", + "\n", + "array-014\n", + "random \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-015\n", + "\n", + "\n", + "array-015\n", + "random (rc)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-014->array-015\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-015->array-016\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-016->array-026\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-026->array-028\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-029\n", + "\n", + "\n", + "array-029\n", + "mean (rc)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-028->array-029\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-032\n", + "\n", + "\n", + "array-032\n", + "mean (bw)\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-029->array-032\n", + "\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.visualize(\"pangeo-vorticity\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0b9b8905", + "metadata": {}, + "outputs": [ + { + "data": { + "application/json": { + "ascii": false, + "bar_format": null, + "colour": null, + "elapsed": 0.00949406623840332, + "initial": 0, + "n": 0, + "ncols": null, + "nrows": 47, + "postfix": null, + "prefix": "array-019", + "rate": null, + "total": 1, + "unit": "it", + "unit_divisor": 1000, + "unit_scale": false + }, + "application/vnd.jupyter.widget-view+json": { + "model_id": "8f268522a58240a180abd2d8f534e021", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "array-019: 0%| | 0/1 [00:00 + + + + + + +num tasks: 5192 +max required memory: 100.0 MB + + +array-017 + + +array-017 +__getitem__ + + + + + +array-020 + + +array-020 +__getitem__ (bw) + + + + + +array-017->array-020 + + + + + +array-018 + + +array-018 +__getitem__ + + + + + +array-019 + + +array-019 +__getitem__ (rc) + + + + + +array-018->array-019 + + + + + +array-019->array-020 + + + + + +array-001 + + +array-001 +random + + + + + +array-004 + + +array-004 +random (bw) + + + + + +array-001->array-004 + + + + + +array-002 + + +array-002 +random + + + + + +array-003 + + +array-003 +random (rc) + + + + + +array-002->array-003 + + + + + +array-003->array-004 + + + + + +array-004->array-020 + + + + + +array-021 + + +array-021 +__mul__ (bw) + + + + + +array-020->array-021 + + + + + +array-009 + + +array-009 +random + + + + + +array-012 + + +array-012 +random (bw) + + + + + +array-009->array-012 + + + + + +array-010 + + +array-010 +random + + + + + +array-011 + + +array-011 +random (rc) + + + + + +array-010->array-011 + + + + + +array-011->array-012 + + + + + +array-012->array-021 + + + + + +array-028 + + +array-028 +mean (bw) + + + + + +array-021->array-028 + + + + + +array-022 + + +array-022 +__getitem__ + + + + + +array-025 + + +array-025 +__getitem__ (bw) + + + + + +array-022->array-025 + + + + + +array-023 + + +array-023 +__getitem__ + + + + + +array-024 + + +array-024 +__getitem__ (rc) + + + + + +array-023->array-024 + + + + + +array-024->array-025 + + + + + +array-005 + + +array-005 +random + + + + + +array-008 + + +array-008 +random (bw) + + + + + +array-005->array-008 + + + + + +array-006 + + +array-006 +random + + + + + +array-007 + + +array-007 +random (rc) + + + + + +array-006->array-007 + + + + + +array-007->array-008 + + + + + +array-008->array-025 + + + + + +array-026 + + +array-026 +__mul__ (bw) + + + + + +array-025->array-026 + + + + + +array-013 + + +array-013 +random + + + + + +array-016 + + +array-016 +random (bw) + + + + + +array-013->array-016 + + + + + +array-014 + + +array-014 +random + + + + + +array-015 + + +array-015 +random (rc) + + + + + +array-014->array-015 + + + + + +array-015->array-016 + + + + + +array-016->array-026 + + + + + +array-026->array-028 + + + + + +array-029 + + +array-029 +mean (rc) + + + + + +array-028->array-029 + + + + + +array-032 + + +array-032 +mean (bw) + + + + + +array-029->array-032 + + + + +