Fix int test

[jperez999]

This PR removes an environment variable for Tensorflow that causes tritonserver to fail to start.

[github-actions]

Documentation preview

https://nvidia-merlin.github.io/Merlin/review/pr-578

[nvidia-merlin-bot]

Click to view CI Results

GitHub pull request #578 of commit bafa365f25ab479fb93df1c588213942439f6e90, no merge conflicts.
Running as SYSTEM
Setting status of bafa365f25ab479fb93df1c588213942439f6e90 to PENDING with url https://10.20.13.93:8080/job/merlin_merlin/395/console and message: 'Pending'
Using context: Jenkins
Building on master in workspace /var/jenkins_home/workspace/merlin_merlin
using credential systems-login
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url https://github.com/NVIDIA-Merlin/Merlin # timeout=10
Fetching upstream changes from https://github.com/NVIDIA-Merlin/Merlin
 > git --version # timeout=10
using GIT_ASKPASS to set credentials login for merlin-systems
 > git fetch --tags --force --progress -- https://github.com/NVIDIA-Merlin/Merlin +refs/pull/578/*:refs/remotes/origin/pr/578/* # timeout=10
 > git rev-parse bafa365f25ab479fb93df1c588213942439f6e90^{commit} # timeout=10
Checking out Revision bafa365f25ab479fb93df1c588213942439f6e90 (detached)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f bafa365f25ab479fb93df1c588213942439f6e90 # timeout=10
Commit message: "remove ENV VAR for tf causing tritonserer failure"
 > git rev-list --no-walk eaab88f0519a916e012115d7a9e77af6725ffec0 # timeout=10
[merlin_merlin] $ /bin/bash /tmp/jenkins4365790805678306383.sh
============================= test session starts ==============================
platform linux -- Python 3.8.10, pytest-7.1.2, pluggy-1.0.0
rootdir: /var/jenkins_home/workspace/merlin_merlin/merlin
plugins: anyio-3.6.1, xdist-2.5.0, forked-1.4.0, cov-3.0.0
collected 3 items
tests/unit/test_version.py .                                             [ 33%]

tests/unit/examples/test_building_deploying_multi_stage_RecSys.py s      [ 66%]

tests/unit/examples/test_scaling_criteo_merlin_models.py F               [100%]
=================================== FAILURES ===================================

__________________________________ test_func ___________________________________
def test_func():
    with testbook(
        REPO_ROOT / "examples" / "scaling-criteo" / "02-ETL-with-NVTabular.ipynb",
        execute=False,
        timeout=180,
    ) as tb1:
        tb1.inject(
            """
            import os
            os.environ["BASE_DIR"] = "/tmp/input/criteo/"
            os.environ["INPUT_DATA_DIR"] = "/tmp/input/criteo/"
            os.environ["OUTPUT_DATA_DIR"] = "/tmp/output/criteo/"
            os.system("mkdir -p /tmp/input/criteo")
            os.system("mkdir -p /tmp/output/criteo")

            from merlin.datasets.synthetic import generate_data

            train, valid = generate_data("criteo", int(1000000), set_sizes=(0.7, 0.3))

            train.to_ddf().compute().to_parquet('/tmp/input/criteo/day_0.parquet')
            valid.to_ddf().compute().to_parquet('/tmp/input/criteo/day_1.parquet')
            """
        )


      tb1.execute()


tests/unit/examples/test_scaling_criteo_merlin_models.py:37:

/usr/local/lib/python3.8/dist-packages/testbook/client.py:147: in execute

super().execute_cell(cell, index)

/usr/local/lib/python3.8/dist-packages/nbclient/util.py:85: in wrapped

return just_run(coro(*args, **kwargs))

/usr/local/lib/python3.8/dist-packages/nbclient/util.py:60: in just_run

return loop.run_until_complete(coro)

/usr/lib/python3.8/asyncio/base_events.py:616: in run_until_complete

return future.result()

/usr/local/lib/python3.8/dist-packages/nbclient/client.py:1025: in async_execute_cell

await self._check_raise_for_error(cell, cell_index, exec_reply)

self = <testbook.client.TestbookNotebookClient object at 0x7f963c5eebb0>

cell = {'cell_type': 'code', 'execution_count': 7, 'metadata': {'jupyter': {'outputs_hidden': False}, 'execution': {'iopub.st...pool_size=(device_pool_size // 256) * 256\n    )\n\n# Create the distributed client\nclient = Client(cluster)\nclient'}

cell_index = 12

exec_reply = {'buffers': [], 'content': {'ename': 'MemoryError', 'engine_info': {'engine_id': -1, 'engine_uuid': 'f38252ca-58f5-4a5...e, 'engine': 'f38252ca-58f5-4a5f-9acb-a945c20f190b', 'started': '2022-08-31T14:31:01.631114Z', 'status': 'error'}, ...}
async def _check_raise_for_error(
    self, cell: NotebookNode, cell_index: int, exec_reply: t.Optional[t.Dict]
) -> None:

    if exec_reply is None:
        return None

    exec_reply_content = exec_reply['content']
    if exec_reply_content['status'] != 'error':
        return None

    cell_allows_errors = (not self.force_raise_errors) and (
        self.allow_errors
        or exec_reply_content.get('ename') in self.allow_error_names
        or "raises-exception" in cell.metadata.get("tags", [])
    )
    await run_hook(
        self.on_cell_error, cell=cell, cell_index=cell_index, execute_reply=exec_reply
    )
    if not cell_allows_errors:


      raise CellExecutionError.from_cell_and_msg(cell, exec_reply_content)


E           nbclient.exceptions.CellExecutionError: An error occurred while executing the following cell:

E           ------------------

E           # Dask dashboard

E           dashboard_port = "8787"

E

E           # Deploy a Single-Machine Multi-GPU Cluster

E           protocol = "tcp"  # "tcp" or "ucx"

E           if numba.cuda.is_available():

E               NUM_GPUS = list(range(len(numba.cuda.gpus)))

E           else:

E               NUM_GPUS = []

E           visible_devices = ",".join([str(n) for n in NUM_GPUS])  # Select devices to place workers

E           device_limit_frac = 0.7  # Spill GPU-Worker memory to host at this limit.

E           device_pool_frac = 0.8

E           part_mem_frac = 0.15

E

E           # Use total device size to calculate args.device_limit_frac

E           device_size = device_mem_size(kind="total")

E           device_limit = int(device_limit_frac * device_size)

E           device_pool_size = int(device_pool_frac * device_size)

E           part_size = int(part_mem_frac * device_size)

E

E           # Check if any device memory is already occupied

E           for dev in visible_devices.split(","):

E               fmem = pynvml_mem_size(kind="free", index=int(dev))

E               used = (device_size - fmem) / 1e9

E               if used > 1.0:

E                   warnings.warn(f"BEWARE - {used} GB is already occupied on device {int(dev)}!")

E

E           cluster = None  # (Optional) Specify existing scheduler port

E           if cluster is None:

E               cluster = LocalCUDACluster(

E                   protocol=protocol,

E                   n_workers=len(visible_devices.split(",")),

E                   CUDA_VISIBLE_DEVICES=visible_devices,

E                   device_memory_limit=device_limit,

E                   local_directory=dask_workdir,

E                   dashboard_address=":" + dashboard_port,

E                   rmm_pool_size=(device_pool_size // 256) * 256

E               )

E

E           # Create the distributed client

E           client = Client(cluster)

E           client

E           ------------------

E

E           �[0;31m---------------------------------------------------------------------------�[0m

E           �[0;31mMemoryError�[0m                               Traceback (most recent call last)

E           Input �[0;32mIn [7]�[0m, in �[0;36m<cell line: 29>�[0;34m()�[0m

E           �[1;32m     28�[0m cluster �[38;5;241m=�[39m �[38;5;28;01mNone�[39;00m  �[38;5;66;03m# (Optional) Specify existing scheduler port�[39;00m

E           �[1;32m     29�[0m �[38;5;28;01mif�[39;00m cluster �[38;5;129;01mis�[39;00m �[38;5;28;01mNone�[39;00m:

E           �[0;32m---> 30�[0m     cluster �[38;5;241m=�[39m �[43mLocalCUDACluster�[49m�[43m(�[49m

E           �[1;32m     31�[0m �[43m        �[49m�[43mprotocol�[49m�[38;5;241;43m=�[39;49m�[43mprotocol�[49m�[43m,�[49m

E           �[1;32m     32�[0m �[43m        �[49m�[43mn_workers�[49m�[38;5;241;43m=�[39;49m�[38;5;28;43mlen�[39;49m�[43m(�[49m�[43mvisible_devices�[49m�[38;5;241;43m.�[39;49m�[43msplit�[49m�[43m(�[49m�[38;5;124;43m"�[39;49m�[38;5;124;43m,�[39;49m�[38;5;124;43m"�[39;49m�[43m)�[49m�[43m)�[49m�[43m,�[49m

E           �[1;32m     33�[0m �[43m        �[49m�[43mCUDA_VISIBLE_DEVICES�[49m�[38;5;241;43m=�[39;49m�[43mvisible_devices�[49m�[43m,�[49m

E           �[1;32m     34�[0m �[43m        �[49m�[43mdevice_memory_limit�[49m�[38;5;241;43m=�[39;49m�[43mdevice_limit�[49m�[43m,�[49m

E           �[1;32m     35�[0m �[43m        �[49m�[43mlocal_directory�[49m�[38;5;241;43m=�[39;49m�[43mdask_workdir�[49m�[43m,�[49m

E           �[1;32m     36�[0m �[43m        �[49m�[43mdashboard_address�[49m�[38;5;241;43m=�[39;49m�[38;5;124;43m"�[39;49m�[38;5;124;43m:�[39;49m�[38;5;124;43m"�[39;49m�[43m �[49m�[38;5;241;43m+�[39;49m�[43m �[49m�[43mdashboard_port�[49m�[43m,�[49m

E           �[1;32m     37�[0m �[43m        �[49m�[43mrmm_pool_size�[49m�[38;5;241;43m=�[39;49m�[43m(�[49m�[43mdevice_pool_size�[49m�[43m �[49m�[38;5;241;43m/�[39;49m�[38;5;241;43m/�[39;49m�[43m �[49m�[38;5;241;43m256�[39;49m�[43m)�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[43m �[49m�[38;5;241;43m256�[39;49m

E           �[1;32m     38�[0m �[43m    �[49m�[43m)�[49m

E           �[1;32m     40�[0m �[38;5;66;03m# Create the distributed client�[39;00m

E           �[1;32m     41�[0m client �[38;5;241m=�[39m Client(cluster)

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/dask_cuda/local_cuda_cluster.py:366�[0m, in �[0;36mLocalCUDACluster.__init__�[0;34m(self, CUDA_VISIBLE_DEVICES, n_workers, threads_per_worker, memory_limit, device_memory_limit, data, local_directory, shared_filesystem, protocol, enable_tcp_over_ucx, enable_infiniband, enable_nvlink, enable_rdmacm, rmm_pool_size, rmm_maximum_pool_size, rmm_managed_memory, rmm_async, rmm_log_directory, rmm_track_allocations, jit_unspill, log_spilling, worker_class, pre_import, **kwargs)�[0m

E           �[1;32m    364�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mcuda_visible_devices �[38;5;241m=�[39m CUDA_VISIBLE_DEVICES

E           �[1;32m    365�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mscale(n_workers)

E           �[0;32m--> 366�[0m �[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43msync�[49m�[43m(�[49m�[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43m_correct_state�[49m�[43m)�[49m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:309�[0m, in �[0;36mSyncMethodMixin.sync�[0;34m(self, func, asynchronous, callback_timeout, args, **kwargs)�[0m

E           �[1;32m    307�[0m     �[38;5;28;01mreturn�[39;00m future

E           �[1;32m    308�[0m �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 309�[0m     �[38;5;28;01mreturn�[39;00m �[43msync�[49m�[43m(�[49m

E           �[1;32m    310�[0m �[43m        �[49m�[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43mloop�[49m�[43m,�[49m�[43m �[49m�[43mfunc�[49m�[43m,�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[43margs�[49m�[43m,�[49m�[43m �[49m�[43mcallback_timeout�[49m�[38;5;241;43m=�[39;49m�[43mcallback_timeout�[49m�[43m,�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[38;5;241;43m*�[39;49m�[43mkwargs�[49m

E           �[1;32m    311�[0m �[43m    �[49m�[43m)�[49m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:376�[0m, in �[0;36msync�[0;34m(loop, func, callback_timeout, *args, **kwargs)�[0m

E           �[1;32m    374�[0m �[38;5;28;01mif�[39;00m error:

E           �[1;32m    375�[0m     typ, exc, tb �[38;5;241m=�[39m error

E           �[0;32m--> 376�[0m     �[38;5;28;01mraise�[39;00m exc�[38;5;241m.�[39mwith_traceback(tb)

E           �[1;32m    377�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    378�[0m     �[38;5;28;01mreturn�[39;00m result

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:349�[0m, in �[0;36msync..f�[0;34m()�[0m

E           �[1;32m    347�[0m         future �[38;5;241m=�[39m asyncio�[38;5;241m.�[39mwait_for(future, callback_timeout)

E           �[1;32m    348�[0m     future �[38;5;241m=�[39m asyncio�[38;5;241m.�[39mensure_future(future)

E           �[0;32m--> 349�[0m     result �[38;5;241m=�[39m �[38;5;28;01myield�[39;00m future

E           �[1;32m    350�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    351�[0m     error �[38;5;241m=�[39m sys�[38;5;241m.�[39mexc_info()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/tornado/gen.py:769�[0m, in �[0;36mRunner.run�[0;34m(self)�[0m

E           �[1;32m    766�[0m exc_info �[38;5;241m=�[39m �[38;5;28;01mNone�[39;00m

E           �[1;32m    768�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 769�[0m     value �[38;5;241m=�[39m �[43mfuture�[49m�[38;5;241;43m.�[39;49m�[43mresult�[49m�[43m(�[49m�[43m)�[49m

E           �[1;32m    770�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    771�[0m     exc_info �[38;5;241m=�[39m sys�[38;5;241m.�[39mexc_info()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/deploy/spec.py:352�[0m, in �[0;36mSpecCluster._correct_state_internal�[0;34m(self)�[0m

E           �[1;32m    350�[0m     �[38;5;28;01mfor�[39;00m w �[38;5;129;01min�[39;00m workers:

E           �[1;32m    351�[0m         w�[38;5;241m.�[39m_cluster �[38;5;241m=�[39m weakref�[38;5;241m.�[39mref(�[38;5;28mself�[39m)

E           �[0;32m--> 352�[0m         �[38;5;28;01mawait�[39;00m w  �[38;5;66;03m# for tornado gen.coroutine support�[39;00m

E           �[1;32m    353�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mworkers�[38;5;241m.�[39mupdate(�[38;5;28mdict�[39m(�[38;5;28mzip�[39m(to_open, workers)))

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/core.py:299�[0m, in �[0;36mServer.await.._�[0;34m()�[0m

E           �[1;32m    293�[0m             �[38;5;28;01mraise�[39;00m �[38;5;167;01mTimeoutError�[39;00m(

E           �[1;32m    294�[0m                 �[38;5;124m"�[39m�[38;5;132;01m{}�[39;00m�[38;5;124m failed to start in �[39m�[38;5;132;01m{}�[39;00m�[38;5;124m seconds�[39m�[38;5;124m"�[39m�[38;5;241m.�[39mformat(

E           �[1;32m    295�[0m                     �[38;5;28mtype�[39m(�[38;5;28mself�[39m)�[38;5;241m.�[39m�[38;5;18m__name__�[39m, timeout

E           �[1;32m    296�[0m                 )

E           �[1;32m    297�[0m             )

E           �[1;32m    298�[0m     �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 299�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstart()

E           �[1;32m    300�[0m         �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mrunning

E           �[1;32m    301�[0m �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:347�[0m, in �[0;36mNanny.start�[0;34m(self)�[0m

E           �[1;32m    344�[0m     �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mplugin_add(plugin�[38;5;241m=�[39mplugin, name�[38;5;241m=�[39mname)

E           �[1;32m    346�[0m logger�[38;5;241m.�[39minfo(�[38;5;124m"�[39m�[38;5;124m        Start Nanny at: �[39m�[38;5;132;01m%r�[39;00m�[38;5;124m"�[39m, �[38;5;28mself�[39m�[38;5;241m.�[39maddress)

E           �[0;32m--> 347�[0m response �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39minstantiate()

E           �[1;32m    348�[0m �[38;5;28;01mif�[39;00m response �[38;5;241m==�[39m Status�[38;5;241m.�[39mrunning:

E           �[1;32m    349�[0m     �[38;5;28;01massert�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mworker_address

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:430�[0m, in �[0;36mNanny.instantiate�[0;34m(self)�[0m

E           �[1;32m    428�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    429�[0m     �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 430�[0m         result �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mprocess�[38;5;241m.�[39mstart()

E           �[1;32m    431�[0m     �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    432�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mclose()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:685�[0m, in �[0;36mWorkerProcess.start�[0;34m(self)�[0m

E           �[1;32m    683�[0m     �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstatus

E           �[1;32m    684�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 685�[0m     msg �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39m_wait_until_connected(uid)

E           �[1;32m    686�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    687�[0m     �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mfailed

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:803�[0m, in �[0;36mWorkerProcess._wait_until_connected�[0;34m(self, uid)�[0m

E           �[1;32m    799�[0m �[38;5;28;01mif�[39;00m �[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m �[38;5;129;01min�[39;00m msg:

E           �[1;32m    800�[0m     logger�[38;5;241m.�[39merror(

E           �[1;32m    801�[0m         �[38;5;124m"�[39m�[38;5;124mFailed while trying to start worker process: �[39m�[38;5;132;01m%s�[39;00m�[38;5;124m"�[39m, msg[�[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m]

E           �[1;32m    802�[0m     )

E           �[0;32m--> 803�[0m     �[38;5;28;01mraise�[39;00m msg[�[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m]

E           �[1;32m    804�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    805�[0m     �[38;5;28;01mreturn�[39;00m msg

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:869�[0m, in �[0;36mrun�[0;34m()�[0m

E           �[1;32m    865�[0m �[38;5;124;03m"""�[39;00m

E           �[1;32m    866�[0m �[38;5;124;03mTry to start worker and inform parent of outcome.�[39;00m

E           �[1;32m    867�[0m �[38;5;124;03m"""�[39;00m

E           �[1;32m    868�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 869�[0m     �[38;5;28;01mawait�[39;00m worker

E           �[1;32m    870�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m �[38;5;28;01mas�[39;00m e:

E           �[1;32m    871�[0m     logger�[38;5;241m.�[39mexception(�[38;5;124m"�[39m�[38;5;124mFailed to start worker�[39m�[38;5;124m"�[39m)

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/core.py:299�[0m, in �[0;36m_�[0;34m()�[0m

E           �[1;32m    293�[0m             �[38;5;28;01mraise�[39;00m �[38;5;167;01mTimeoutError�[39;00m(

E           �[1;32m    294�[0m                 �[38;5;124m"�[39m�[38;5;132;01m{}�[39;00m�[38;5;124m failed to start in �[39m�[38;5;132;01m{}�[39;00m�[38;5;124m seconds�[39m�[38;5;124m"�[39m�[38;5;241m.�[39mformat(

E           �[1;32m    295�[0m                     �[38;5;28mtype�[39m(�[38;5;28mself�[39m)�[38;5;241m.�[39m�[38;5;18m__name__�[39m, timeout

E           �[1;32m    296�[0m                 )

E           �[1;32m    297�[0m             )

E           �[1;32m    298�[0m     �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 299�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstart()

E           �[1;32m    300�[0m         �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mrunning

E           �[1;32m    301�[0m �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/worker.py:1372�[0m, in �[0;36mstart�[0;34m()�[0m

E           �[1;32m   1370�[0m         �[38;5;28;01mfor�[39;00m exc �[38;5;129;01min�[39;00m plugins_exceptions:

E           �[1;32m   1371�[0m             logger�[38;5;241m.�[39merror(�[38;5;28mrepr�[39m(exc))

E           �[0;32m-> 1372�[0m     �[38;5;28;01mraise�[39;00m plugins_exceptions[�[38;5;241m0�[39m]

E           �[1;32m   1374�[0m �[38;5;28mself�[39m�[38;5;241m.�[39m_pending_plugins �[38;5;241m=�[39m ()

E           �[1;32m   1376�[0m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39m_register_with_scheduler()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/worker.py:3248�[0m, in �[0;36mplugin_add�[0;34m()�[0m

E           �[1;32m   3246�[0m �[38;5;28;01mif�[39;00m �[38;5;28mhasattr�[39m(plugin, �[38;5;124m"�[39m�[38;5;124msetup�[39m�[38;5;124m"�[39m):

E           �[1;32m   3247�[0m     �[38;5;28;01mtry�[39;00m:

E           �[0;32m-> 3248�[0m         result �[38;5;241m=�[39m plugin�[38;5;241m.�[39msetup(worker�[38;5;241m=�[39m�[38;5;28mself�[39m)

E           �[1;32m   3249�[0m         �[38;5;28;01mif�[39;00m isawaitable(result):

E           �[1;32m   3250�[0m             result �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m result

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py:78�[0m, in �[0;36msetup�[0;34m()�[0m

E           �[1;32m     74�[0m     �[38;5;28;01mimport�[39;00m �[38;5;21;01mrmm�[39;00m

E           �[1;32m     76�[0m     pool_allocator �[38;5;241m=�[39m �[38;5;28;01mFalse�[39;00m �[38;5;28;01mif�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39minitial_pool_size �[38;5;129;01mis�[39;00m �[38;5;28;01mNone�[39;00m �[38;5;28;01melse�[39;00m �[38;5;28;01mTrue�[39;00m

E           �[0;32m---> 78�[0m     rmm�[38;5;241m.�[39mreinitialize(

E           �[1;32m     79�[0m         pool_allocator�[38;5;241m=�[39mpool_allocator,

E           �[1;32m     80�[0m         managed_memory�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mmanaged_memory,

E           �[1;32m     81�[0m         initial_pool_size�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39minitial_pool_size,

E           �[1;32m     82�[0m         maximum_pool_size�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mmaximum_pool_size,

E           �[1;32m     83�[0m         logging�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mlogging,

E           �[1;32m     84�[0m         log_file_name�[38;5;241m=�[39mget_rmm_log_file_name(

E           �[1;32m     85�[0m             worker, �[38;5;28mself�[39m�[38;5;241m.�[39mlogging, �[38;5;28mself�[39m�[38;5;241m.�[39mlog_directory

E           �[1;32m     86�[0m         ),

E           �[1;32m     87�[0m     )

E           �[1;32m     88�[0m �[38;5;28;01mif�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mrmm_track_allocations:

E           �[1;32m     89�[0m     �[38;5;28;01mimport�[39;00m �[38;5;21;01mrmm�[39;00m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/rmm/rmm.py:84�[0m, in �[0;36mreinitialize�[0;34m()�[0m

E           �[1;32m     31�[0m �[38;5;28;01mdef�[39;00m �[38;5;21mreinitialize�[39m(

E           �[1;32m     32�[0m     pool_allocator�[38;5;241m=�[39m�[38;5;28;01mFalse�[39;00m,

E           �[1;32m     33�[0m     managed_memory�[38;5;241m=�[39m�[38;5;28;01mFalse�[39;00m,

E           �[0;32m   (...)�[0m

E           �[1;32m     38�[0m     log_file_name�[38;5;241m=�[39m�[38;5;28;01mNone�[39;00m,

E           �[1;32m     39�[0m ):

E           �[1;32m     40�[0m     �[38;5;124;03m"""�[39;00m

E           �[1;32m     41�[0m �[38;5;124;03m    Finalizes and then initializes RMM using the options passed. Using memory�[39;00m

E           �[1;32m     42�[0m �[38;5;124;03m    from a previous initialization of RMM is undefined behavior and should be�[39;00m

E           �[0;32m   (...)�[0m

E           �[1;32m     82�[0m �[38;5;124;03m    corresponding to each device.�[39;00m

E           �[1;32m     83�[0m �[38;5;124;03m    """�[39;00m

E           �[0;32m---> 84�[0m     rmm�[38;5;241m.�[39mmr�[38;5;241m.�[39m_initialize(

E           �[1;32m     85�[0m         pool_allocator�[38;5;241m=�[39mpool_allocator,

E           �[1;32m     86�[0m         managed_memory�[38;5;241m=�[39mmanaged_memory,

E           �[1;32m     87�[0m         initial_pool_size�[38;5;241m=�[39minitial_pool_size,

E           �[1;32m     88�[0m         maximum_pool_size�[38;5;241m=�[39mmaximum_pool_size,

E           �[1;32m     89�[0m         devices�[38;5;241m=�[39mdevices,

E           �[1;32m     90�[0m         logging�[38;5;241m=�[39mlogging,

E           �[1;32m     91�[0m         log_file_name�[38;5;241m=�[39mlog_file_name,

E           �[1;32m     92�[0m     )

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:674�[0m, in �[0;36mrmm._lib.memory_resource._initialize�[0;34m()�[0m

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:734�[0m, in �[0;36mrmm._lib.memory_resource._initialize�[0;34m()�[0m

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:272�[0m, in �[0;36mrmm._lib.memory_resource.PoolMemoryResource.__cinit__�[0;34m()�[0m

E

E           �[0;31mMemoryError�[0m: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

E           MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded
/usr/local/lib/python3.8/dist-packages/nbclient/client.py:919: CellExecutionError

----------------------------- Captured stderr call -----------------------------

2022-08-31 14:31:03,200 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:31:03,227 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:31:03,521 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:03,522 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:04,992 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-pofxhuhy', purging

2022-08-31 14:31:04,992 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:31:05,228 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:05,229 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:06,704 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-w9k_9833', purging

2022-08-31 14:31:06,705 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:31:06,746 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:31:06,972 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:06,973 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:07,053 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:07,054 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:08,445 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-7m14e_tk', purging

2022-08-31 14:31:08,445 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-9w67pxbb', purging

2022-08-31 14:31:08,445 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:31:08,679 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:31:08,680 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

/usr/lib/python3.8/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown

warnings.warn('resource_tracker: There appear to be %d '

=============================== warnings summary ===============================

../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_INTEGERSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DOUBLESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_STRINGSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BUCKETSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DECIMALSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_TIMESTAMPSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BINARYSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_COLUMNSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33

/usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

DASK_VERSION = LooseVersion(dask.version)
../../../.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: 34 warnings

/var/jenkins_home/.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

other = LooseVersion(other)
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATATYPE = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE_SECONDARYDEVICEKIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINPUT_FORMAT = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHINPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHOUTPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELOPTIMIZATIONPOLICY_MODELPRIORITY = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELQUEUEPOLICY_TIMEOUTACTION = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELSEQUENCEBATCHING_CONTROL_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER_RESOURCE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html

=========================== short test summary info ============================

FAILED tests/unit/examples/test_scaling_criteo_merlin_models.py::test_func - ...

============ 1 failed, 1 passed, 1 skipped, 135 warnings in 22.02s =============

Build step 'Execute shell' marked build as failure

Performing Post build task...

Match found for : : True

Logical operation result is TRUE

Running script  : #!/bin/bash

cd /var/jenkins_home/

CUDA_VISIBLE_DEVICES=1 python test_res_push.py "https://api.GitHub.com/repos/NVIDIA-Merlin/Merlin/issues/$ghprbPullId/comments" "/var/jenkins_home/jobs/$JOB_NAME/builds/$BUILD_NUMBER/log"

[merlin_merlin] $ /bin/bash /tmp/jenkins5328545049955213368.sh

[karlhigley]

Is this related to the (rmm related?) failures in the Scaling Criteo test?

[jperez999]

rerun tests

[nvidia-merlin-bot]

Click to view CI Results

GitHub pull request #578 of commit bafa365f25ab479fb93df1c588213942439f6e90, no merge conflicts.
Running as SYSTEM
Setting status of bafa365f25ab479fb93df1c588213942439f6e90 to PENDING with url https://10.20.13.93:8080/job/merlin_merlin/396/console and message: 'Pending'
Using context: Jenkins
Building on master in workspace /var/jenkins_home/workspace/merlin_merlin
using credential systems-login
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url https://github.com/NVIDIA-Merlin/Merlin # timeout=10
Fetching upstream changes from https://github.com/NVIDIA-Merlin/Merlin
 > git --version # timeout=10
using GIT_ASKPASS to set credentials login for merlin-systems
 > git fetch --tags --force --progress -- https://github.com/NVIDIA-Merlin/Merlin +refs/pull/578/*:refs/remotes/origin/pr/578/* # timeout=10
 > git rev-parse bafa365f25ab479fb93df1c588213942439f6e90^{commit} # timeout=10
Checking out Revision bafa365f25ab479fb93df1c588213942439f6e90 (detached)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f bafa365f25ab479fb93df1c588213942439f6e90 # timeout=10
Commit message: "remove ENV VAR for tf causing tritonserer failure"
 > git rev-list --no-walk bafa365f25ab479fb93df1c588213942439f6e90 # timeout=10
[merlin_merlin] $ /bin/bash /tmp/jenkins5263587319498346878.sh
============================= test session starts ==============================
platform linux -- Python 3.8.10, pytest-7.1.2, pluggy-1.0.0
rootdir: /var/jenkins_home/workspace/merlin_merlin/merlin
plugins: anyio-3.6.1, xdist-2.5.0, forked-1.4.0, cov-3.0.0
collected 3 items
tests/unit/test_version.py .                                             [ 33%]

tests/unit/examples/test_building_deploying_multi_stage_RecSys.py s      [ 66%]

tests/unit/examples/test_scaling_criteo_merlin_models.py F               [100%]
=================================== FAILURES ===================================

__________________________________ test_func ___________________________________
def test_func():
    with testbook(
        REPO_ROOT / "examples" / "scaling-criteo" / "02-ETL-with-NVTabular.ipynb",
        execute=False,
        timeout=180,
    ) as tb1:
        tb1.inject(
            """
            import os
            os.environ["BASE_DIR"] = "/tmp/input/criteo/"
            os.environ["INPUT_DATA_DIR"] = "/tmp/input/criteo/"
            os.environ["OUTPUT_DATA_DIR"] = "/tmp/output/criteo/"
            os.system("mkdir -p /tmp/input/criteo")
            os.system("mkdir -p /tmp/output/criteo")

            from merlin.datasets.synthetic import generate_data

            train, valid = generate_data("criteo", int(1000000), set_sizes=(0.7, 0.3))

            train.to_ddf().compute().to_parquet('/tmp/input/criteo/day_0.parquet')
            valid.to_ddf().compute().to_parquet('/tmp/input/criteo/day_1.parquet')
            """
        )


      tb1.execute()


tests/unit/examples/test_scaling_criteo_merlin_models.py:37:

/usr/local/lib/python3.8/dist-packages/testbook/client.py:147: in execute

super().execute_cell(cell, index)

/usr/local/lib/python3.8/dist-packages/nbclient/util.py:85: in wrapped

return just_run(coro(*args, **kwargs))

/usr/local/lib/python3.8/dist-packages/nbclient/util.py:60: in just_run

return loop.run_until_complete(coro)

/usr/lib/python3.8/asyncio/base_events.py:616: in run_until_complete

return future.result()

/usr/local/lib/python3.8/dist-packages/nbclient/client.py:1025: in async_execute_cell

await self._check_raise_for_error(cell, cell_index, exec_reply)

self = <testbook.client.TestbookNotebookClient object at 0x7f6551eacbe0>

cell = {'cell_type': 'code', 'execution_count': 7, 'metadata': {'jupyter': {'outputs_hidden': False}, 'execution': {'iopub.st...pool_size=(device_pool_size // 256) * 256\n    )\n\n# Create the distributed client\nclient = Client(cluster)\nclient'}

cell_index = 12

exec_reply = {'buffers': [], 'content': {'ename': 'MemoryError', 'engine_info': {'engine_id': -1, 'engine_uuid': '0eb3b02b-ffbe-4af...e, 'engine': '0eb3b02b-ffbe-4afa-a70b-6c0d3724eaef', 'started': '2022-08-31T14:34:25.403037Z', 'status': 'error'}, ...}
async def _check_raise_for_error(
    self, cell: NotebookNode, cell_index: int, exec_reply: t.Optional[t.Dict]
) -> None:

    if exec_reply is None:
        return None

    exec_reply_content = exec_reply['content']
    if exec_reply_content['status'] != 'error':
        return None

    cell_allows_errors = (not self.force_raise_errors) and (
        self.allow_errors
        or exec_reply_content.get('ename') in self.allow_error_names
        or "raises-exception" in cell.metadata.get("tags", [])
    )
    await run_hook(
        self.on_cell_error, cell=cell, cell_index=cell_index, execute_reply=exec_reply
    )
    if not cell_allows_errors:


      raise CellExecutionError.from_cell_and_msg(cell, exec_reply_content)


E           nbclient.exceptions.CellExecutionError: An error occurred while executing the following cell:

E           ------------------

E           # Dask dashboard

E           dashboard_port = "8787"

E

E           # Deploy a Single-Machine Multi-GPU Cluster

E           protocol = "tcp"  # "tcp" or "ucx"

E           if numba.cuda.is_available():

E               NUM_GPUS = list(range(len(numba.cuda.gpus)))

E           else:

E               NUM_GPUS = []

E           visible_devices = ",".join([str(n) for n in NUM_GPUS])  # Select devices to place workers

E           device_limit_frac = 0.7  # Spill GPU-Worker memory to host at this limit.

E           device_pool_frac = 0.8

E           part_mem_frac = 0.15

E

E           # Use total device size to calculate args.device_limit_frac

E           device_size = device_mem_size(kind="total")

E           device_limit = int(device_limit_frac * device_size)

E           device_pool_size = int(device_pool_frac * device_size)

E           part_size = int(part_mem_frac * device_size)

E

E           # Check if any device memory is already occupied

E           for dev in visible_devices.split(","):

E               fmem = pynvml_mem_size(kind="free", index=int(dev))

E               used = (device_size - fmem) / 1e9

E               if used > 1.0:

E                   warnings.warn(f"BEWARE - {used} GB is already occupied on device {int(dev)}!")

E

E           cluster = None  # (Optional) Specify existing scheduler port

E           if cluster is None:

E               cluster = LocalCUDACluster(

E                   protocol=protocol,

E                   n_workers=len(visible_devices.split(",")),

E                   CUDA_VISIBLE_DEVICES=visible_devices,

E                   device_memory_limit=device_limit,

E                   local_directory=dask_workdir,

E                   dashboard_address=":" + dashboard_port,

E                   rmm_pool_size=(device_pool_size // 256) * 256

E               )

E

E           # Create the distributed client

E           client = Client(cluster)

E           client

E           ------------------

E

E           �[0;31m---------------------------------------------------------------------------�[0m

E           �[0;31mMemoryError�[0m                               Traceback (most recent call last)

E           Input �[0;32mIn [7]�[0m, in �[0;36m<cell line: 29>�[0;34m()�[0m

E           �[1;32m     28�[0m cluster �[38;5;241m=�[39m �[38;5;28;01mNone�[39;00m  �[38;5;66;03m# (Optional) Specify existing scheduler port�[39;00m

E           �[1;32m     29�[0m �[38;5;28;01mif�[39;00m cluster �[38;5;129;01mis�[39;00m �[38;5;28;01mNone�[39;00m:

E           �[0;32m---> 30�[0m     cluster �[38;5;241m=�[39m �[43mLocalCUDACluster�[49m�[43m(�[49m

E           �[1;32m     31�[0m �[43m        �[49m�[43mprotocol�[49m�[38;5;241;43m=�[39;49m�[43mprotocol�[49m�[43m,�[49m

E           �[1;32m     32�[0m �[43m        �[49m�[43mn_workers�[49m�[38;5;241;43m=�[39;49m�[38;5;28;43mlen�[39;49m�[43m(�[49m�[43mvisible_devices�[49m�[38;5;241;43m.�[39;49m�[43msplit�[49m�[43m(�[49m�[38;5;124;43m"�[39;49m�[38;5;124;43m,�[39;49m�[38;5;124;43m"�[39;49m�[43m)�[49m�[43m)�[49m�[43m,�[49m

E           �[1;32m     33�[0m �[43m        �[49m�[43mCUDA_VISIBLE_DEVICES�[49m�[38;5;241;43m=�[39;49m�[43mvisible_devices�[49m�[43m,�[49m

E           �[1;32m     34�[0m �[43m        �[49m�[43mdevice_memory_limit�[49m�[38;5;241;43m=�[39;49m�[43mdevice_limit�[49m�[43m,�[49m

E           �[1;32m     35�[0m �[43m        �[49m�[43mlocal_directory�[49m�[38;5;241;43m=�[39;49m�[43mdask_workdir�[49m�[43m,�[49m

E           �[1;32m     36�[0m �[43m        �[49m�[43mdashboard_address�[49m�[38;5;241;43m=�[39;49m�[38;5;124;43m"�[39;49m�[38;5;124;43m:�[39;49m�[38;5;124;43m"�[39;49m�[43m �[49m�[38;5;241;43m+�[39;49m�[43m �[49m�[43mdashboard_port�[49m�[43m,�[49m

E           �[1;32m     37�[0m �[43m        �[49m�[43mrmm_pool_size�[49m�[38;5;241;43m=�[39;49m�[43m(�[49m�[43mdevice_pool_size�[49m�[43m �[49m�[38;5;241;43m/�[39;49m�[38;5;241;43m/�[39;49m�[43m �[49m�[38;5;241;43m256�[39;49m�[43m)�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[43m �[49m�[38;5;241;43m256�[39;49m

E           �[1;32m     38�[0m �[43m    �[49m�[43m)�[49m

E           �[1;32m     40�[0m �[38;5;66;03m# Create the distributed client�[39;00m

E           �[1;32m     41�[0m client �[38;5;241m=�[39m Client(cluster)

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/dask_cuda/local_cuda_cluster.py:366�[0m, in �[0;36mLocalCUDACluster.__init__�[0;34m(self, CUDA_VISIBLE_DEVICES, n_workers, threads_per_worker, memory_limit, device_memory_limit, data, local_directory, shared_filesystem, protocol, enable_tcp_over_ucx, enable_infiniband, enable_nvlink, enable_rdmacm, rmm_pool_size, rmm_maximum_pool_size, rmm_managed_memory, rmm_async, rmm_log_directory, rmm_track_allocations, jit_unspill, log_spilling, worker_class, pre_import, **kwargs)�[0m

E           �[1;32m    364�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mcuda_visible_devices �[38;5;241m=�[39m CUDA_VISIBLE_DEVICES

E           �[1;32m    365�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mscale(n_workers)

E           �[0;32m--> 366�[0m �[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43msync�[49m�[43m(�[49m�[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43m_correct_state�[49m�[43m)�[49m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:309�[0m, in �[0;36mSyncMethodMixin.sync�[0;34m(self, func, asynchronous, callback_timeout, args, **kwargs)�[0m

E           �[1;32m    307�[0m     �[38;5;28;01mreturn�[39;00m future

E           �[1;32m    308�[0m �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 309�[0m     �[38;5;28;01mreturn�[39;00m �[43msync�[49m�[43m(�[49m

E           �[1;32m    310�[0m �[43m        �[49m�[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43mloop�[49m�[43m,�[49m�[43m �[49m�[43mfunc�[49m�[43m,�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[43margs�[49m�[43m,�[49m�[43m �[49m�[43mcallback_timeout�[49m�[38;5;241;43m=�[39;49m�[43mcallback_timeout�[49m�[43m,�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[38;5;241;43m*�[39;49m�[43mkwargs�[49m

E           �[1;32m    311�[0m �[43m    �[49m�[43m)�[49m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:376�[0m, in �[0;36msync�[0;34m(loop, func, callback_timeout, *args, **kwargs)�[0m

E           �[1;32m    374�[0m �[38;5;28;01mif�[39;00m error:

E           �[1;32m    375�[0m     typ, exc, tb �[38;5;241m=�[39m error

E           �[0;32m--> 376�[0m     �[38;5;28;01mraise�[39;00m exc�[38;5;241m.�[39mwith_traceback(tb)

E           �[1;32m    377�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    378�[0m     �[38;5;28;01mreturn�[39;00m result

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:349�[0m, in �[0;36msync..f�[0;34m()�[0m

E           �[1;32m    347�[0m         future �[38;5;241m=�[39m asyncio�[38;5;241m.�[39mwait_for(future, callback_timeout)

E           �[1;32m    348�[0m     future �[38;5;241m=�[39m asyncio�[38;5;241m.�[39mensure_future(future)

E           �[0;32m--> 349�[0m     result �[38;5;241m=�[39m �[38;5;28;01myield�[39;00m future

E           �[1;32m    350�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    351�[0m     error �[38;5;241m=�[39m sys�[38;5;241m.�[39mexc_info()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/tornado/gen.py:769�[0m, in �[0;36mRunner.run�[0;34m(self)�[0m

E           �[1;32m    766�[0m exc_info �[38;5;241m=�[39m �[38;5;28;01mNone�[39;00m

E           �[1;32m    768�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 769�[0m     value �[38;5;241m=�[39m �[43mfuture�[49m�[38;5;241;43m.�[39;49m�[43mresult�[49m�[43m(�[49m�[43m)�[49m

E           �[1;32m    770�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    771�[0m     exc_info �[38;5;241m=�[39m sys�[38;5;241m.�[39mexc_info()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/deploy/spec.py:352�[0m, in �[0;36mSpecCluster._correct_state_internal�[0;34m(self)�[0m

E           �[1;32m    350�[0m     �[38;5;28;01mfor�[39;00m w �[38;5;129;01min�[39;00m workers:

E           �[1;32m    351�[0m         w�[38;5;241m.�[39m_cluster �[38;5;241m=�[39m weakref�[38;5;241m.�[39mref(�[38;5;28mself�[39m)

E           �[0;32m--> 352�[0m         �[38;5;28;01mawait�[39;00m w  �[38;5;66;03m# for tornado gen.coroutine support�[39;00m

E           �[1;32m    353�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mworkers�[38;5;241m.�[39mupdate(�[38;5;28mdict�[39m(�[38;5;28mzip�[39m(to_open, workers)))

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/core.py:299�[0m, in �[0;36mServer.await.._�[0;34m()�[0m

E           �[1;32m    293�[0m             �[38;5;28;01mraise�[39;00m �[38;5;167;01mTimeoutError�[39;00m(

E           �[1;32m    294�[0m                 �[38;5;124m"�[39m�[38;5;132;01m{}�[39;00m�[38;5;124m failed to start in �[39m�[38;5;132;01m{}�[39;00m�[38;5;124m seconds�[39m�[38;5;124m"�[39m�[38;5;241m.�[39mformat(

E           �[1;32m    295�[0m                     �[38;5;28mtype�[39m(�[38;5;28mself�[39m)�[38;5;241m.�[39m�[38;5;18m__name__�[39m, timeout

E           �[1;32m    296�[0m                 )

E           �[1;32m    297�[0m             )

E           �[1;32m    298�[0m     �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 299�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstart()

E           �[1;32m    300�[0m         �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mrunning

E           �[1;32m    301�[0m �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:347�[0m, in �[0;36mNanny.start�[0;34m(self)�[0m

E           �[1;32m    344�[0m     �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mplugin_add(plugin�[38;5;241m=�[39mplugin, name�[38;5;241m=�[39mname)

E           �[1;32m    346�[0m logger�[38;5;241m.�[39minfo(�[38;5;124m"�[39m�[38;5;124m        Start Nanny at: �[39m�[38;5;132;01m%r�[39;00m�[38;5;124m"�[39m, �[38;5;28mself�[39m�[38;5;241m.�[39maddress)

E           �[0;32m--> 347�[0m response �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39minstantiate()

E           �[1;32m    348�[0m �[38;5;28;01mif�[39;00m response �[38;5;241m==�[39m Status�[38;5;241m.�[39mrunning:

E           �[1;32m    349�[0m     �[38;5;28;01massert�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mworker_address

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:430�[0m, in �[0;36mNanny.instantiate�[0;34m(self)�[0m

E           �[1;32m    428�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    429�[0m     �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 430�[0m         result �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mprocess�[38;5;241m.�[39mstart()

E           �[1;32m    431�[0m     �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    432�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mclose()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:685�[0m, in �[0;36mWorkerProcess.start�[0;34m(self)�[0m

E           �[1;32m    683�[0m     �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstatus

E           �[1;32m    684�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 685�[0m     msg �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39m_wait_until_connected(uid)

E           �[1;32m    686�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    687�[0m     �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mfailed

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:803�[0m, in �[0;36mWorkerProcess._wait_until_connected�[0;34m(self, uid)�[0m

E           �[1;32m    799�[0m �[38;5;28;01mif�[39;00m �[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m �[38;5;129;01min�[39;00m msg:

E           �[1;32m    800�[0m     logger�[38;5;241m.�[39merror(

E           �[1;32m    801�[0m         �[38;5;124m"�[39m�[38;5;124mFailed while trying to start worker process: �[39m�[38;5;132;01m%s�[39;00m�[38;5;124m"�[39m, msg[�[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m]

E           �[1;32m    802�[0m     )

E           �[0;32m--> 803�[0m     �[38;5;28;01mraise�[39;00m msg[�[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m]

E           �[1;32m    804�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    805�[0m     �[38;5;28;01mreturn�[39;00m msg

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:869�[0m, in �[0;36mrun�[0;34m()�[0m

E           �[1;32m    865�[0m �[38;5;124;03m"""�[39;00m

E           �[1;32m    866�[0m �[38;5;124;03mTry to start worker and inform parent of outcome.�[39;00m

E           �[1;32m    867�[0m �[38;5;124;03m"""�[39;00m

E           �[1;32m    868�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 869�[0m     �[38;5;28;01mawait�[39;00m worker

E           �[1;32m    870�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m �[38;5;28;01mas�[39;00m e:

E           �[1;32m    871�[0m     logger�[38;5;241m.�[39mexception(�[38;5;124m"�[39m�[38;5;124mFailed to start worker�[39m�[38;5;124m"�[39m)

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/core.py:299�[0m, in �[0;36m_�[0;34m()�[0m

E           �[1;32m    293�[0m             �[38;5;28;01mraise�[39;00m �[38;5;167;01mTimeoutError�[39;00m(

E           �[1;32m    294�[0m                 �[38;5;124m"�[39m�[38;5;132;01m{}�[39;00m�[38;5;124m failed to start in �[39m�[38;5;132;01m{}�[39;00m�[38;5;124m seconds�[39m�[38;5;124m"�[39m�[38;5;241m.�[39mformat(

E           �[1;32m    295�[0m                     �[38;5;28mtype�[39m(�[38;5;28mself�[39m)�[38;5;241m.�[39m�[38;5;18m__name__�[39m, timeout

E           �[1;32m    296�[0m                 )

E           �[1;32m    297�[0m             )

E           �[1;32m    298�[0m     �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 299�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstart()

E           �[1;32m    300�[0m         �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mrunning

E           �[1;32m    301�[0m �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/worker.py:1372�[0m, in �[0;36mstart�[0;34m()�[0m

E           �[1;32m   1370�[0m         �[38;5;28;01mfor�[39;00m exc �[38;5;129;01min�[39;00m plugins_exceptions:

E           �[1;32m   1371�[0m             logger�[38;5;241m.�[39merror(�[38;5;28mrepr�[39m(exc))

E           �[0;32m-> 1372�[0m     �[38;5;28;01mraise�[39;00m plugins_exceptions[�[38;5;241m0�[39m]

E           �[1;32m   1374�[0m �[38;5;28mself�[39m�[38;5;241m.�[39m_pending_plugins �[38;5;241m=�[39m ()

E           �[1;32m   1376�[0m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39m_register_with_scheduler()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/worker.py:3248�[0m, in �[0;36mplugin_add�[0;34m()�[0m

E           �[1;32m   3246�[0m �[38;5;28;01mif�[39;00m �[38;5;28mhasattr�[39m(plugin, �[38;5;124m"�[39m�[38;5;124msetup�[39m�[38;5;124m"�[39m):

E           �[1;32m   3247�[0m     �[38;5;28;01mtry�[39;00m:

E           �[0;32m-> 3248�[0m         result �[38;5;241m=�[39m plugin�[38;5;241m.�[39msetup(worker�[38;5;241m=�[39m�[38;5;28mself�[39m)

E           �[1;32m   3249�[0m         �[38;5;28;01mif�[39;00m isawaitable(result):

E           �[1;32m   3250�[0m             result �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m result

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py:78�[0m, in �[0;36msetup�[0;34m()�[0m

E           �[1;32m     74�[0m     �[38;5;28;01mimport�[39;00m �[38;5;21;01mrmm�[39;00m

E           �[1;32m     76�[0m     pool_allocator �[38;5;241m=�[39m �[38;5;28;01mFalse�[39;00m �[38;5;28;01mif�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39minitial_pool_size �[38;5;129;01mis�[39;00m �[38;5;28;01mNone�[39;00m �[38;5;28;01melse�[39;00m �[38;5;28;01mTrue�[39;00m

E           �[0;32m---> 78�[0m     rmm�[38;5;241m.�[39mreinitialize(

E           �[1;32m     79�[0m         pool_allocator�[38;5;241m=�[39mpool_allocator,

E           �[1;32m     80�[0m         managed_memory�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mmanaged_memory,

E           �[1;32m     81�[0m         initial_pool_size�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39minitial_pool_size,

E           �[1;32m     82�[0m         maximum_pool_size�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mmaximum_pool_size,

E           �[1;32m     83�[0m         logging�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mlogging,

E           �[1;32m     84�[0m         log_file_name�[38;5;241m=�[39mget_rmm_log_file_name(

E           �[1;32m     85�[0m             worker, �[38;5;28mself�[39m�[38;5;241m.�[39mlogging, �[38;5;28mself�[39m�[38;5;241m.�[39mlog_directory

E           �[1;32m     86�[0m         ),

E           �[1;32m     87�[0m     )

E           �[1;32m     88�[0m �[38;5;28;01mif�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mrmm_track_allocations:

E           �[1;32m     89�[0m     �[38;5;28;01mimport�[39;00m �[38;5;21;01mrmm�[39;00m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/rmm/rmm.py:84�[0m, in �[0;36mreinitialize�[0;34m()�[0m

E           �[1;32m     31�[0m �[38;5;28;01mdef�[39;00m �[38;5;21mreinitialize�[39m(

E           �[1;32m     32�[0m     pool_allocator�[38;5;241m=�[39m�[38;5;28;01mFalse�[39;00m,

E           �[1;32m     33�[0m     managed_memory�[38;5;241m=�[39m�[38;5;28;01mFalse�[39;00m,

E           �[0;32m   (...)�[0m

E           �[1;32m     38�[0m     log_file_name�[38;5;241m=�[39m�[38;5;28;01mNone�[39;00m,

E           �[1;32m     39�[0m ):

E           �[1;32m     40�[0m     �[38;5;124;03m"""�[39;00m

E           �[1;32m     41�[0m �[38;5;124;03m    Finalizes and then initializes RMM using the options passed. Using memory�[39;00m

E           �[1;32m     42�[0m �[38;5;124;03m    from a previous initialization of RMM is undefined behavior and should be�[39;00m

E           �[0;32m   (...)�[0m

E           �[1;32m     82�[0m �[38;5;124;03m    corresponding to each device.�[39;00m

E           �[1;32m     83�[0m �[38;5;124;03m    """�[39;00m

E           �[0;32m---> 84�[0m     rmm�[38;5;241m.�[39mmr�[38;5;241m.�[39m_initialize(

E           �[1;32m     85�[0m         pool_allocator�[38;5;241m=�[39mpool_allocator,

E           �[1;32m     86�[0m         managed_memory�[38;5;241m=�[39mmanaged_memory,

E           �[1;32m     87�[0m         initial_pool_size�[38;5;241m=�[39minitial_pool_size,

E           �[1;32m     88�[0m         maximum_pool_size�[38;5;241m=�[39mmaximum_pool_size,

E           �[1;32m     89�[0m         devices�[38;5;241m=�[39mdevices,

E           �[1;32m     90�[0m         logging�[38;5;241m=�[39mlogging,

E           �[1;32m     91�[0m         log_file_name�[38;5;241m=�[39mlog_file_name,

E           �[1;32m     92�[0m     )

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:674�[0m, in �[0;36mrmm._lib.memory_resource._initialize�[0;34m()�[0m

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:734�[0m, in �[0;36mrmm._lib.memory_resource._initialize�[0;34m()�[0m

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:272�[0m, in �[0;36mrmm._lib.memory_resource.PoolMemoryResource.__cinit__�[0;34m()�[0m

E

E           �[0;31mMemoryError�[0m: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

E           MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded
/usr/local/lib/python3.8/dist-packages/nbclient/client.py:919: CellExecutionError

----------------------------- Captured stderr call -----------------------------

2022-08-31 14:34:26,959 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:34:26,961 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:34:27,305 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:27,307 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:28,727 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-y3jp6uzt', purging

2022-08-31 14:34:28,727 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:34:28,976 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:28,977 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:30,395 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-m9nezujy', purging

2022-08-31 14:34:30,395 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:34:30,447 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:34:30,665 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:30,666 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:30,736 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:30,737 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:32,107 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-trkg_4b1', purging

2022-08-31 14:34:32,108 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-dzkx5b44', purging

2022-08-31 14:34:32,108 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:34:32,351 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:34:32,352 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

/usr/lib/python3.8/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown

warnings.warn('resource_tracker: There appear to be %d '

=============================== warnings summary ===============================

../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_INTEGERSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DOUBLESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_STRINGSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BUCKETSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DECIMALSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_TIMESTAMPSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BINARYSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_COLUMNSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33

/usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

DASK_VERSION = LooseVersion(dask.version)
../../../.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: 34 warnings

/var/jenkins_home/.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

other = LooseVersion(other)
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATATYPE = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE_SECONDARYDEVICEKIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINPUT_FORMAT = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHINPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHOUTPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELOPTIMIZATIONPOLICY_MODELPRIORITY = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELQUEUEPOLICY_TIMEOUTACTION = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELSEQUENCEBATCHING_CONTROL_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER_RESOURCE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html

=========================== short test summary info ============================

FAILED tests/unit/examples/test_scaling_criteo_merlin_models.py::test_func - ...

============ 1 failed, 1 passed, 1 skipped, 135 warnings in 21.64s =============

Build step 'Execute shell' marked build as failure

Performing Post build task...

Match found for : : True

Logical operation result is TRUE

Running script  : #!/bin/bash

cd /var/jenkins_home/

CUDA_VISIBLE_DEVICES=1 python test_res_push.py "https://api.GitHub.com/repos/NVIDIA-Merlin/Merlin/issues/$ghprbPullId/comments" "/var/jenkins_home/jobs/$JOB_NAME/builds/$BUILD_NUMBER/log"

[merlin_merlin] $ /bin/bash /tmp/jenkins5843305117502265374.sh

[jperez999]

rerun tests

[nvidia-merlin-bot]

Click to view CI Results

GitHub pull request #578 of commit bafa365f25ab479fb93df1c588213942439f6e90, no merge conflicts.
Running as SYSTEM
Setting status of bafa365f25ab479fb93df1c588213942439f6e90 to PENDING with url https://10.20.13.93:8080/job/merlin_merlin/397/console and message: 'Pending'
Using context: Jenkins
Building on master in workspace /var/jenkins_home/workspace/merlin_merlin
using credential systems-login
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url https://github.com/NVIDIA-Merlin/Merlin # timeout=10
Fetching upstream changes from https://github.com/NVIDIA-Merlin/Merlin
 > git --version # timeout=10
using GIT_ASKPASS to set credentials login for merlin-systems
 > git fetch --tags --force --progress -- https://github.com/NVIDIA-Merlin/Merlin +refs/pull/578/*:refs/remotes/origin/pr/578/* # timeout=10
 > git rev-parse bafa365f25ab479fb93df1c588213942439f6e90^{commit} # timeout=10
Checking out Revision bafa365f25ab479fb93df1c588213942439f6e90 (detached)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f bafa365f25ab479fb93df1c588213942439f6e90 # timeout=10
Commit message: "remove ENV VAR for tf causing tritonserer failure"
 > git rev-list --no-walk bafa365f25ab479fb93df1c588213942439f6e90 # timeout=10
[merlin_merlin] $ /bin/bash /tmp/jenkins15790778846354691290.sh
============================= test session starts ==============================
platform linux -- Python 3.8.10, pytest-7.1.2, pluggy-1.0.0
rootdir: /var/jenkins_home/workspace/merlin_merlin/merlin
plugins: anyio-3.6.1, xdist-2.5.0, forked-1.4.0, cov-3.0.0
collected 3 items
tests/unit/test_version.py .                                             [ 33%]

tests/unit/examples/test_building_deploying_multi_stage_RecSys.py s      [ 66%]

tests/unit/examples/test_scaling_criteo_merlin_models.py F               [100%]
=================================== FAILURES ===================================

__________________________________ test_func ___________________________________
def test_func():
    with testbook(
        REPO_ROOT / "examples" / "scaling-criteo" / "02-ETL-with-NVTabular.ipynb",
        execute=False,
        timeout=180,
    ) as tb1:
        tb1.inject(
            """
            import os
            os.environ["BASE_DIR"] = "/tmp/input/criteo/"
            os.environ["INPUT_DATA_DIR"] = "/tmp/input/criteo/"
            os.environ["OUTPUT_DATA_DIR"] = "/tmp/output/criteo/"
            os.system("mkdir -p /tmp/input/criteo")
            os.system("mkdir -p /tmp/output/criteo")

            from merlin.datasets.synthetic import generate_data

            train, valid = generate_data("criteo", int(1000000), set_sizes=(0.7, 0.3))

            train.to_ddf().compute().to_parquet('/tmp/input/criteo/day_0.parquet')
            valid.to_ddf().compute().to_parquet('/tmp/input/criteo/day_1.parquet')
            """
        )


      tb1.execute()


tests/unit/examples/test_scaling_criteo_merlin_models.py:37:

/usr/local/lib/python3.8/dist-packages/testbook/client.py:147: in execute

super().execute_cell(cell, index)

/usr/local/lib/python3.8/dist-packages/nbclient/util.py:85: in wrapped

return just_run(coro(*args, **kwargs))

/usr/local/lib/python3.8/dist-packages/nbclient/util.py:60: in just_run

return loop.run_until_complete(coro)

/usr/lib/python3.8/asyncio/base_events.py:616: in run_until_complete

return future.result()

/usr/local/lib/python3.8/dist-packages/nbclient/client.py:1025: in async_execute_cell

await self._check_raise_for_error(cell, cell_index, exec_reply)

self = <testbook.client.TestbookNotebookClient object at 0x7efac0e15af0>

cell = {'cell_type': 'code', 'execution_count': 7, 'metadata': {'jupyter': {'outputs_hidden': False}, 'execution': {'iopub.st...pool_size=(device_pool_size // 256) * 256\n    )\n\n# Create the distributed client\nclient = Client(cluster)\nclient'}

cell_index = 12

exec_reply = {'buffers': [], 'content': {'ename': 'MemoryError', 'engine_info': {'engine_id': -1, 'engine_uuid': 'b9b09ba4-c113-45c...e, 'engine': 'b9b09ba4-c113-45c1-be1e-f6f5a0c4adb2', 'started': '2022-08-31T14:36:55.371588Z', 'status': 'error'}, ...}
async def _check_raise_for_error(
    self, cell: NotebookNode, cell_index: int, exec_reply: t.Optional[t.Dict]
) -> None:

    if exec_reply is None:
        return None

    exec_reply_content = exec_reply['content']
    if exec_reply_content['status'] != 'error':
        return None

    cell_allows_errors = (not self.force_raise_errors) and (
        self.allow_errors
        or exec_reply_content.get('ename') in self.allow_error_names
        or "raises-exception" in cell.metadata.get("tags", [])
    )
    await run_hook(
        self.on_cell_error, cell=cell, cell_index=cell_index, execute_reply=exec_reply
    )
    if not cell_allows_errors:


      raise CellExecutionError.from_cell_and_msg(cell, exec_reply_content)


E           nbclient.exceptions.CellExecutionError: An error occurred while executing the following cell:

E           ------------------

E           # Dask dashboard

E           dashboard_port = "8787"

E

E           # Deploy a Single-Machine Multi-GPU Cluster

E           protocol = "tcp"  # "tcp" or "ucx"

E           if numba.cuda.is_available():

E               NUM_GPUS = list(range(len(numba.cuda.gpus)))

E           else:

E               NUM_GPUS = []

E           visible_devices = ",".join([str(n) for n in NUM_GPUS])  # Select devices to place workers

E           device_limit_frac = 0.7  # Spill GPU-Worker memory to host at this limit.

E           device_pool_frac = 0.8

E           part_mem_frac = 0.15

E

E           # Use total device size to calculate args.device_limit_frac

E           device_size = device_mem_size(kind="total")

E           device_limit = int(device_limit_frac * device_size)

E           device_pool_size = int(device_pool_frac * device_size)

E           part_size = int(part_mem_frac * device_size)

E

E           # Check if any device memory is already occupied

E           for dev in visible_devices.split(","):

E               fmem = pynvml_mem_size(kind="free", index=int(dev))

E               used = (device_size - fmem) / 1e9

E               if used > 1.0:

E                   warnings.warn(f"BEWARE - {used} GB is already occupied on device {int(dev)}!")

E

E           cluster = None  # (Optional) Specify existing scheduler port

E           if cluster is None:

E               cluster = LocalCUDACluster(

E                   protocol=protocol,

E                   n_workers=len(visible_devices.split(",")),

E                   CUDA_VISIBLE_DEVICES=visible_devices,

E                   device_memory_limit=device_limit,

E                   local_directory=dask_workdir,

E                   dashboard_address=":" + dashboard_port,

E                   rmm_pool_size=(device_pool_size // 256) * 256

E               )

E

E           # Create the distributed client

E           client = Client(cluster)

E           client

E           ------------------

E

E           �[0;31m---------------------------------------------------------------------------�[0m

E           �[0;31mMemoryError�[0m                               Traceback (most recent call last)

E           Input �[0;32mIn [7]�[0m, in �[0;36m<cell line: 29>�[0;34m()�[0m

E           �[1;32m     28�[0m cluster �[38;5;241m=�[39m �[38;5;28;01mNone�[39;00m  �[38;5;66;03m# (Optional) Specify existing scheduler port�[39;00m

E           �[1;32m     29�[0m �[38;5;28;01mif�[39;00m cluster �[38;5;129;01mis�[39;00m �[38;5;28;01mNone�[39;00m:

E           �[0;32m---> 30�[0m     cluster �[38;5;241m=�[39m �[43mLocalCUDACluster�[49m�[43m(�[49m

E           �[1;32m     31�[0m �[43m        �[49m�[43mprotocol�[49m�[38;5;241;43m=�[39;49m�[43mprotocol�[49m�[43m,�[49m

E           �[1;32m     32�[0m �[43m        �[49m�[43mn_workers�[49m�[38;5;241;43m=�[39;49m�[38;5;28;43mlen�[39;49m�[43m(�[49m�[43mvisible_devices�[49m�[38;5;241;43m.�[39;49m�[43msplit�[49m�[43m(�[49m�[38;5;124;43m"�[39;49m�[38;5;124;43m,�[39;49m�[38;5;124;43m"�[39;49m�[43m)�[49m�[43m)�[49m�[43m,�[49m

E           �[1;32m     33�[0m �[43m        �[49m�[43mCUDA_VISIBLE_DEVICES�[49m�[38;5;241;43m=�[39;49m�[43mvisible_devices�[49m�[43m,�[49m

E           �[1;32m     34�[0m �[43m        �[49m�[43mdevice_memory_limit�[49m�[38;5;241;43m=�[39;49m�[43mdevice_limit�[49m�[43m,�[49m

E           �[1;32m     35�[0m �[43m        �[49m�[43mlocal_directory�[49m�[38;5;241;43m=�[39;49m�[43mdask_workdir�[49m�[43m,�[49m

E           �[1;32m     36�[0m �[43m        �[49m�[43mdashboard_address�[49m�[38;5;241;43m=�[39;49m�[38;5;124;43m"�[39;49m�[38;5;124;43m:�[39;49m�[38;5;124;43m"�[39;49m�[43m �[49m�[38;5;241;43m+�[39;49m�[43m �[49m�[43mdashboard_port�[49m�[43m,�[49m

E           �[1;32m     37�[0m �[43m        �[49m�[43mrmm_pool_size�[49m�[38;5;241;43m=�[39;49m�[43m(�[49m�[43mdevice_pool_size�[49m�[43m �[49m�[38;5;241;43m/�[39;49m�[38;5;241;43m/�[39;49m�[43m �[49m�[38;5;241;43m256�[39;49m�[43m)�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[43m �[49m�[38;5;241;43m256�[39;49m

E           �[1;32m     38�[0m �[43m    �[49m�[43m)�[49m

E           �[1;32m     40�[0m �[38;5;66;03m# Create the distributed client�[39;00m

E           �[1;32m     41�[0m client �[38;5;241m=�[39m Client(cluster)

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/dask_cuda/local_cuda_cluster.py:366�[0m, in �[0;36mLocalCUDACluster.__init__�[0;34m(self, CUDA_VISIBLE_DEVICES, n_workers, threads_per_worker, memory_limit, device_memory_limit, data, local_directory, shared_filesystem, protocol, enable_tcp_over_ucx, enable_infiniband, enable_nvlink, enable_rdmacm, rmm_pool_size, rmm_maximum_pool_size, rmm_managed_memory, rmm_async, rmm_log_directory, rmm_track_allocations, jit_unspill, log_spilling, worker_class, pre_import, **kwargs)�[0m

E           �[1;32m    364�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mcuda_visible_devices �[38;5;241m=�[39m CUDA_VISIBLE_DEVICES

E           �[1;32m    365�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mscale(n_workers)

E           �[0;32m--> 366�[0m �[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43msync�[49m�[43m(�[49m�[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43m_correct_state�[49m�[43m)�[49m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:309�[0m, in �[0;36mSyncMethodMixin.sync�[0;34m(self, func, asynchronous, callback_timeout, args, **kwargs)�[0m

E           �[1;32m    307�[0m     �[38;5;28;01mreturn�[39;00m future

E           �[1;32m    308�[0m �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 309�[0m     �[38;5;28;01mreturn�[39;00m �[43msync�[49m�[43m(�[49m

E           �[1;32m    310�[0m �[43m        �[49m�[38;5;28;43mself�[39;49m�[38;5;241;43m.�[39;49m�[43mloop�[49m�[43m,�[49m�[43m �[49m�[43mfunc�[49m�[43m,�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[43margs�[49m�[43m,�[49m�[43m �[49m�[43mcallback_timeout�[49m�[38;5;241;43m=�[39;49m�[43mcallback_timeout�[49m�[43m,�[49m�[43m �[49m�[38;5;241;43m�[39;49m�[38;5;241;43m*�[39;49m�[43mkwargs�[49m

E           �[1;32m    311�[0m �[43m    �[49m�[43m)�[49m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:376�[0m, in �[0;36msync�[0;34m(loop, func, callback_timeout, *args, **kwargs)�[0m

E           �[1;32m    374�[0m �[38;5;28;01mif�[39;00m error:

E           �[1;32m    375�[0m     typ, exc, tb �[38;5;241m=�[39m error

E           �[0;32m--> 376�[0m     �[38;5;28;01mraise�[39;00m exc�[38;5;241m.�[39mwith_traceback(tb)

E           �[1;32m    377�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    378�[0m     �[38;5;28;01mreturn�[39;00m result

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/utils.py:349�[0m, in �[0;36msync..f�[0;34m()�[0m

E           �[1;32m    347�[0m         future �[38;5;241m=�[39m asyncio�[38;5;241m.�[39mwait_for(future, callback_timeout)

E           �[1;32m    348�[0m     future �[38;5;241m=�[39m asyncio�[38;5;241m.�[39mensure_future(future)

E           �[0;32m--> 349�[0m     result �[38;5;241m=�[39m �[38;5;28;01myield�[39;00m future

E           �[1;32m    350�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    351�[0m     error �[38;5;241m=�[39m sys�[38;5;241m.�[39mexc_info()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/tornado/gen.py:769�[0m, in �[0;36mRunner.run�[0;34m(self)�[0m

E           �[1;32m    766�[0m exc_info �[38;5;241m=�[39m �[38;5;28;01mNone�[39;00m

E           �[1;32m    768�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 769�[0m     value �[38;5;241m=�[39m �[43mfuture�[49m�[38;5;241;43m.�[39;49m�[43mresult�[49m�[43m(�[49m�[43m)�[49m

E           �[1;32m    770�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    771�[0m     exc_info �[38;5;241m=�[39m sys�[38;5;241m.�[39mexc_info()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/deploy/spec.py:352�[0m, in �[0;36mSpecCluster._correct_state_internal�[0;34m(self)�[0m

E           �[1;32m    350�[0m     �[38;5;28;01mfor�[39;00m w �[38;5;129;01min�[39;00m workers:

E           �[1;32m    351�[0m         w�[38;5;241m.�[39m_cluster �[38;5;241m=�[39m weakref�[38;5;241m.�[39mref(�[38;5;28mself�[39m)

E           �[0;32m--> 352�[0m         �[38;5;28;01mawait�[39;00m w  �[38;5;66;03m# for tornado gen.coroutine support�[39;00m

E           �[1;32m    353�[0m �[38;5;28mself�[39m�[38;5;241m.�[39mworkers�[38;5;241m.�[39mupdate(�[38;5;28mdict�[39m(�[38;5;28mzip�[39m(to_open, workers)))

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/core.py:299�[0m, in �[0;36mServer.await.._�[0;34m()�[0m

E           �[1;32m    293�[0m             �[38;5;28;01mraise�[39;00m �[38;5;167;01mTimeoutError�[39;00m(

E           �[1;32m    294�[0m                 �[38;5;124m"�[39m�[38;5;132;01m{}�[39;00m�[38;5;124m failed to start in �[39m�[38;5;132;01m{}�[39;00m�[38;5;124m seconds�[39m�[38;5;124m"�[39m�[38;5;241m.�[39mformat(

E           �[1;32m    295�[0m                     �[38;5;28mtype�[39m(�[38;5;28mself�[39m)�[38;5;241m.�[39m�[38;5;18m__name__�[39m, timeout

E           �[1;32m    296�[0m                 )

E           �[1;32m    297�[0m             )

E           �[1;32m    298�[0m     �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 299�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstart()

E           �[1;32m    300�[0m         �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mrunning

E           �[1;32m    301�[0m �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:347�[0m, in �[0;36mNanny.start�[0;34m(self)�[0m

E           �[1;32m    344�[0m     �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mplugin_add(plugin�[38;5;241m=�[39mplugin, name�[38;5;241m=�[39mname)

E           �[1;32m    346�[0m logger�[38;5;241m.�[39minfo(�[38;5;124m"�[39m�[38;5;124m        Start Nanny at: �[39m�[38;5;132;01m%r�[39;00m�[38;5;124m"�[39m, �[38;5;28mself�[39m�[38;5;241m.�[39maddress)

E           �[0;32m--> 347�[0m response �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39minstantiate()

E           �[1;32m    348�[0m �[38;5;28;01mif�[39;00m response �[38;5;241m==�[39m Status�[38;5;241m.�[39mrunning:

E           �[1;32m    349�[0m     �[38;5;28;01massert�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mworker_address

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:430�[0m, in �[0;36mNanny.instantiate�[0;34m(self)�[0m

E           �[1;32m    428�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    429�[0m     �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 430�[0m         result �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mprocess�[38;5;241m.�[39mstart()

E           �[1;32m    431�[0m     �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    432�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mclose()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:685�[0m, in �[0;36mWorkerProcess.start�[0;34m(self)�[0m

E           �[1;32m    683�[0m     �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstatus

E           �[1;32m    684�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 685�[0m     msg �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39m_wait_until_connected(uid)

E           �[1;32m    686�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m:

E           �[1;32m    687�[0m     �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mfailed

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:803�[0m, in �[0;36mWorkerProcess._wait_until_connected�[0;34m(self, uid)�[0m

E           �[1;32m    799�[0m �[38;5;28;01mif�[39;00m �[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m �[38;5;129;01min�[39;00m msg:

E           �[1;32m    800�[0m     logger�[38;5;241m.�[39merror(

E           �[1;32m    801�[0m         �[38;5;124m"�[39m�[38;5;124mFailed while trying to start worker process: �[39m�[38;5;132;01m%s�[39;00m�[38;5;124m"�[39m, msg[�[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m]

E           �[1;32m    802�[0m     )

E           �[0;32m--> 803�[0m     �[38;5;28;01mraise�[39;00m msg[�[38;5;124m"�[39m�[38;5;124mexception�[39m�[38;5;124m"�[39m]

E           �[1;32m    804�[0m �[38;5;28;01melse�[39;00m:

E           �[1;32m    805�[0m     �[38;5;28;01mreturn�[39;00m msg

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/nanny.py:869�[0m, in �[0;36mrun�[0;34m()�[0m

E           �[1;32m    865�[0m �[38;5;124;03m"""�[39;00m

E           �[1;32m    866�[0m �[38;5;124;03mTry to start worker and inform parent of outcome.�[39;00m

E           �[1;32m    867�[0m �[38;5;124;03m"""�[39;00m

E           �[1;32m    868�[0m �[38;5;28;01mtry�[39;00m:

E           �[0;32m--> 869�[0m     �[38;5;28;01mawait�[39;00m worker

E           �[1;32m    870�[0m �[38;5;28;01mexcept�[39;00m �[38;5;167;01mException�[39;00m �[38;5;28;01mas�[39;00m e:

E           �[1;32m    871�[0m     logger�[38;5;241m.�[39mexception(�[38;5;124m"�[39m�[38;5;124mFailed to start worker�[39m�[38;5;124m"�[39m)

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/core.py:299�[0m, in �[0;36m_�[0;34m()�[0m

E           �[1;32m    293�[0m             �[38;5;28;01mraise�[39;00m �[38;5;167;01mTimeoutError�[39;00m(

E           �[1;32m    294�[0m                 �[38;5;124m"�[39m�[38;5;132;01m{}�[39;00m�[38;5;124m failed to start in �[39m�[38;5;132;01m{}�[39;00m�[38;5;124m seconds�[39m�[38;5;124m"�[39m�[38;5;241m.�[39mformat(

E           �[1;32m    295�[0m                     �[38;5;28mtype�[39m(�[38;5;28mself�[39m)�[38;5;241m.�[39m�[38;5;18m__name__�[39m, timeout

E           �[1;32m    296�[0m                 )

E           �[1;32m    297�[0m             )

E           �[1;32m    298�[0m     �[38;5;28;01melse�[39;00m:

E           �[0;32m--> 299�[0m         �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mstart()

E           �[1;32m    300�[0m         �[38;5;28mself�[39m�[38;5;241m.�[39mstatus �[38;5;241m=�[39m Status�[38;5;241m.�[39mrunning

E           �[1;32m    301�[0m �[38;5;28;01mreturn�[39;00m �[38;5;28mself�[39m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/worker.py:1372�[0m, in �[0;36mstart�[0;34m()�[0m

E           �[1;32m   1370�[0m         �[38;5;28;01mfor�[39;00m exc �[38;5;129;01min�[39;00m plugins_exceptions:

E           �[1;32m   1371�[0m             logger�[38;5;241m.�[39merror(�[38;5;28mrepr�[39m(exc))

E           �[0;32m-> 1372�[0m     �[38;5;28;01mraise�[39;00m plugins_exceptions[�[38;5;241m0�[39m]

E           �[1;32m   1374�[0m �[38;5;28mself�[39m�[38;5;241m.�[39m_pending_plugins �[38;5;241m=�[39m ()

E           �[1;32m   1376�[0m �[38;5;28;01mawait�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39m_register_with_scheduler()

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/distributed/worker.py:3248�[0m, in �[0;36mplugin_add�[0;34m()�[0m

E           �[1;32m   3246�[0m �[38;5;28;01mif�[39;00m �[38;5;28mhasattr�[39m(plugin, �[38;5;124m"�[39m�[38;5;124msetup�[39m�[38;5;124m"�[39m):

E           �[1;32m   3247�[0m     �[38;5;28;01mtry�[39;00m:

E           �[0;32m-> 3248�[0m         result �[38;5;241m=�[39m plugin�[38;5;241m.�[39msetup(worker�[38;5;241m=�[39m�[38;5;28mself�[39m)

E           �[1;32m   3249�[0m         �[38;5;28;01mif�[39;00m isawaitable(result):

E           �[1;32m   3250�[0m             result �[38;5;241m=�[39m �[38;5;28;01mawait�[39;00m result

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py:78�[0m, in �[0;36msetup�[0;34m()�[0m

E           �[1;32m     74�[0m     �[38;5;28;01mimport�[39;00m �[38;5;21;01mrmm�[39;00m

E           �[1;32m     76�[0m     pool_allocator �[38;5;241m=�[39m �[38;5;28;01mFalse�[39;00m �[38;5;28;01mif�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39minitial_pool_size �[38;5;129;01mis�[39;00m �[38;5;28;01mNone�[39;00m �[38;5;28;01melse�[39;00m �[38;5;28;01mTrue�[39;00m

E           �[0;32m---> 78�[0m     rmm�[38;5;241m.�[39mreinitialize(

E           �[1;32m     79�[0m         pool_allocator�[38;5;241m=�[39mpool_allocator,

E           �[1;32m     80�[0m         managed_memory�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mmanaged_memory,

E           �[1;32m     81�[0m         initial_pool_size�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39minitial_pool_size,

E           �[1;32m     82�[0m         maximum_pool_size�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mmaximum_pool_size,

E           �[1;32m     83�[0m         logging�[38;5;241m=�[39m�[38;5;28mself�[39m�[38;5;241m.�[39mlogging,

E           �[1;32m     84�[0m         log_file_name�[38;5;241m=�[39mget_rmm_log_file_name(

E           �[1;32m     85�[0m             worker, �[38;5;28mself�[39m�[38;5;241m.�[39mlogging, �[38;5;28mself�[39m�[38;5;241m.�[39mlog_directory

E           �[1;32m     86�[0m         ),

E           �[1;32m     87�[0m     )

E           �[1;32m     88�[0m �[38;5;28;01mif�[39;00m �[38;5;28mself�[39m�[38;5;241m.�[39mrmm_track_allocations:

E           �[1;32m     89�[0m     �[38;5;28;01mimport�[39;00m �[38;5;21;01mrmm�[39;00m

E

E           File �[0;32m/usr/local/lib/python3.8/dist-packages/rmm/rmm.py:84�[0m, in �[0;36mreinitialize�[0;34m()�[0m

E           �[1;32m     31�[0m �[38;5;28;01mdef�[39;00m �[38;5;21mreinitialize�[39m(

E           �[1;32m     32�[0m     pool_allocator�[38;5;241m=�[39m�[38;5;28;01mFalse�[39;00m,

E           �[1;32m     33�[0m     managed_memory�[38;5;241m=�[39m�[38;5;28;01mFalse�[39;00m,

E           �[0;32m   (...)�[0m

E           �[1;32m     38�[0m     log_file_name�[38;5;241m=�[39m�[38;5;28;01mNone�[39;00m,

E           �[1;32m     39�[0m ):

E           �[1;32m     40�[0m     �[38;5;124;03m"""�[39;00m

E           �[1;32m     41�[0m �[38;5;124;03m    Finalizes and then initializes RMM using the options passed. Using memory�[39;00m

E           �[1;32m     42�[0m �[38;5;124;03m    from a previous initialization of RMM is undefined behavior and should be�[39;00m

E           �[0;32m   (...)�[0m

E           �[1;32m     82�[0m �[38;5;124;03m    corresponding to each device.�[39;00m

E           �[1;32m     83�[0m �[38;5;124;03m    """�[39;00m

E           �[0;32m---> 84�[0m     rmm�[38;5;241m.�[39mmr�[38;5;241m.�[39m_initialize(

E           �[1;32m     85�[0m         pool_allocator�[38;5;241m=�[39mpool_allocator,

E           �[1;32m     86�[0m         managed_memory�[38;5;241m=�[39mmanaged_memory,

E           �[1;32m     87�[0m         initial_pool_size�[38;5;241m=�[39minitial_pool_size,

E           �[1;32m     88�[0m         maximum_pool_size�[38;5;241m=�[39mmaximum_pool_size,

E           �[1;32m     89�[0m         devices�[38;5;241m=�[39mdevices,

E           �[1;32m     90�[0m         logging�[38;5;241m=�[39mlogging,

E           �[1;32m     91�[0m         log_file_name�[38;5;241m=�[39mlog_file_name,

E           �[1;32m     92�[0m     )

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:674�[0m, in �[0;36mrmm._lib.memory_resource._initialize�[0;34m()�[0m

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:734�[0m, in �[0;36mrmm._lib.memory_resource._initialize�[0;34m()�[0m

E

E           File �[0;32mrmm/_lib/memory_resource.pyx:272�[0m, in �[0;36mrmm._lib.memory_resource.PoolMemoryResource.__cinit__�[0;34m()�[0m

E

E           �[0;31mMemoryError�[0m: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

E           MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded
/usr/local/lib/python3.8/dist-packages/nbclient/client.py:919: CellExecutionError

----------------------------- Captured stderr call -----------------------------

2022-08-31 14:36:56,916 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:36:56,934 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:36:57,227 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:36:57,228 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:36:58,765 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-m77s93zg', purging

2022-08-31 14:36:58,765 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:36:59,011 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:36:59,012 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:37:00,542 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-a825mpju', purging

2022-08-31 14:37:00,543 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:37:00,543 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:37:00,878 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:37:00,879 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:37:00,880 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:37:00,881 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:37:02,339 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-c5ppleln', purging

2022-08-31 14:37:02,340 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/output/criteo/test_dask/workdir/dask-worker-space/worker-phpf3zqg', purging

2022-08-31 14:37:02,340 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

2022-08-31 14:37:02,589 - distributed.utils - ERROR - std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 693, in log_errors

yield

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

2022-08-31 14:37:02,590 - distributed.nanny - ERROR - Failed to start worker

Traceback (most recent call last):

File "/usr/local/lib/python3.8/dist-packages/distributed/nanny.py", line 869, in run

await worker

File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 299, in _

await self.start()

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 1372, in start

raise plugins_exceptions[0]

File "/usr/local/lib/python3.8/dist-packages/distributed/worker.py", line 3248, in plugin_add

result = plugin.setup(worker=self)

File "/usr/local/lib/python3.8/dist-packages/dask_cuda/utils.py", line 78, in setup

rmm.reinitialize(

File "/usr/local/lib/python3.8/dist-packages/rmm/rmm.py", line 84, in reinitialize

rmm.mr._initialize(

File "rmm/_lib/memory_resource.pyx", line 674, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 734, in rmm._lib.memory_resource._initialize

File "rmm/_lib/memory_resource.pyx", line 272, in rmm._lib.memory_resource.PoolMemoryResource.cinit

MemoryError: std::bad_alloc: out_of_memory: RMM failure at:/usr/include/rmm/mr/device/pool_memory_resource.hpp:183: Maximum pool size exceeded

/usr/lib/python3.8/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown

warnings.warn('resource_tracker: There appear to be %d '

=============================== warnings summary ===============================

../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_INTEGERSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DOUBLESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_STRINGSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BUCKETSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DECIMALSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_TIMESTAMPSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BINARYSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_COLUMNSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33

/usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

DASK_VERSION = LooseVersion(dask.version)
../../../.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: 34 warnings

/var/jenkins_home/.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

other = LooseVersion(other)
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATATYPE = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE_SECONDARYDEVICEKIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINPUT_FORMAT = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHINPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHOUTPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELOPTIMIZATIONPOLICY_MODELPRIORITY = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELQUEUEPOLICY_TIMEOUTACTION = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELSEQUENCEBATCHING_CONTROL_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER_RESOURCE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html

=========================== short test summary info ============================

FAILED tests/unit/examples/test_scaling_criteo_merlin_models.py::test_func - ...

============ 1 failed, 1 passed, 1 skipped, 135 warnings in 21.97s =============

Build step 'Execute shell' marked build as failure

Performing Post build task...

Match found for : : True

Logical operation result is TRUE

Running script  : #!/bin/bash

cd /var/jenkins_home/

CUDA_VISIBLE_DEVICES=1 python test_res_push.py "https://api.GitHub.com/repos/NVIDIA-Merlin/Merlin/issues/$ghprbPullId/comments" "/var/jenkins_home/jobs/$JOB_NAME/builds/$BUILD_NUMBER/log"

[merlin_merlin] $ /bin/bash /tmp/jenkins10588184411307687193.sh

[jperez999]

rerun tests

[nvidia-merlin-bot]

Click to view CI Results

GitHub pull request #578 of commit bafa365f25ab479fb93df1c588213942439f6e90, no merge conflicts.
Running as SYSTEM
Setting status of bafa365f25ab479fb93df1c588213942439f6e90 to PENDING with url https://10.20.13.93:8080/job/merlin_merlin/398/console and message: 'Pending'
Using context: Jenkins
Building on master in workspace /var/jenkins_home/workspace/merlin_merlin
using credential systems-login
 > git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
 > git config remote.origin.url https://github.com/NVIDIA-Merlin/Merlin # timeout=10
Fetching upstream changes from https://github.com/NVIDIA-Merlin/Merlin
 > git --version # timeout=10
using GIT_ASKPASS to set credentials login for merlin-systems
 > git fetch --tags --force --progress -- https://github.com/NVIDIA-Merlin/Merlin +refs/pull/578/*:refs/remotes/origin/pr/578/* # timeout=10
 > git rev-parse bafa365f25ab479fb93df1c588213942439f6e90^{commit} # timeout=10
Checking out Revision bafa365f25ab479fb93df1c588213942439f6e90 (detached)
 > git config core.sparsecheckout # timeout=10
 > git checkout -f bafa365f25ab479fb93df1c588213942439f6e90 # timeout=10
Commit message: "remove ENV VAR for tf causing tritonserer failure"
 > git rev-list --no-walk bafa365f25ab479fb93df1c588213942439f6e90 # timeout=10
[merlin_merlin] $ /bin/bash /tmp/jenkins12659458059619321287.sh
============================= test session starts ==============================
platform linux -- Python 3.8.10, pytest-7.1.2, pluggy-1.0.0
rootdir: /var/jenkins_home/workspace/merlin_merlin/merlin
plugins: anyio-3.6.1, xdist-2.5.0, forked-1.4.0, cov-3.0.0
collected 3 items
tests/unit/test_version.py .                                             [ 33%]

tests/unit/examples/test_building_deploying_multi_stage_RecSys.py s      [ 66%]

tests/unit/examples/test_scaling_criteo_merlin_models.py .               [100%]
=============================== warnings summary ===============================

../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:37: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:44: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:51: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:30: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_INTEGERSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:82: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:89: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:96: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:75: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DOUBLESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:127: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:134: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:141: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:120: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_STRINGSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:172: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:165: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BUCKETSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:203: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:210: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:217: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:196: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DECIMALSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:248: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:255: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:241: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATESTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:286: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:293: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:300: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:307: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:279: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_TIMESTAMPSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:338: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:331: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BINARYSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:369: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:376: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:383: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:390: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:397: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:404: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:411: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:418: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:425: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:432: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362

/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata/orc_column_statistics_pb2.py:362: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_COLUMNSTATISTICS = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33

/usr/local/lib/python3.8/dist-packages/dask_cudf/core.py:33: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

DASK_VERSION = LooseVersion(dask.version)
../../../.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: 34 warnings

/var/jenkins_home/.local/lib/python3.8/site-packages/setuptools/_distutils/version.py:346: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.

other = LooseVersion(other)
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:19: DeprecationWarning: Call to deprecated create function FileDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

DESCRIPTOR = _descriptor.FileDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:33: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:37: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:41: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:45: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:49: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:53: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:57: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:61: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:65: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:69: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:73: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:77: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:81: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:85: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:89: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:27: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_DATATYPE = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:125: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:119: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE_SECONDARYDEVICEKIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:143: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:147: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:151: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:155: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:137: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:173: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:177: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:181: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:167: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINPUT_FORMAT = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:199: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:203: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:207: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:211: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:215: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:219: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:193: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHINPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:237: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:231: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_BATCHOUTPUT_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:255: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:259: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:263: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:249: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELOPTIMIZATIONPOLICY_MODELPRIORITY = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:281: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:285: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:275: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELQUEUEPOLICY_TIMEOUTACTION = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:303: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:307: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:311: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:315: DeprecationWarning: Call to deprecated create function EnumValueDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.EnumValueDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:297: DeprecationWarning: Call to deprecated create function EnumDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELSEQUENCEBATCHING_CONTROL_KIND = _descriptor.EnumDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:335: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:342: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:349: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:328: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER_RESOURCE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:379: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:386: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:372: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELRATELIMITER = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:417: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:424: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:410: DeprecationWarning: Call to deprecated create function Descriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_MODELINSTANCEGROUP_SECONDARYDEVICE = _descriptor.Descriptor(
../../../../../usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455

/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/model_config_pb2.py:455: DeprecationWarning: Call to deprecated create function FieldDescriptor(). Note: Create unlinked descriptors is going to go away. Please use get/find descriptors from generated code or query the descriptor_pool.

_descriptor.FieldDescriptor(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html

============ 2 passed, 1 skipped, 135 warnings in 112.07s (0:01:52) ============

Performing Post build task...

Match found for : : True

Logical operation result is TRUE

Running script  : #!/bin/bash

cd /var/jenkins_home/

CUDA_VISIBLE_DEVICES=1 python test_res_push.py "https://api.GitHub.com/repos/NVIDIA-Merlin/Merlin/issues/$ghprbPullId/comments" "/var/jenkins_home/jobs/$JOB_NAME/builds/$BUILD_NUMBER/log"

[merlin_merlin] $ /bin/bash /tmp/jenkins7303426167494417.sh