diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 47b76a92..6fa45683 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -19,8 +19,6 @@ jobs: sudo apt-get install mpich python -m pip install --upgrade pip pip install -r requirements.txt - python setup.py build - python setup.py install - name: test-tf-loader-tfrecord run: | touch __init__.py @@ -47,4 +45,4 @@ jobs: mpirun -n 2 python ./src/dlio_benchmark.py workload=unet3d ++workload.framework=tensorflow ++workload.data_reader.data_loader=tensorflow ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=False ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=4 output=`ls -d ./hydra_log/unet3d/* | tail -1` python ./src/dlio_postprocessor.py --output-folder=${output} - cat ${output}/*.txt \ No newline at end of file + cat ${output}/*.txt diff --git a/Dockerfile b/Dockerfile index 7443c3b6..eba223eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,15 @@ -# We may be able to further simplify this Dockerfile, but this works - -# Has tensorflow >= 2.2.0 -FROM nvcr.io/nvidia/tensorflow:20.12-tf2-py3 +FROM python:3.10.8-slim # Add contents of the current directory to /workspace/dlio in the container ADD . /workspace/dlio -# Remove scripts that are used to launch the container -RUN rm /workspace/dlio/start_dlio.sh - WORKDIR /workspace/dlio RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y git vim sysstat + apt-get install -y git vim sysstat && \ + apt-get install -y mpich -RUN pip install --upgrade pip -RUN pip install --disable-pip-version-check -r requirements.txt +RUN python -m pip install --upgrade pip +RUN pip install -r requirements.txt ENV PYTHONPATH="${PYTHONPATH}:/workspace/dlio" - -# Set the timezone in the container to UTC -ENV TZ=Etc/UTC -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone diff --git a/requirements.txt b/requirements.txt index 6008af92..553fd87a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,55 @@ -numpy>=1.19.1 -h5py>=2.10.0 -pandas>=1.1.3 -mpi4py>=3.1.3 -tensorflow>=2.0 -torch>=1.7.0 -torchvision>=0.8.0 -torchaudio>=0.7.0 -pillow>=8.0.0 -hydra-core>=1.1.0 +# Use cpu version of torch +--extra-index-url https://download.pytorch.org/whl/cpu + +absl-py==1.3.0 +antlr4-python3-runtime==4.9.3 +astunparse==1.6.3 +cachetools==5.2.0 +certifi==2022.9.24 +charset-normalizer==2.1.1 +flatbuffers==22.10.26 +gast==0.4.0 +google-auth==2.14.1 +google-auth-oauthlib==0.4.6 +google-pasta==0.2.0 +grpcio==1.51.0 +h5py==3.7.0 +hydra-core==1.2.0 +idna==3.4 +keras==2.11.0 +libclang==14.0.6 +Markdown==3.4.1 +MarkupSafe==2.1.1 +mpi4py==3.1.4 +numpy==1.23.5 +oauthlib==3.2.2 +omegaconf==2.2.3 +opt-einsum==3.3.0 +packaging==21.3 +pandas==1.5.1 +Pillow==9.3.0 +protobuf==3.19.6 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pyparsing==3.0.9 +python-dateutil==2.8.2 +pytz==2022.6 +PyYAML==6.0 +requests==2.28.1 +requests-oauthlib==1.3.1 +rsa==4.9 +six==1.16.0 +tensorboard==2.11.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorflow==2.11.0 +tensorflow-estimator==2.11.0 +tensorflow-io-gcs-filesystem==0.28.0 +termcolor==2.1.1 +torch==1.13.0+cpu +torchaudio==0.13.0+cpu +torchvision==0.14.0+cpu +typing_extensions==4.4.0 +urllib3==1.26.12 +Werkzeug==2.2.2 +wrapt==1.14.1