diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b719d4..ac7f944 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,541 +1,202 @@ -version: 2 +version: 2.1 + +parameters: + pkg_name: + type: string + default: "genutil" + repo_name: + type: string + default: "genutil" + last_stable: + type: string + default: "8.2" + user: + type: string + default: "cdat" + label: + type: string + default: "nightly" + env_name: + type: string + default: "test_genutil" aliases: + - &setup_env + name: setup_env + command: | + export OS_TYPE=`uname` + if [[ $OS_TYPE == "Darwin" ]]; then + export PROJECT_DIR=/Users/distiller/project/workdir/macos + else + export PROJECT_DIR=/home/circleci/project/workdir/linux + fi + echo "export WORKDIR=$PROJECT_DIR/$PY_VER" >> $BASH_ENV + cat $BASH_ENV + source $BASH_ENV + mkdir -p $WORKDIR + - &setup_miniconda name: setup_miniconda command: | - mkdir -p workspace - git clone -b validateNightlyNew git@github.com:CDAT/cdat workspace/cdat + source $BASH_ENV + mkdir -p $WORKDIR + git clone https://github.com/CDAT/cdat.git $WORKDIR/cdat # install_miniconda.py installs miniconda3 under $WORKDIR/miniconda - python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3' + python $WORKDIR/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3' + - &conda_rerender name: conda_rerender command: | - git clone https://github.com/CDAT/conda-recipes.git $WORKDIR/conda-recipes + source $BASH_ENV source $WORKDIR/miniconda/etc/profile.d/conda.sh conda activate base - BUILD_SCRIPT="$WORKDIR/conda-recipes/build_tools/conda_build.py" - python $BUILD_SCRIPT -w $WORKDIR -l $LAST_STABLE -B 0 -p $PKG_NAME -r $REPO_NAME -b $CIRCLE_BRANCH --do_rerender + echo "make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH" + make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH - &conda_build name: conda_build command: | + source $BASH_ENV source $WORKDIR/miniconda/etc/profile.d/conda.sh conda activate base - conda config --add channels conda-forge --force - conda config --add channels cdat/label/nightly --force - conda config --set channel_priority strict - BUILD_SCRIPT="$WORKDIR/conda-recipes/build_tools/conda_build.py" - python $BUILD_SCRIPT -w $WORKDIR -p $PKG_NAME -r $REPO_NAME --build_version $BUILD_VARIANT_VER --do_build + os=`uname` + artifacts_dir="artifacts/artifacts.${os}.py_${PY_VER}" + make conda-build conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR artifact_dir=$PWD/$artifacts_dir build_version=$PY_VER - &setup_run_tests name: setup_run_tests - environment: - PKGS: "cdms2 cdat_info testsrunner udunits2 numpy" - CHANNELS: "-c cdat/label/nightly -c conda-forge" command: | + source $BASH_ENV source $WORKDIR/miniconda/etc/profile.d/conda.sh conda activate base - echo "conda create -y -n test_genutil --use-local $CHANNELS \"$CONDA_PY_VER\" $PKG_NAME $PKGS $COVERAGE_PKGS \"$LIBNETCDF\"" - conda create -y -n test_genutil --use-local $CHANNELS "$CONDA_PY_VER" $PKG_NAME $PKGS $COVERAGE_PKGS "$LIBNETCDF" - conda activate test_genutil - conda list + export CONDA_PY_VER="python=$PY_VER" + export LIBNETCDF_VER="libnetcdf=*=${LIBNETCDF}_*" + + echo "make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs=\"$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS\"" + make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS" + + make conda-dump-env conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB - &run_tests name: run_tests command: | + source $BASH_ENV source $WORKDIR/miniconda/etc/profile.d/conda.sh - conda activate test_genutil - python run_tests.py -H -v2 $COVERAGE + conda activate $ENV_NAME + make run-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME workdir=$WORKDIR + no_output_timeout: 10m - &conda_upload name: conda_upload command: | + source $BASH_ENV source $WORKDIR/miniconda/etc/profile.d/conda.sh conda activate base - UPLOAD_OPTIONS="-t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL" - anaconda $UPLOAD_OPTIONS linux_build_py*/miniconda/conda-bld/linux-64/$PKG_NAME-$VERSION.`date +%Y*`0.tar.bz2 --force - anaconda $UPLOAD_OPTIONS macos_build_py*/miniconda/conda-bld/osx-64/$PKG_NAME-$VERSION.`date +%Y*`0.tar.bz2 --force - - - &run_coveralls - name: run_coveralls - command: | - source $WORKDIR/miniconda/etc/profile.d/conda.sh - conda activate $ENV_NAME - coveralls - conda deactivate - -jobs: - macos_setup: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build - PKG_NAME: "genutil" - REPO_NAME: "genutil" - LAST_STABLE: "8.2" - steps: - - checkout - - run: *setup_miniconda - - run: *conda_rerender - - persist_to_workspace: - root: . - paths: - - macos_build + UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN user=$USER label=$LABEL" + make conda-upload $UPLOAD_OPTIONS conda=$WORKDIR/miniconda/bin/conda artifact_dir="$PWD/artifacts/*/" - linux_setup: +executors: + linux: machine: image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build - PKG_NAME: "genutil" - REPO_NAME: "genutil" - LAST_STABLE: "8.2" - steps: - - checkout - - run: *setup_miniconda - - run: *conda_rerender - - persist_to_workspace: - root: . - paths: - - linux_build - - # - # build - # - macos_genutil_py36: + macos: macos: xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py36 - PKG_NAME: "genutil" - REPO_NAME: "genutil" - BUILD_VARIANT_VER: "3.6" - LAST_STABLE: "8.2" - steps: - - checkout - - run: *setup_miniconda - - run: *conda_rerender - - run: *conda_build - - persist_to_workspace: - root: . - paths: - - macos_build_py36 - - macos_genutil_py37: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py37 - PKG_NAME: "genutil" - REPO_NAME: "genutil" - BUILD_VARIANT_VER: "3.7" - LAST_STABLE: "8.2" - steps: - - checkout - - run: *setup_miniconda - - run: *conda_rerender - - run: *conda_build - - persist_to_workspace: - root: . - paths: - - macos_build_py37 - - macos_genutil_py38: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py38 - PKG_NAME: "genutil" - REPO_NAME: "genutil" - BUILD_VARIANT_VER: "3.8" - LAST_STABLE: "8.2" - steps: - - checkout - - run: *setup_miniconda - - run: *conda_rerender - - run: *conda_build - - persist_to_workspace: - root: . - paths: - - macos_build_py38 - linux_genutil_py36: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py36 - PKG_NAME: "genutil" - REPO_NAME: "genutil" - BUILD_VARIANT_VER: "3.6" - LAST_STABLE: "8.2" - steps: - - checkout - - run: *setup_miniconda - - run: *conda_rerender - - run: *conda_build - - persist_to_workspace: - root: . - paths: - - linux_build_py36 - - linux_genutil_py37: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py37 - PKG_NAME: "genutil" - REPO_NAME: "genutil" - BUILD_VARIANT_VER: "3.7" - LAST_STABLE: "8.2" - steps: - - checkout - - run: *setup_miniconda - - run: *conda_rerender - - run: *conda_build - - persist_to_workspace: - root: . - paths: - - linux_build_py37 - - linux_genutil_py38: - machine: - image: circleci/classic:latest +jobs: + build: + parameters: + os: + type: executor + py_ver: + type: string + executor: << parameters.os >> environment: - WORKDIR: /home/circleci/project/linux_build_py38 - PKG_NAME: "genutil" - REPO_NAME: "genutil" - BUILD_VARIANT_VER: "3.8" - LAST_STABLE: "8.2" + PKG_NAME: << pipeline.parameters.pkg_name >> + REPO_NAME: << pipeline.parameters.repo_name >> + LAST_STABLE: << pipeline.parameters.last_stable >> + PY_VER: << parameters.py_ver >> steps: - checkout + - attach_workspace: + at: . + - run: *setup_env - run: *setup_miniconda - run: *conda_rerender - run: *conda_build - persist_to_workspace: root: . paths: - - linux_build_py38 - - - # - # run tests with libnetcdf nompi - # - macos_genutil_nompi_py36: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py36 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.6,<3.7" - LIBNETCDF: "libnetcdf=*=nompi_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html - - macos_genutil_nompi_py37: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py37 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.7,<3.8" - LIBNETCDF: "libnetcdf=*=nompi_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html - - macos_genutil_nompi_py38: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py38 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.8,<3.9" - LIBNETCDF: "libnetcdf=*=nompi_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html - - linux_genutil_nompi_py36: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py36 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.6,<3.7" - LIBNETCDF: "libnetcdf=*=nompi_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html - - linux_genutil_nompi_py37: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py37 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.7,<3.8" - COVERAGE: "-c tests/coverage.json --coverage-from-egg" - COVERAGE_PKGS: "coverage coveralls" - LIBNETCDF: "libnetcdf=*=nompi_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - run: *run_coveralls - - store_artifacts: - path: tests_html - destination: tests_html - - linux_genutil_nompi_py38: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py38 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.8,<3.9" - LIBNETCDF: "libnetcdf=*=nompi_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html - - # - # run tests with libnetcdf mpich - # - macos_genutil_mpich_py36: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py36 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.6,<3.7" - LIBNETCDF: "libnetcdf=*=mpi_mpich_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html - - macos_genutil_mpich_py37: - macos: - xcode: "11.4.0" - environment: - WORKDIR: /Users/distiller/project/macos_build_py37 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.7,<3.8" - LIBNETCDF: "libnetcdf=*=mpi_mpich_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html + - workdir + - artifacts - macos_genutil_mpich_py38: - macos: - xcode: "11.4.0" + test: + parameters: + os: + type: executor + py_ver: + type: string + libnetcdf: + type: string + executor: << parameters.os >> environment: - WORKDIR: /Users/distiller/project/macos_build_py38 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.8,<3.9" - LIBNETCDF: "libnetcdf=*=mpi_mpich_*" + PKG_NAME: << pipeline.parameters.pkg_name >> + ENV_NAME: << pipeline.parameters.env_name >> + PY_VER: << parameters.py_ver >> + LIBNETCDF: << parameters.libnetcdf >> steps: - checkout - attach_workspace: at: . + - run: echo "LIBNETCDF=$LIBNETCDF" >> $BASH_ENV + - run: *setup_env - run: *setup_run_tests - run: *run_tests - store_artifacts: path: tests_html destination: tests_html - - linux_genutil_mpich_py36: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py36 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.6,<3.7" - LIBNETCDF: "libnetcdf=*=mpi_mpich_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - store_artifacts: - path: tests_html - destination: tests_html - - linux_genutil_mpich_py37: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py37 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.7,<3.8" - LIBNETCDF: "libnetcdf=*=mpi_mpich_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html - - linux_genutil_mpich_py38: - machine: - image: circleci/classic:latest - environment: - WORKDIR: /home/circleci/project/linux_build_py38 - PKG_NAME: "genutil" - ENV_NAME: "test_genutil" - CONDA_PY_VER: "python>=3.8,<3.9" - LIBNETCDF: "libnetcdf=*=mpi_mpich_*" - steps: - - checkout - - attach_workspace: - at: . - - run: *setup_run_tests - - run: *run_tests - - store_artifacts: - path: tests_html - destination: tests_html + path: spec_artifacts + destination: spec_artifacts upload: machine: image: circleci/classic:latest environment: - WORKDIR: /home/circleci/project/linux_build_py36 - PKG_NAME: "genutil" - VERSION: "8.2" - USER: "cdat" - LABEL: "nightly" + USER: << pipeline.parameters.user >> + LABEL: << pipeline.parameters.label >> + PY_VER: "3.7" steps: + - checkout - attach_workspace: at: . - - run: pwd - - run: ls -l + - run: *setup_env - run: *conda_upload workflows: - version: 2 genutil: jobs: - - macos_genutil_py36 - - macos_genutil_py37 - - macos_genutil_py38 - - linux_genutil_py36 - - linux_genutil_py37 - - linux_genutil_py38 - - - macos_genutil_mpich_py36: - requires: - - macos_genutil_py36 - - macos_genutil_mpich_py37: - requires: - - macos_genutil_py37 - - macos_genutil_mpich_py38: + - build: + matrix: + parameters: + os: [ linux, macos ] + py_ver: [ "3.6", "3.7", "3.8" ] + name: build-<< matrix.os >>-<< matrix.py_ver >> + + - test: + matrix: + parameters: + os: [ linux, macos ] + py_ver: [ "3.6", "3.7", "3.8" ] + libnetcdf: [ "nompi", "mpi_mpich", "mpi_openmpi" ] + name: test-<< matrix.os >>-<< matrix.py_ver >>-<< matrix.libnetcdf >> requires: - - macos_genutil_py38 - - linux_genutil_mpich_py36: - requires: - - linux_genutil_py36 - - linux_genutil_mpich_py37: - requires: - - linux_genutil_py37 - - linux_genutil_mpich_py38: - requires: - - linux_genutil_py38 - - - macos_genutil_nompi_py36: - requires: - - macos_genutil_py36 - - macos_genutil_nompi_py37: - requires: - - macos_genutil_py37 - - macos_genutil_nompi_py38: - requires: - - macos_genutil_py38 - - linux_genutil_nompi_py36: - requires: - - linux_genutil_py36 - - linux_genutil_nompi_py37: - requires: - - linux_genutil_py37 - - linux_genutil_nompi_py38: - requires: - - linux_genutil_py38 + - build-<< matrix.os >>-<< matrix.py_ver >> - upload: requires: - - macos_genutil_nompi_py36 - - macos_genutil_nompi_py37 - - macos_genutil_nompi_py38 - - linux_genutil_nompi_py36 - - linux_genutil_nompi_py37 - - linux_genutil_nompi_py38 - - - macos_genutil_mpich_py36 - - macos_genutil_mpich_py37 - - macos_genutil_mpich_py38 - - linux_genutil_mpich_py36 - - linux_genutil_mpich_py37 - - linux_genutil_mpich_py38 + - test filters: branches: only: master diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e5e2a4b --- /dev/null +++ b/Makefile @@ -0,0 +1,83 @@ +.PHONY: conda-info conda-list setup-build setup-tests conda-rerender \ + conda-build conda-upload conda-dump-env \ + run-tests run-coveralls + +SHELL = /bin/bash + +os = $(shell uname) +pkg_name = genutil +repo_name = genutil +build_script = conda-recipes/build_tools/conda_build.py + +test_pkgs = testsrunner +last_stable ?= 8.2 + +conda_env ?= base +workdir ?= $(PWD)/workspace +branch ?= $(shell git rev-parse --abbrev-ref HEAD) +extra_channels ?= cdat/label/nightly conda-forge +conda ?= $(or $(CONDA_EXE),$(shell find /opt/*conda*/bin $(HOME)/*conda* -type f -iname conda)) +artifact_dir ?= $(PWD)/artifacts +conda_env_filename ?= spec-file +build_version ?= 3.7 + +ifneq ($(coverage),) +coverage = -c tests/coverage.json --coverage-from-egg +endif + +conda_recipes_branch ?= master + +conda_base = $(patsubst %/bin/conda,%,$(conda)) +conda_activate = $(conda_base)/bin/activate + +conda_build_extra = --copy_conda_package $(artifact_dir)/ + +ifndef $(local_repo) +local_repo = $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) +endif + +conda-info: + source $(conda_activate) $(conda_env); conda info + +conda-list: + source $(conda_activate) $(conda_env); conda list + +setup-build: +ifeq ($(wildcard $(workdir)/conda-recipes),) + git clone -b $(conda_recipes_branch) https://github.com/CDAT/conda-recipes $(workdir)/conda-recipes +else + cd $(workdir)/conda-recipes; git pull +endif + +setup-tests: + source $(conda_activate) base; conda create -y -n $(conda_env) --use-local \ + $(foreach x,$(extra_channels),-c $(x)) $(pkg_name) $(foreach x,$(test_pkgs),"$(x)") \ + $(foreach x,$(extra_pkgs),"$(x)") + +conda-rerender: setup-build + python $(workdir)/$(build_script) -w $(workdir) -l $(last_stable) -B 0 -p $(pkg_name) -r $(repo_name) \ + -b $(branch) --do_rerender --conda_env $(conda_env) --ignore_conda_missmatch \ + --conda_activate $(conda_activate) + +conda-build: + mkdir -p $(artifact_dir) + + python $(workdir)/$(build_script) -w $(workdir) -p $(pkg_name) --build_version $(build_version) \ + --do_build --conda_env $(conda_env) --extra_channels $(extra_channels) \ + --conda_activate $(conda_activate) $(conda_build_extra) + +conda-upload: + source $(conda_activate) $(conda_env); \ + anaconda -t $(conda_upload_token) upload -u $(user) -l $(label) --force $(artifact_dir)/*.tar.bz2 + + +conda-dump-env: + mkdir -p $(artifact_dir) + + source $(conda_activate) $(conda_env); conda list --explicit > $(artifact_dir)/$(conda_env_filename).txt + +run-tests: + source $(conda_activate) $(conda_env); python run_tests.py -H -v2 $(coverage) + +run-coveralls: + source $(conda_activate) $(conda_env); coveralls; diff --git a/recipe/meta.yaml.in b/recipe/meta.yaml.in index d6eab1c..baefb75 100644 --- a/recipe/meta.yaml.in +++ b/recipe/meta.yaml.in @@ -14,10 +14,11 @@ requirements: - {{ compiler("c") }} host: - python + - pip - cython + - setuptools - numpy - udunits2 - - cdms2 run: - python - cdms2