diff --git a/.ci/install.sh b/.ci/install.sh new file mode 100755 index 0000000..20e60cb --- /dev/null +++ b/.ci/install.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# failure is a natural part of life +set -e + +if [[ "$TASK" == "rpkg" ]]; then + R_PACKAGE_DIR=$(pwd)/r-pkg + R CMD INSTALL \ + --clean \ + ${R_PACKAGE_DIR} +fi + +if [[ "$TASK" == "pypkg" ]]; then + PY_PACKAGE_DIR=$(pwd)/py-pkg + pip install ${PY_PACKAGE_DIR} +fi diff --git a/.ci/report_to_covr.sh b/.ci/report_to_covr.sh new file mode 100755 index 0000000..06dcfa3 --- /dev/null +++ b/.ci/report_to_covr.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# failure is a natural part of life +set -e + +if [[ "$TASK" == "rpkg" ]]; then + Rscript -e " \ + install.packages('covr'); \ + Sys.setenv(NOT_CRAN = 'true'); \ + covr::codecov('r-pkg/') \ + " +fi + +if [[ "$TASK" == "pypkg" ]]; then + echo "This is a Python build. No post-build actions configured." +fi diff --git a/.ci/seed_es_on_travis.sh b/.ci/seed_es_on_travis.sh new file mode 100755 index 0000000..d5ca480 --- /dev/null +++ b/.ci/seed_es_on_travis.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# failure is a natural part of life +set -e + +# set up some parameters +ES_HOST=http://127.0.0.1:9200 +SLEEP_TIL_STARTUP_SECONDS=20 + +# where can you get ES binaries? +ES1_ARCHIVE=https://download.elasticsearch.org/elasticsearch/elasticsearch +ES2_ARCHIVE=https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch +ES5PLUS_ARCHIVE=https://artifacts.elastic.co/downloads/elasticsearch + +# where is the test data? what file has the mapping for the test "shakespeare" index? +TEST_DATA_DIR=$(pwd)/test_data +LEGACY_MAPPING_FILE="${TEST_DATA_DIR}/legacy_shakespeare_mapping.json" +ES5_MAPPING_FILE="${TEST_DATA_DIR}/es5_shakespeare_mapping.json" +ES6_MAPPING_FILE="${TEST_DATA_DIR}/es6_shakespeare_mapping.json" + + +case "$ES_VERSION" in + "") ;; + + "1.0.0") + export ES_VERSION=1.0.0; + export MAPPING_FILE=${LEGACY_MAPPING_FILE}; + export ES_BINARY_URL="${ES1_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "1.4.4") + export ES_VERSION=1.4.4; + export MAPPING_FILE=${LEGACY_MAPPING_FILE}; + export ES_BINARY_URL="${ES1_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "1.7.2") + export ES_VERSION=1.7.2; + export MAPPING_FILE=${LEGACY_MAPPING_FILE}; + export ES_BINARY_URL="${ES1_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "2.0.2") + export ES_VERSION=2.0.2; + export MAPPING_FILE=${LEGACY_MAPPING_FILE}; + export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb" + ;; + + "2.1.2") + export ES_VERSION=2.1.2; + export MAPPING_FILE=${LEGACY_MAPPING_FILE}; + export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb" + ;; + + "2.2.2") + export ES_VERSION=2.2.2; + export MAPPING_FILE=${LEGACY_MAPPING_FILE}; + export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb" + ;; + + "2.3.5") + export ES_VERSION=2.3.5; + export MAPPING_FILE=${LEGACY_MAPPING_FILE}; + export ES_BINARY_URL="${ES2_ARCHIVE}/$ES_VERSION/elasticsearch-$ES_VERSION.deb" + ;; + + "5.0.2") + export ES_VERSION=5.0.2; + export MAPPING_FILE=${ES5_MAPPING_FILE}; + export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "5.3.3") + export ES_VERSION=5.3.3; + export MAPPING_FILE=${ES5_MAPPING_FILE}; + export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "5.4.3") + export ES_VERSION=5.4.3; + export MAPPING_FILE=${ES5_MAPPING_FILE}; + export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "5.6.9") + export ES_VERSION=5.6.9; + export MAPPING_FILE=${ES5_MAPPING_FILE}; + export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "6.0.1") + export ES_VERSION=6.0.1; + export MAPPING_FILE=${ES6_MAPPING_FILE}; + export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "6.1.4") + export ES_VERSION=6.1.4; + export MAPPING_FILE=${ES6_MAPPING_FILE}; + export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + + "6.2.4") + export ES_VERSION=6.2.4; + export MAPPING_FILE=${ES6_MAPPING_FILE}; + export ES_BINARY_URL="${ES5PLUS_ARCHIVE}/elasticsearch-$ES_VERSION.deb" + ;; + esac + +# pull the binary +curl -O ${ES_BINARY_URL} + +# start the service and wait a bit +sudo dpkg \ + -i \ + --force-confnew \ + elasticsearch-$ES_VERSION.deb +sudo service elasticsearch start +sleep ${SLEEP_TIL_STARTUP_SECONDS} +sudo service elasticsearch status + +# seed ES with data +mv ${MAPPING_FILE} shakespeare_mapping.json +echo $(ls) + +curl -X PUT \ + "${ES_HOST}/shakespeare" \ + --silent \ + -H 'Content-Type:application/json' \ + -d @shakespeare_mapping.json + +curl -X PUT \ + "${ES_HOST}/empty_index" \ + --silent \ + -H 'Content-Type:application/json' \ + -d @shakespeare_mapping.json + +mv test_data/sample.json sample.json + +curl -X POST \ + "${ES_HOST}/shakespeare/_bulk" \ + --silent \ + -H 'Content-Type:application/json' \ + --data-binary @sample.json + +# test that the expected data made it +curl -X POST \ + "${ES_HOST}/_refresh" + +curl -X GET \ + "${ES_HOST}/shakespeare/_search?size=1" \ + -H 'Content-Type:application/json' diff --git a/.ci/test.sh b/.ci/test.sh new file mode 100755 index 0000000..30e0e85 --- /dev/null +++ b/.ci/test.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# failure is a natural part of life +set -e + +if [[ "$TASK" == "rpkg" ]]; then + R_PACKAGE_DIR=$(pwd)/r-pkg + R CMD build ${R_PACKAGE_DIR} + R CMD check \ + --as-cran \ + *.tar.gz +fi + +if [[ "$TASK" == "pypkg" ]]; then + PY_PACKAGE_DIR=$(pwd)/py-pkg + pytest \ + --verbose \ + ${PY_PACKAGE_DI} +fi diff --git a/.travis.yml b/.travis.yml index d4da3b9..fd8859f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,15 @@ dist: trusty before_install: - .ci/setup.sh +install: + - .ci/install.sh + +before_script: + - .ci/seed_es_on_travis.sh + +script: + - .ci/test.sh + # Manually specifying each build configuration. # Would be better to figure out how to build a matrix with two # languages but share the ES_VERSION matrix across all builds. @@ -32,408 +41,156 @@ matrix: env: - ES_VERSION=1.0.0 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=1.4.4 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=1.7.2 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=2.0.2 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=2.1.2 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=2.2.2 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=2.3.5 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=5.0.2 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=5.3.3 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=5.4.3 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=5.6.9 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=6.0.1 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=6.1.4 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz - after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' - language: r warnings_are_errors: true cache: packages env: - ES_VERSION=6.2.4 - TASK=rpkg - install: - - Rscript -e "devtools::install('r-pkg')" - script: - - R CMD build r-pkg/ - - R CMD check *.tar.gz after_success: - - Rscript -e 'install.packages("covr"); Sys.setenv(NOT_CRAN = "true"); covr::codecov("r-pkg/")' + - .ci/report_to_covr.sh ################# # Python builds # ################# - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=1.0.0 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=1.4.4 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=1.7.2 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=2.0.2 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=2.1.2 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=2.2.2 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=2.3.5 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=5.0.2 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=5.3.3 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=5.4.3 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=5.6.9 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=6.0.1 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=6.1.4 - TASK=pypkg - language: python python: 3.5 - install: - pip install py-pkg/ - script: - - pytest --verbose py-pkg/ env: - ES_VERSION=6.2.4 - TASK=pypkg - -before_script: - - case "$ES_VERSION" in - "") ;; - - "1.0.0") - export ES_VERSION=1.0.0 ; - export MAPPING_FILE="legacy_shakespeare_mapping.json"; - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "1.4.4") - export ES_VERSION=1.4.4 ; - export MAPPING_FILE="legacy_shakespeare_mapping.json"; - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "1.7.2") - export ES_VERSION=1.7.2 ; - export MAPPING_FILE="legacy_shakespeare_mapping.json"; - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "2.0.2") - export ES_VERSION=2.0.2 ; - export MAPPING_FILE="legacy_shakespeare_mapping.json"; - curl -O https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "2.1.2") - export ES_VERSION=2.1.2 ; - export MAPPING_FILE="legacy_shakespeare_mapping.json"; - curl -O https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "2.2.2") - export ES_VERSION=2.2.2 ; - export MAPPING_FILE="legacy_shakespeare_mapping.json"; - curl -O https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "2.3.5") - export ES_VERSION=2.3.5 ; - export MAPPING_FILE="legacy_shakespeare_mapping.json"; - curl -O https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "5.0.2") - export ES_VERSION=5.0.2 ; - export MAPPING_FILE="es5_shakespeare_mapping.json"; - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "5.3.3") - export ES_VERSION=5.3.3 ; - export MAPPING_FILE="es5_shakespeare_mapping.json"; - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "5.4.3") - export ES_VERSION=5.4.3 ; - export MAPPING_FILE="es5_shakespeare_mapping.json"; - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "5.6.9") - export ES_VERSION=5.6.9 ; - export MAPPING_FILE="es5_shakespeare_mapping.json"; - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "6.0.1") - export ES_VERSION=6.0.1 ; - export MAPPING_FILE="es6_shakespeare_mapping.json"; - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "6.1.4") - export ES_VERSION=6.1.4 ; - export MAPPING_FILE="es6_shakespeare_mapping.json"; - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - - "6.2.4") - export ES_VERSION=6.2.4 ; - export MAPPING_FILE="es6_shakespeare_mapping.json"; - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.deb && sudo dpkg -i --force-confnew elasticsearch-$ES_VERSION.deb && sudo service elasticsearch start - ;; - esac - - - sleep 20 - - sudo service elasticsearch status - - mv test_data/${MAPPING_FILE} shakespeare_mapping.json - - echo $(ls) - - curl --silent -X PUT "http://127.0.0.1:9200/shakespeare" -H 'Content-Type:application/json' -d @shakespeare_mapping.json - - curl --silent -X PUT "http://127.0.0.1:9200/empty_index" -H 'Content-Type:application/json' -d @shakespeare_mapping.json - - mv test_data/sample.json sample.json - - curl --silent -X POST "http://127.0.0.1:9200/shakespeare/_bulk" -H 'Content-Type:application/json' --data-binary @sample.json - - curl -X POST "http://127.0.0.1:9200/_refresh" - - curl -X GET "http://127.0.0.1:9200/shakespeare/_search?size=1" -H 'Content-Type:application/json'