Packaging for ubuntu bionic

.buildkite/build.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+set -eu
+
+if [ -n "${KX_BUILD_DEBUG-}" ]; then
+  echo "Enabling script debugging..."
+  set -x
+fi
+
+export TIMEFORMAT='🕑 %1lR'
+export DEBEMAIL=support@koordinates.com
+export DEBFULLNAME="Koordinates CI Builder"
+
+echo "Updating changelog..."
+
+DEB_BASE_VERSION=$(cat gdal/VERSION)
+DEB_VERSION="${DEB_BASE_VERSION}+ci${BUILDKITE_BUILD_NUMBER}-$(git show -s --date=format:%Y%m%d --format=git%cd.%h)"
+echo "Debian Package Version: ${DEB_VERSION}"
+
+if [ -n "${BUILDKITE_AGENT_ACCESS_TOKEN-}" ] ; then 
+  buildkite-agent meta-data set deb-base-version "$DEB_BASE_VERSION"
+  buildkite-agent meta-data set deb-version "$DEB_VERSION"
+
+  echo -e ":debian: Package Version: \`${DEB_VERSION}\`" \
+      | buildkite-agent annotate --style info --context deb-version
+fi
+
+time docker run \
+  -v "$(pwd):/src" \
+  -w "/src/gdal" \
+  -e DEBEMAIL \
+  -e DEBFULLNAME \
+  "${ECR}/ci-tools:latest" \
+    dch --distribution bionic --newversion "${DEB_VERSION}" "Koordinates CI build of ${BUILDKITE_COMMIT}: branch=${BUILDKITE_BRANCH} tag=${BUILDKITE_TAG-}"
+
+BUILD_CONTAINER="build-${BUILDKITE_JOB_ID}"
+
+echo "--- Building debian package ..."
+# Uses a docker volume for ccache
+time docker run \
+  --name "${BUILD_CONTAINER}" \
+  -v "$(pwd):/kx/source" \
+  -v "ccache:/ccache" \
+  -e CCACHE_DIR=/ccache \
+  -w "/kx/source/gdal" \
+  "${ECR}/bionicbuild:latest" \
+    /kx/buildscripts/build_binary_package.sh -uc -us
+
+echo "--- Signing debian archives ..."
+time docker run \
+  -v "$(pwd):/src" \
+  -e "GPG_KEY=${APT_GPG_KEY}" \
+  -w "/src" \
+  "${ECR}/ci-tools:latest" \
+    sign-debs "/src/build-bionic/*.deb"
+
+echo "--- Running tests ..."
+TEST_IMAGE="test-${BUILDKITE_JOB_ID}"
+docker commit "${BUILD_CONTAINER}" "${TEST_IMAGE}"
+docker rm "${BUILD_CONTAINER}"
+R=0
+time docker run --rm -i \
+  -v "$(pwd):/src" \
+  -w "/src/autotest" \
+  "${TEST_IMAGE}" \
+  /bin/bash -exs << EOF || R=$?
+DEBIAN_FRONTEND=noninteractive apt-get update -q
+DEBIAN_FRONTEND=noninteractive apt-get install -y curl
+curl --silent https://bootstrap.pypa.io/get-pip.py 'pip<19' | python -
+pip install 'pytest<5'
+
+DEBIAN_FRONTEND=noninteractive dpkg -i ../build-bionic/{gdal-bin,gdal-data,libgdal20,python-gdal,python3-gdal}*.deb
+
+# skip known failures
+rm gcore/rfc30.py
+TRAVIS=YES TRAVIS_BRANCH=ubuntu_1804 pytest -v -k 'not (test_ogr_fgdb_13 or test_vsisync or test_vsis3_init or test_vsis3_1 or test_vsis3_cleanup or test_eedai_GOOGLE_APPLICATION_CREDENTIALS)'
+EOF
+
+if [ $R -ne 0 ]; then
+  echo "^^^ +++"
+  echo "⚠️ Errors running GDAL tests ($R). But may be kinda expected. Check them and edit build.sh to skip?"
+else
+  echo "--- ✅ GDAL tests passed!"
+fi
+

.buildkite/pipeline.yml

@@ -0,0 +1,44 @@
+---
+steps:
+  - label: ':debian: Build & Package & Test'
+    command: ".buildkite/build.sh"
+    artifact_paths:
+      - "build-bionic/*"
+
+  - block: "Release?"
+    prompt: "Tag & Release to archive? Check the test results first!"
+
+  - label: ':git: Add git tag'
+    command: .buildkite/tag.sh
+    plugins:
+      docker#v1.4.0:
+        image: "${ECR}/ci-tools"
+        always-pull: true
+        workdir: /src
+        environment:
+          - GITHUB_TOKEN
+          - BUILDKITE_COMMIT
+          - BUILDKITE_BUILD_NUMBER
+          - BUILDKITE_BRANCH
+
+  - label: ':debian: Publish deb'
+    command:
+      - >
+        aptly-upload
+        --aptly-url https://apt-repo.kx.gd
+        --retries 3
+        --repo kx-builds-bionic
+        --series bionic
+        build-bionic/*.deb
+    retry:
+      automatic: true
+    plugins:
+      artifacts#v1.2.0:
+        download: "build-bionic/*.deb"
+      docker#v1.4.0:
+        image: "${ECR}/ci-tools"
+        always-pull: true
+        workdir: /src
+        environment:
+          - APTLY_UNAME
+          - APTLY_PASSWD

.buildkite/tag.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -eu
+
+if [ -n "${KX_BUILD_DEBUG-}" ]; then
+  echo "Enabling script debugging..."
+  set -x
+fi
+
+DEB_BASE_VERSION="$(buildkite-agent meta-data get deb-base-version)"
+if [ -z "${DEB_BASE_VERSION}" ]; then
+    echo "Missing deb-base-version: ${DEB_BASE_VERSION}"
+    exit 2
+elif [ "${BUILDKITE_COMMIT}" = "HEAD" ]; then
+    echo "Invalid BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
+    exit 2
+fi
+
+TAG="kx-release-${DEB_BASE_VERSION}+ci${BUILDKITE_BUILD_NUMBER}"
+echo "Creating tag: ${TAG} for ${BUILDKITE_COMMIT} ..."
+
+hub release create \
+    -t "${BUILDKITE_COMMIT}" \
+    -m "CI: ${BUILDKITE_BRANCH}.${BUILDKITE_BUILD_NUMBER}" \
+    "${TAG}"
+
+echo -e ":git: Git Tag: \`${TAG}\`" \
+    | buildkite-agent annotate --style info --context git-tag

.gitignore

@@ -19,3 +19,7 @@
 __pycache__/
 .env*/
 .venv*/
+
+# koordinates
+/build-trusty/
+/build-bionic/

Vagrantfile

@@ -64,7 +64,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   ppaRepos = [
     "ppa:openjdk-r/ppa",
-    "ppa:ubuntugis/ubuntugis-unstable",
+    "ppa:ubuntugis/ppa",
+    "ppa:ubuntugis/ubuntugis-testing",
     "ppa:miurahr/gdal-depends"
   ]
 

autotest/cpp/test_ogr.cpp

@@ -1470,4 +1470,76 @@ namespace tut
                strcmp("2001-02-03T04:05:06.789-03:00", pszDateTime) == 0);
         CPLFree(pszDateTime);
     }
+
+    // Test OGRLinearRing::isPointOnRingBoundary()
+    template<>
+    template<>
+    void object::test<16>()
+    {
+        OGRPolygon oPoly;
+        const char* pszPoly = "POLYGON((10 9,11 10,10 11,9 10,10 9))";
+        oPoly.importFromWkt(&pszPoly);
+        auto poRing = oPoly.getExteriorRing();
+
+        // On first vertex
+        {
+            OGRPoint p(10, 9);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // On second vertex
+        {
+            OGRPoint p(11, 10);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // Middle of first segment
+        {
+            OGRPoint p(10.5, 9.5);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // "Before" first segment
+        {
+            OGRPoint p(10-1, 9-1);
+            ensure(!poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // "After" first segment
+        {
+            OGRPoint p(11+1, 10+1);
+            ensure(!poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // On third vertex
+        {
+            OGRPoint p(10, 11);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // Middle of second segment
+        {
+            OGRPoint p(10.5, 10.5);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // On fourth vertex
+        {
+            OGRPoint p(9, 10);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // Middle of third segment
+        {
+            OGRPoint p(9.5, 10.5);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+
+        // Middle of fourth segment
+        {
+            OGRPoint p(9.5, 9.5);
+            ensure(poRing->isPointOnRingBoundary(&p, false));
+        }
+    }
+
 } // namespace tut

autotest/cpp/testblockcachewrite.cpp

@@ -76,37 +76,94 @@ class MyDataset: public GDALDataset
         }
 };
 
-static void thread_func(void* /* unused */ )
+static void thread_func1(void* /* unused */ )
 {
     printf("begin thread\n");
     GDALFlushCacheBlock();
     printf("end of thread\n\n");
 }
 
-int main(int argc, char* argv[])
+static void test1()
 {
     CPLJoinableThread* hThread;
 
+    printf("Start test1\n");
     printf("main thread %p\n", (void*)CPLGetPID());
 
-    argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );
-
-    CPLSetConfigOption("GDAL_CACHEMAX", "0");
-    CPLSetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "ON");
+    GDALSetCacheMax(0);
 
     MyDataset* poDS = new MyDataset();
 
     char buf1[] = { 1 } ;
     CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 0, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));
 
-    hThread = CPLCreateJoinableThread(thread_func, nullptr);
+    hThread = CPLCreateJoinableThread(thread_func1, nullptr);
     CPLSleep(0.3);
     CPL_IGNORE_RET_VAL(GDALRasterIO(GDALGetRasterBand(poDS, 1), GF_Write, 1, 0, 1, 1, buf1, 1, 1, GDT_Byte, 0, 0));
     GDALFlushCacheBlock();
 
     CPLJoinThread(hThread);
 
     delete poDS;
+    printf("End test1\n");
+}
+
+
+static void thread_func2(void* /* unused */)
+{
+    printf("begin thread %p\n", (void*)CPLGetPID());
+    GDALDatasetH hDS = GDALOpen("data/byte.tif", GA_ReadOnly);
+    GByte c = 0;
+    CPL_IGNORE_RET_VAL(GDALDataset::FromHandle(hDS)->GetRasterBand(1)->RasterIO(GF_Read, 0,0,1,1,&c,1,1,GDT_Byte,0,0,nullptr));
+    GDALClose(hDS);
+    printf("end of thread\n\n");
+}
+
+static void test2()
+{
+    printf("Start test2\n");
+    printf("main thread %p\n", (void*)CPLGetPID());
+
+    CPLJoinableThread* hThread;
+
+    CPLSetConfigOption("GDAL_RB_INTERNALIZE_SLEEP_AFTER_DETACH_BEFORE_WRITE", "0.5");
+    GDALSetCacheMax(1000*1000);
+
+    auto poDS = GetGDALDriverManager()->GetDriverByName("GTiff")->Create("/vsimem/foo.tif", 1, 1, 2, GDT_Byte, nullptr);
+    poDS->GetRasterBand(1)->Fill(0);
+    poDS->GetRasterBand(2)->Fill(0);
+    poDS->FlushCache();
+    GDALSetCacheMax(0);
+
+    poDS->GetRasterBand(1)->Fill(1);
+    hThread = CPLCreateJoinableThread(thread_func2, nullptr);
+    CPLSleep(0.2);
+
+    GByte c = 0;
+    CPL_IGNORE_RET_VAL(poDS->GetRasterBand(1)->RasterIO(GF_Read,0,0,1,1,&c,1,1,GDT_Byte,0,0,nullptr));
+    printf("%d\n", c);
+    assert(c == 1);
+    CPLJoinThread(hThread);
+
+    CPLSetConfigOption("GDAL_RB_INTERNALIZE_SLEEP_AFTER_DETACH_BEFORE_WRITE", nullptr);
+    delete poDS;
+    VSIUnlink("/vsimem/foo.tif");
+    printf("End test2\n");
+}
+
+
+int main(int argc, char* argv[])
+{
+    argc = GDALGeneralCmdLineProcessor( argc, &argv, 0 );
+
+    CPLSetConfigOption("GDAL_DEBUG_BLOCK_CACHE", "ON");
+    GDALGetCacheMax();
+
+    GDALAllRegister();
+
+    test1();
+    test2();
+
     GDALDestroyDriverManager();
     CSLDestroy( argv );
 

autotest/cpp/testcopywords.cpp

@@ -630,29 +630,29 @@ int main(int /* argc */, char* /* argv */ [])
             memset(pIn, 0xff, 256);
             for(int i=0;i<17;i++)
             {
-                pIn[spacing*i] = (GByte)i;
+                pIn[spacing*i] = (GByte)(17-i);
             }
             memset(pOut, 0xff, 256);
             GDALCopyWords(pIn, GDT_Byte, spacing,
                         pOut, GDT_Byte, 1,
                         17);
             for(int i=0;i<17;i++)
             {
-                AssertRes(GDT_Byte, i, GDT_Byte, i, pOut[i], __LINE__);
+                AssertRes(GDT_Byte, 17-i, GDT_Byte, 17-i, pOut[i], __LINE__);
             }
 
             memset(pIn, 0xff, 256);
             memset(pOut, 0xff, 256);
             for(int i=0;i<17;i++)
             {
-                pIn[i] = (GByte)i;
+                pIn[i] = (GByte)(17-i);
             }
             GDALCopyWords(pIn, GDT_Byte, 1,
                         pOut, GDT_Byte, spacing,
                         17);
             for(int i=0;i<17;i++)
             {
-                AssertRes(GDT_Byte, i, GDT_Byte, i, pOut[i*spacing], __LINE__);
+                AssertRes(GDT_Byte, 17-i, GDT_Byte, 17-i, pOut[i*spacing], __LINE__);
                 for(int j=1;j<spacing;j++)
                 {
                     AssertRes(GDT_Byte, 0xff, GDT_Byte, 0xff, pOut[i*spacing+j], __LINE__);