Merge branch 'release/0.8.5'

.circleci/config.yml

@@ -5,7 +5,7 @@ _defaults: &defaults
  environment:
  TERM: dumb
  docker:
- - image: s22s/rasterframes-circleci:latest
+ - image: s22s/rasterframes-circleci:9b7682ef
 
 _setenv: &setenv
  name: set CloudRepo credentials
@@ -195,6 +195,7 @@ workflows:
  branches:
  only:
  - /feature\/.*docs.*/
+ - /fix\/.*docs.*/
  - /docs\/.*/
 
  nightly:

RELEASE.md

@@ -0,0 +1,23 @@
+# RasterFrames Release Process
+
+1. Make sure `release-notes.md` is updated.
+2. Use `git flow release start x.y.z` to create release branch.
+3. Manually edit `version.sbt` and `version.py` to set value to `x.y.z` and commit changes.
+4. Do `docker login` if necessary.
+5. `sbt` shell commands: 
+ a. `clean` 
+ b. `test it:test` 
+ c. `makeSite` 
+ d. `publishSigned` (LocationTech credentials required) 
+ e. `sonatypeReleaseAll`. It can take a while, but should eventually show up [here](https://search.maven.org/search?q=g:org.locationtech.rasterframes). 
+ f. `docs/ghpagesPushSite` 
+ g. `rf-notebook/publish` 
+6. `cd pyrasterframes/target/python/dist`
+7. `python3 -m twine upload pyrasterframes-x.y.z-py2.py3-none-any.whl`
+8. Commit any changes that were necessary.
+9. `git-flow finish release`. Make sure to push tags, develop and master
+ branches.
+10. On `develop`, update `version.sbt` and `version.py` to next development
+ version (`x.y.(z+1)-SNAPSHOT` and `x.y.(z+1).dev0`). Commit and push.
+11. In GitHub, create a new release with the created tag. Copy relevant
+ section of release notes into the description.

build.sbt

@@ -34,7 +34,7 @@ lazy val root = project
  .enablePlugins(RFReleasePlugin)
  .settings(
  publish / skip := true,
- clean := clean.dependsOn(`rf-notebook`/clean).value
+ clean := clean.dependsOn(`rf-notebook`/clean, docs/clean).value
  )
 
 lazy val `rf-notebook` = project
@@ -76,7 +76,6 @@ lazy val core = project
  buildInfoObject := "RFBuildInfo",
  buildInfoOptions := Seq(
  BuildInfoOption.ToMap,
- BuildInfoOption.BuildTime,
  BuildInfoOption.ToJson
  )
  )

build/circleci/Dockerfile

@@ -2,6 +2,7 @@ FROM circleci/openjdk:8-jdk
 
 ENV OPENJPEG_VERSION 2.3.1
 ENV GDAL_VERSION 2.4.1
+ENV SPATIALINDEX_VERSION 1.9.3
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
 
 # most of these libraries required for
@@ -74,3 +75,15 @@ RUN \
  sudo make install && \
  sudo ldconfig && \
  cd /tmp && sudo rm -Rf gdal*
+
+# Compile and install libspatialindex
+RUN \
+ cd /tmp && \
+ wget https://github.com/libspatialindex/libspatialindex/releases/download/${SPATIALINDEX_VERSION}/spatialindex-src-${SPATIALINDEX_VERSION}.tar.gz && \
+ tar -xf spatialindex-src-${SPATIALINDEX_VERSION}.tar.gz && \
+ cd spatialindex-src-${SPATIALINDEX_VERSION}/ && \
+ cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+ make && \
+ sudo make install && \
+ sudo ldconfig && \
+ cd /tmp && sudo rm -Rf spatialindex*

core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala

@@ -30,7 +30,7 @@ import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggreg
 
 class RasterRefIT extends TestEnvironment {
  describe("practical subregion reads") {
- ignore("should construct a natural color composite") {
+ it("should construct a natural color composite") {
  import spark.implicits._
  def scene(idx: Int) = URI.create(s"https://landsat-pds.s3.us-west-2.amazonaws.com" +
  s"/c1/L8/176/039/LC08_L1TP_176039_20190703_20190718_01_T1/LC08_L1TP_176039_20190703_20190718_01_T1_B$idx.TIF")
@@ -55,11 +55,11 @@ class RasterRefIT extends TestEnvironment {
  stats.get.dataCells shouldBe > (1000L)
  }
 
- //import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tiled}
- //import geotrellis.raster.io.geotiff.compression.{DeflateCompression, NoCompression}
- //import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
- //val tiffOptions = GeoTiffOptions(Tiled, DeflateCompression, ColorSpace.RGB)
- //MultibandGeoTiff(raster, raster.crs, tiffOptions).write("target/composite.tif")
+ import geotrellis.raster.io.geotiff.compression.DeflateCompression
+ import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
+ import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tiled}
+ val tiffOptions = GeoTiffOptions(Tiled, DeflateCompression, ColorSpace.RGB)
+ MultibandGeoTiff(raster.raster, raster.crs, tiffOptions).write("target/composite.tif")
  }
  }
 }

core/src/main/resources/reference.conf

@@ -17,6 +17,8 @@ vlm.gdal {
  GDAL_CACHEMAX = 512
  GDAL_PAM_ENABLED = "NO"
  CPL_VSIL_CURL_CHUNK_SIZE = 1000000
+ GDAL_HTTP_MAX_RETRY=4
+ GDAL_HTTP_RETRY_DELAY=1
  }
  // set this to `false` if CPL_DEBUG is `ON`
  useExceptions = true

core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala

@@ -38,7 +38,7 @@ import org.locationtech.rasterframes.util.KryoSupport
 @SQLUserDefinedType(udt = classOf[RasterSourceUDT])
 class RasterSourceUDT extends UserDefinedType[RasterSource] {
  import RasterSourceUDT._
- override def typeName = "rf_rastersource"
+ override def typeName = "rastersource"
 
  override def pyUDT: String = "pyrasterframes.rf_types.RasterSourceUDT"
 
@@ -58,7 +58,6 @@ class RasterSourceUDT extends UserDefinedType[RasterSource] {
  }
  .orNull
 
-
  private[sql] override def acceptsType(dataType: DataType) = dataType match {
  case _: RasterSourceUDT ⇒ true
  case _ ⇒ super.acceptsType(dataType)

core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala

@@ -73,7 +73,7 @@ case object TileUDT {
 
  final val typeName: String = "tile"
 
- implicit def tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] {
+ implicit val tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] {
 
  override val schema: StructType = StructType(Seq(
  StructField("cell_context", schemaOf[TileDataContext], true),