Run travis-ci

CMakeLists.txt

@@ -880,6 +880,7 @@ set(SOURCES
         utilities/fault_injection_env.cc
         utilities/fault_injection_fs.cc
         utilities/fault_injection_secondary_cache.cc
+        utilities/flink/flink_compaction_filter.cc
         utilities/leveldb_options/leveldb_options.cc
         utilities/memory/memory_util.cc
         utilities/merge_operators.cc
@@ -1332,6 +1333,7 @@ if(WITH_TESTS)
         utilities/cassandra/cassandra_row_merge_test.cc
         utilities/cassandra/cassandra_serialize_test.cc
         utilities/checkpoint/checkpoint_test.cc
+        utilities/flink/flink_compaction_filter_test.cc
         utilities/memory/memory_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
         utilities/object_registry_test.cc

FROCKSDB-RELEASE.md

@@ -0,0 +1,249 @@
+# FRocksDB Release Process
+
+## Summary
+
+FrocksDB-6.x releases are a fat jar file that contain the following binaries:
+* .so files for linux32 (glibc and musl-libc)
+* .so files for linux64 (glibc and musl-libc)
+* .so files for linux [aarch64](https://en.wikipedia.org/wiki/AArch64) (glibc and musl-libc)
+* .so files for linux [ppc64le](https://en.wikipedia.org/wiki/Ppc64le) (glibc and musl-libc)
+* .jnilib file for Mac OSX
+* .dll for Windows x64
+
+To build the binaries for a FrocksDB release, building on native architectures is advised. Building the binaries for ppc64le and aarch64 *can* be done using QEMU, but you may run into emulation bugs and the build times will be dramatically slower (up to x20).
+
+We recommend building the binaries on environments with at least 4 cores, 16GB RAM and 40GB of storage. The following environments are recommended for use in the build process:
+* Windows x64
+* Linux aarch64
+* Linux ppc64le
+* Mac OSX
+
+## Build for Windows
+
+For the Windows binary build, we recommend using a base [AWS Windows EC2 instance](https://aws.amazon.com/windows/products/ec2/) with 4 cores, 16GB RAM, 40GB storage for the build.
+
+Firstly, install [chocolatey](https://chocolatey.org/install). Once installed, the following required components can be installed using Powershell:
+
+    choco install git.install jdk8 maven visualstudio2017community visualstudio2017-workload-nativedesktop
+
+Open the "Developer Command Prompt for VS 2017" and run the following commands:
+
+    git clone git@github.com:ververica/frocksdb.git
+    cd frocksdb
+    git checkout FRocksDB-6.20.3 # release branch
+    java\crossbuild\build-win.bat
+
+The resulting native binary will be built and available at `build\java\Release\rocksdbjni-shared.dll`. You can also find it under project folder with name `librocksdbjni-win64.dll`.
+The result windows jar is `build\java\rocksdbjni_classes.jar`.
+
+There is also a how-to in CMakeLists.txt.
+
+**Once finished, extract the `librocksdbjni-win64.dll` from the build environment. You will need this .dll in the final crossbuild.**
+
+## Build for aarch64
+
+For the Linux aarch64 binary build, we recommend using a base [AWS Ubuntu Server 20.04 LTS EC2](https://aws.amazon.com/windows/products/ec2/) with a 4 core Arm processor, 16GB RAM, 40GB storage for the build. You can also attempt to build with QEMU on a non-aarch64 processor, but you may run into emulation bugs and very long build times.
+
+### Building in aarch64 environment
+
+First, install the required packages such as Java 8 and make:
+
+     sudo apt-get update
+     sudo apt-get install build-essential openjdk-8-jdk
+
+then, install and setup [Docker](https://docs.docker.com/engine/install/ubuntu/):
+
+     sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release
+
+     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+     echo "deb [arch=arm64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+     sudo apt-get update
+     sudo apt-get install docker-ce docker-ce-cli containerd.io
+
+     sudo groupadd docker
+     sudo usermod -aG docker $USER
+     newgrp docker
+
+Then, clone the FrocksDB repo:
+
+     git clone https://github.com/ververica/frocksdb.git
+     cd frocksdb
+     git checkout FRocksDB-6.20.3 # release branch
+
+
+First, build the glibc binary:
+
+    make jclean clean rocksdbjavastaticdockerarm64v8
+
+**Once finished, extract the `java/target/librocksdbjni-linux-aarch64.so` from the build environment. You will need this .so in the final crossbuild.**
+
+Next, build the musl-libc binary:
+
+     make jclean clean rocksdbjavastaticdockerarm64v8musl
+
+**Once finished, extract the `java/target/librocksdbjni-linux-aarch64-musl.so` from the build environment. You will need this .so in the final crossbuild.**
+
+### Building via QEMU
+
+You can use QEMU on, for example, an `x86_64` system to build the aarch64 binaries. To set this up on an Ubuntu envirnment:
+
+    sudo apt-get install qemu binfmt-support qemu-user-static
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+To verify that you can now run aarch64 docker images:
+
+    docker run --rm -t arm64v8/ubuntu uname -m
+    > aarch64
+
+You can now attempt to build the aarch64 binaries as in the previous section.
+
+## Build in PPC64LE
+
+For the ppc64le binaries, we recommend building on a PowerPC machine if possible, as it can be tricky to spin up a ppc64le cloud environment. However, if a PowerPC machine is not available, [Travis-CI](https://www.travis-ci.com/) offers ppc64le build environments that work perfectly for building these binaries. If neither a machine or Travis are an option, you can use QEMU but the build may take a very long time and be prone to emulation errors.
+
+### Building in ppc64le environment
+
+As with the aarch64 environment, the ppc64le environment will require Java 8, Docker and build-essentials installed. Once installed, you can build the 2 binaries:
+
+    make jclean clean rocksdbjavastaticdockerppc64le
+
+**Once finished, extract the `java/target/librocksdbjni-linux-ppc64le.so` from the build environment. You will need this .so in the final crossbuild.**
+
+    make jclean clean rocksdbjavastaticdockerppc64lemusl
+
+**Once finished, extract the `java/target/librocksdbjni-linux-ppc64le-musl.so` from the build environment. You will need this .so in the final crossbuild.**
+
+### Building via Travis
+
+Travis-CI supports ppc64le build environments, and this can be a convienient way of building in the absence of a PowerPC machine. Assuming that you have an S3 bucket called **my-frocksdb-release-artifacts**, the following Travis configuration will build the release artifacts and push them to the S3 bucket:
+
+```
+dist: xenial
+language: cpp
+os:
+  - linux
+arch:
+  - ppc64le
+
+services:
+  - docker
+addons:
+  artifacts:
+    paths:
+      - $TRAVIS_BUILD_DIR/java/target/librocksdbjni-linux-ppc64le-musl.so
+      - $TRAVIS_BUILD_DIR/java/target/librocksdbjni-linux-ppc64le.so
+
+env:
+  global:
+    - ARTIFACTS_BUCKET=my-rocksdb-release-artifacts
+  jobs:
+    - CMD=rocksdbjavastaticdockerppc64le
+    - CMD=rocksdbjavastaticdockerppc64lemusl
+
+install:
+  - sudo apt-get install -y openjdk-8-jdk || exit $?
+  - export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
+  - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
+  - echo "JAVA_HOME=${JAVA_HOME}"
+  - which java && java -version
+  - which javac && javac -version
+
+script:
+  - make jclean clean $CMD
+```
+
+**Make sure to set the `ARTIFACTS_KEY` and `ARTIFACTS_SECRET` environment variables in the Travis Job with valid AWS credentials to access the S3 bucket you defined.**
+
+**Once finished, the`librocksdbjni-linux-ppce64le.so` and `librocksdbjni-linux-ppce64le-musl.so` binaries will be in the S3 bucket. You will need these .so binaries in the final crossbuild.**
+
+
+### Building via QEMU
+
+You can use QEMU on, for example, an `x86_64` system to build the ppc64le binaries. To set this up on an Ubuntu envirnment:
+
+    sudo apt-get install qemu binfmt-support qemu-user-static
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+To verify that you can now run ppc64le docker images:
+
+    docker run --rm -t ppc64le/ubuntu uname -m
+    > ppc64le
+
+You can now attempt to build the ppc64le binaries as in the previous section.
+
+## Final crossbuild in Mac OSX
+
+Documentation for the final crossbuild for Mac OSX and Linux is described in [java/RELEASE.md](java/RELEASE.md) as has information on dependencies that should be installed. As above, this tends to be Java 8, build-essentials and Docker.
+
+Before you run this step, you should have 5 binaries from the previous build steps:
+
+ 1. `librocksdbjni-win64.dll` from the Windows build step.
+ 2. `librocksdbjni-linux-aarch64.so` from the aarch64 build step.
+ 3. `librocksdbjni-linux-aarch64-musl.so` from the aarch64 build step.
+ 3. `librocksdbjni-linux-ppc64le.so` from the ppc64le build step.
+ 4. `librocksdbjni-linux-ppc64le-musl.so` from the ppc64le build step.
+
+To start the crossbuild within a Mac OSX environment:
+
+    make jclean clean
+    mkdir -p java/target
+    cp <path-to-windows-dll>/librocksdbjni-win64.dll java/target/librocksdbjni-win64.dll
+    cp <path-to-ppc64le-lib-so>/librocksdbjni-linux-ppc64le.so java/target/librocksdbjni-linux-ppc64le.so
+    cp <path-to-ppc64le-musl-lib-so>/librocksdbjni-linux-ppc64le-musl.so java/target/librocksdbjni-linux-ppc64le-musl.so
+    cp <path-to-arm-lib-so>/librocksdbjni-linux-aarch64.so java/target/librocksdbjni-linux-aarch64.so
+    cp <path-to-arm-musl-lib-so>/librocksdbjni-linux-aarch64-musl.so java/target/librocksdbjni-linux-aarch64-musl.so
+    FROCKSDB_VERSION=1.0 PORTABLE=1 ROCKSDB_DISABLE_JEMALLOC=true DEBUG_LEVEL=0 make frocksdbjavastaticreleasedocker
+
+*Note, we disable jemalloc on mac due to https://github.com/facebook/rocksdb/issues/5787*.
+
+Once finished, there should be a directory at `java/target/frocksdb-release` with the FRocksDB jar, javadoc jar, sources jar and pom in it. You can inspect the jar file and ensure that contains the binaries, history file, etc:
+
+```
+$ jar tf frocksdbjni-6.20.3-ververica-1.0.jar
+META-INF/
+META-INF/MANIFEST.MF
+HISTORY-JAVA.md
+HISTORY.md
+librocksdbjni-linux-aarch64-musl.so
+librocksdbjni-linux-aarch64.so
+librocksdbjni-linux-ppc64le-musl.so
+librocksdbjni-linux-ppc64le.so
+librocksdbjni-linux32-musl.so
+librocksdbjni-linux32.so
+librocksdbjni-linux64-musl.so
+librocksdbjni-linux64.so
+librocksdbjni-osx.jnilib
+librocksdbjni-win64.dl
+...
+```
+
+*Note that it contains linux32/64.so binaries as well as librocksdbjni-osx.jnilib*.
+
+## Push to Maven Central
+
+For this step, you will need the following:
+
+- The OSX Crossbuild artifacts built in `java/target/frocksdb-release` as above.
+- A Sonatype account with access to the staging repository. If you do not have permission, open a ticket with Sonatype, [such as this one](https://issues.sonatype.org/browse/OSSRH-72185).
+- A GPG key to sign the release, with your public key available for verification (for example, by uploading it to https://keys.openpgp.org/)
+
+To upload the release to the Sonatype staging repository:
+```bash
+VERSION=<release version> \
+USER=<sonatype user> \
+PASSWORD=<sonatype password> \
+KEYNAME=<gpg key name> \
+PASSPHRASE=<gpg key passphrase> \
+java/publish-frocksdbjni.sh
+```
+
+Go to the staging repositories on Sonatype:
+
+https://oss.sonatype.org/#stagingRepositories
+
+Select the open staging repository and click on "Close".
+
+The staging repository will look something like  `https://oss.sonatype.org/content/repositories/xxxx-1020`. You can use this staged release to test the artifacts and ensure they are correct.
+
+Once you have verified the artifacts are correct, press the "Release" button. **WARNING: this can not be undone**. Within 24-48 hours, the artifact will be available on Maven Central for use.

HISTORY.md

@@ -1,3 +1,9 @@
+# FRocksdb Change Log
+## 6.20.2-ververica-1.0 (08/09/2021)
+### Improvement
+* [Flink TTL] compaction filter for background cleanup of state with time-to-live
+* [FLINK-19710] Revert implementation of PerfContext back to __thread to avoid performance regression
+
 # Rocksdb Change Log
 ## 6.29.5 (03/29/2022)
 ### Bug Fixes

Makefile

@@ -6,6 +6,8 @@
 
 #-----------------------------------------------
 
+FROCKSDB_VERSION ?= 1.0
+
 BASH_EXISTS := $(shell which bash)
 SHELL := $(shell which bash)
 # Default to python3. Some distros like CentOS 8 do not have `python`.
@@ -1441,6 +1443,9 @@ histogram_test: $(OBJ_DIR)/monitoring/histogram_test.o $(TEST_LIBRARY) $(LIBRARY
 thread_local_test: $(OBJ_DIR)/util/thread_local_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+flink_compaction_filter_test: $(OBJ_DIR)/utilities/flink/flink_compaction_filter_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 work_queue_test: $(OBJ_DIR)/util/work_queue_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2086,8 +2091,8 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-sources.jar
 SHA256_CMD = sha256sum
 
-ZLIB_VER ?= 1.2.11
-ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
+ZLIB_VER ?= 1.2.12
+ZLIB_SHA256 ?= 91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9
 ZLIB_DOWNLOAD_BASE ?= http://zlib.net
 BZIP2_VER ?= 1.0.8
 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
@@ -2254,7 +2259,7 @@ rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs
 	cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
 	openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
 
-rocksdbjavastaticosx_archs: 
+rocksdbjavastaticosx_archs:
 	$(MAKE) rocksdbjavastaticosx_arch_x86_64
 	$(MAKE) rocksdbjavastaticosx_arch_arm64
 
@@ -2313,10 +2318,42 @@ rocksdbjavastaticrelease: rocksdbjavastaticosx rocksdbjava_javadocs_jar rocksdbj
 
 rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl rocksdbjava_javadocs_jar rocksdbjava_sources_jar
 	cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
-	cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
+	$(JAR_CMD) -uf java/target/$(ROCKSDB_JAR_ALL) HISTORY*.md
+	cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib librocksdbjni-win64.dll
 	cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
 	openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
 
+frocksdbjavastaticreleasedocker: rocksdbjavastaticreleasedocker
+	# update apache license
+	mkdir -p java/target/META-INF
+	cp LICENSE.Apache java/target/META-INF/LICENSE
+	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) META-INF/LICENSE
+
+	# jars to be released
+	$(eval JAR_PREF=rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH))
+	$(eval JAR_DOCS=$(JAR_PREF)-javadoc.jar)
+	$(eval JAR_SOURCES=$(JAR_PREF)-sources.jar)
+
+	# update docs and sources jars
+	cd java/target;jar -uf $(JAR_DOCS) META-INF/LICENSE
+	cd java/target;jar -uf $(JAR_SOURCES) META-INF/LICENSE
+
+	# prepare frocksdb release
+	cd java/target;mkdir -p frocksdb-release
+
+	$(eval FROCKSDB_JAVA_VERSION=$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-ververica-$(FROCKSDB_VERSION))
+	$(eval FJAR_PREF=frocksdbjni-$(FROCKSDB_JAVA_VERSION))
+	$(eval FJAR=$(FJAR_PREF).jar)
+	$(eval FJAR_DOCS=$(FJAR_PREF)-javadoc.jar)
+	$(eval FJAR_SOURCES=$(FJAR_PREF)-sources.jar)
+
+	cd java/target;cp $(ROCKSDB_JAR_ALL) frocksdb-release/$(FJAR)
+	cd java/target;cp $(JAR_DOCS) frocksdb-release/$(FJAR_DOCS)
+	cd java/target;cp $(JAR_SOURCES) frocksdb-release/$(FJAR_SOURCES)
+	openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
+	cd java;cat pom.xml.template | sed 's/\$${FROCKSDB_JAVA_VERSION}/$(FROCKSDB_JAVA_VERSION)/' > pom.xml
+	cd java;cp pom.xml target/frocksdb-release/$(FJAR_PREF).pom
+
 rocksdbjavastaticdockerx86:
 	mkdir -p java/target
 	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh

TARGETS

@@ -399,6 +399,7 @@ cpp_library(
         "utilities/fault_injection_env.cc",
         "utilities/fault_injection_fs.cc",
         "utilities/fault_injection_secondary_cache.cc",
+        "utilities/flink/flink_compaction_filter.cc",
         "utilities/leveldb_options/leveldb_options.cc",
         "utilities/memory/memory_util.cc",
         "utilities/merge_operators.cc",
@@ -729,6 +730,7 @@ cpp_library(
         "utilities/fault_injection_env.cc",
         "utilities/fault_injection_fs.cc",
         "utilities/fault_injection_secondary_cache.cc",
+        "utilities/flink/flink_compaction_filter.cc",
         "utilities/leveldb_options/leveldb_options.cc",
         "utilities/memory/memory_util.cc",
         "utilities/merge_operators.cc",
@@ -1656,6 +1658,13 @@ ROCKS_TESTS = [
         [],
         [],
     ],
+    [
+        "flink_compaction_filter_test",
+        "utilities/flink/flink_compaction_filter_test.cc",
+        "parallel",
+        [],
+        [],
+    ],
     [
         "flush_job_test",
         "db/flush_job_test.cc",

Vagrantfile

@@ -2,7 +2,7 @@
 Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |v|
-    v.memory = 4096
+    v.memory = 6096
     v.cpus = 2
   end