Merge remote-tracking branch 'apache-master/ignite-2.5-master' into i…

…gnite-2.5-master

bin/control.bat

@@ -156,6 +156,11 @@ if %ERRORLEVEL% equ 0 (
     if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms256m -Xmx1g
 )
 
+::
+:: Uncomment to enable experimental commands [--wal]
+::
+:: set JVM_OPTS=%JVM_OPTS% -DIGNITE_ENABLE_EXPERIMENTAL_COMMAND=true
+
 ::
 :: Uncomment the following GC settings if you see spikes in your throughput due to Garbage Collection.
 ::

docker/web-console/standalone/Dockerfile

@@ -1,11 +1,12 @@
 #
-# Copyright 2019 GridGain Systems, Inc. and Contributors.
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
 #
-# Licensed under the GridGain Community Edition License (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
+#      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/diagnostic/pagelocktracker/JmhPageLockTrackerBenchmark.java

@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jmh.diagnostic.pagelocktracker;
+
+import org.apache.ignite.internal.benchmarks.jmh.diagnostic.pagelocktracker.stack.LockTrackerNoBarrier;
+import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory;
+import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTracker;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.HEAP_LOG;
+import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.HEAP_STACK;
+import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.OFF_HEAP_LOG;
+import static org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.LockTrackerFactory.OFF_HEAP_STACK;
+
+/**
+ * Benchmark PageLockTracker (factory LockTrackerFactory)
+ */
+public class JmhPageLockTrackerBenchmark {
+    /**
+     * @param args Params.
+     */
+    public static void main(String[] args) throws Exception {
+        Options opt = new OptionsBuilder()
+            .include(JmhPageLockTrackerBenchmark.class.getSimpleName())
+            .build();
+
+        new Runner(opt).run();
+    }
+
+    /** */
+    @State(Scope.Thread)
+    public static class ThreadLocalState {
+        PageLockListener pl;
+
+        @Param({"2", "4", "8", "16"})
+        int stackSize;
+
+        @Param({
+            "HeapArrayLockStack",
+            "HeapArrayLockLog",
+            "OffHeapLockStack",
+            "OffHeapLockLog"
+        })
+        String type;
+
+        @Param({"true", "false"})
+        boolean barrier;
+
+        int StructureId = 123;
+
+        @Setup
+        public void doSetup() {
+            pl = create(Thread.currentThread().getName(), type, barrier);
+        }
+    }
+
+    /**
+     *  Mesure cost for (beforelock -> lock -> unlock) operation.
+     */
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @Fork(1)
+    @Warmup(iterations = 10)
+    @Measurement(iterations = 10)
+    //@OutputTimeUnit(TimeUnit.MICROSECONDS)
+    public void lockUnlock(ThreadLocalState localState) {
+        PageLockListener pl = localState.pl;
+
+        for (int i = 0; i < localState.stackSize; i++) {
+            int pageId = i + 1;
+
+            pl.onBeforeReadLock(localState.StructureId, pageId, pageId);
+
+            pl.onReadLock(localState.StructureId, pageId, pageId, pageId);
+        }
+
+        for (int i = localState.stackSize; i > 0; i--) {
+            int pageId = i;
+
+            pl.onReadUnlock(localState.StructureId, pageId, pageId, pageId);
+        }
+    }
+
+    /**
+     * Factory method.
+     *
+     * @param name Lock tracer name.
+     * @param type Lock tracer type.
+     * @param barrier If {@code True} use real implementation,
+     * if {@code False} use implementation with safety dump barrier.
+     * @return Page lock tracker as PageLockListener.
+     */
+    private static PageLockListener create(String name, String type, boolean barrier) {
+        PageLockTracker tracker;
+
+        switch (type) {
+            case "HeapArrayLockStack":
+                tracker = LockTrackerFactory.create(HEAP_STACK, name);
+                break;
+            case "HeapArrayLockLog":
+                tracker = LockTrackerFactory.create(HEAP_LOG, name);
+                break;
+            case "OffHeapLockStack":
+                tracker = LockTrackerFactory.create(OFF_HEAP_STACK, name);
+                break;
+
+            case "OffHeapLockLog":
+                tracker = LockTrackerFactory.create(OFF_HEAP_LOG, name);
+                break;
+            default:
+                throw new IllegalArgumentException("type:" + type);
+        }
+
+        return barrier ? tracker : new LockTrackerNoBarrier(tracker);
+    }
+}

modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/diagnostic/pagelocktracker/stack/LockTrackerNoBarrier.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jmh.diagnostic.pagelocktracker.stack;
+
+import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTracker;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
+
+/**
+ * Local without barrier syncronization on operation.
+ */
+public class LockTrackerNoBarrier implements PageLockListener {
+    /** */
+    private final PageLockTracker delegate;
+
+    /** */
+    public LockTrackerNoBarrier(
+        PageLockTracker delegate
+    ) {
+        this.delegate = delegate;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onBeforeWriteLock(int cacheId, long pageId, long page) {
+        delegate.onBeforeWriteLock0(cacheId, pageId, page);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onWriteLock(int cacheId, long pageId, long page, long pageAddr) {
+        delegate.onWriteLock0(cacheId, pageId, page, pageAddr);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onWriteUnlock(int cacheId, long pageId, long page, long pageAddr) {
+        delegate.onWriteUnlock0(cacheId, pageId, page, pageAddr);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onBeforeReadLock(int cacheId, long pageId, long page) {
+        delegate.onBeforeReadLock0(cacheId, pageId, page);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onReadLock(int cacheId, long pageId, long page, long pageAddr) {
+        delegate.onReadLock0(cacheId, pageId, page, pageAddr);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onReadUnlock(int cacheId, long pageId, long page, long pageAddr) {
+        delegate.onReadUnlock(cacheId, pageId, page, pageAddr);
+    }
+}

modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java

@@ -175,8 +175,19 @@ protected static class TestTree extends BPlusTree<Long, Long> {
          */
         TestTree(ReuseList reuseList, int cacheId, PageMemory pageMem, long metaPageId)
             throws IgniteCheckedException {
-            super("test", cacheId, pageMem, null, new AtomicLong(), metaPageId, reuseList,
-                new IOVersions<>(new LongInnerIO()), new IOVersions<>(new LongLeafIO()), null);
+            super(
+                "test",
+                cacheId,
+                pageMem,
+                null,
+                new AtomicLong(),
+                metaPageId,
+                reuseList,
+                new IOVersions<>(new LongInnerIO()),
+                new IOVersions<>(new LongLeafIO()),
+                null,
+                null
+            );
 
             PageIO.registerTest(latestInnerIO(), latestLeafIO());
 

modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jol;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.openjdk.jol.info.GraphLayout;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE;
+
+/**
+ *
+ */
+public class FileStoreHeapUtilizationJolBenchmark {
+    /** */
+    private void benchmark() throws IgniteCheckedException {
+        FilePageStoreFactory factory = new FileVersionCheckingFactory(
+            new AsyncFileIOFactory(),
+            new AsyncFileIOFactory(),
+            new DataStorageConfiguration()
+                .setPageSize(4096)
+        );
+
+        List<PageStore> stores = new LinkedList<>();
+
+        File workDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
+
+        for (int i = 0; i < 10000; i++) {
+            final int p = i;
+
+            PageStore ps = factory.createPageStore(
+                PageMemory.FLAG_DATA,
+                () -> getPartitionFilePath(workDir, p),
+                d -> { }
+            );
+
+            ps.ensure();
+
+            ps.write(0, ByteBuffer.allocate(256), 1, false);
+
+            stores.add(ps);
+        }
+
+        System.gc();
+
+        GraphLayout layout = GraphLayout.parseInstance(stores);
+
+        System.out.println("heap usage: " + layout.totalSize());
+
+        U.delete(workDir);
+    }
+
+    /** */
+    private Path getPartitionFilePath(File cacheWorkDir, int partId) {
+        return new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)).toPath();
+    }
+
+    /** */
+    public static void main(String[] args) throws Exception {
+        new FileStoreHeapUtilizationJolBenchmark().benchmark();
+    }
+}

modules/camel/pom.xml

@@ -35,7 +35,7 @@
     <url>http://ignite.apache.org</url>
 
     <properties>
-        <guava.version>18.0</guava.version>
+        <guava.version>25.1-jre</guava.version>
         <okhttp.version>2.5.0</okhttp.version>
     </properties>
 

modules/cassandra/store/pom.xml

@@ -39,7 +39,7 @@
         <cassandra-driver.version>3.0.0</cassandra-driver.version>
         <cassandra-all.version>3.3</cassandra-all.version>
         <netty.version>4.1.27.Final</netty.version>
-        <guava.version>19.0</guava.version>
+        <guava.version>25.1-jre</guava.version>
         <metrics-core.version>3.0.2</metrics-core.version>
     </properties>