IGNITE-15922 Implement NUMA-aware allocator - Fixes apache#9569.

Signed-off-by: Ivan Daschinsky <ivandasch@apache.org>

.travis.yml

@@ -21,7 +21,10 @@ matrix:
   include:
     - language: java
       os: linux
-      dist: xenial
+      dist: bionic
+      before_install:
+        - sudo apt-get update
+        - sudo apt-get -y install libnuma-dev
       install: skip
       jdk: openjdk8
       before_script:
@@ -32,7 +35,10 @@ matrix:
 
     - language: java
       os: linux
-      dist: xenial
+      dist: bionic
+      before_install:
+        - sudo apt-get update
+        - sudo apt-get -y install libnuma-dev
       install: skip
       jdk: openjdk11
       before_script:
@@ -58,7 +64,10 @@ matrix:
     - language: java
       name: "Check test suites"
       os: linux
-      dist: xenial
+      dist: bionic
+      before_install:
+        - sudo apt-get update
+        - sudo apt-get -y install libnuma-dev
       install: skip
       jdk: openjdk8
       script: mvn test -Pcheck-test-suites,all-java,all-scala,scala -B -V

modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java

@@ -20,6 +20,7 @@
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.mem.MemoryAllocator;
 import org.apache.ignite.mxbean.DataRegionMetricsMXBean;
 import org.apache.ignite.mxbean.MetricsMxBean;
 import org.jetbrains.annotations.Nullable;
@@ -152,6 +153,9 @@ public final class DataRegionConfiguration implements Serializable {
     /** Warm-up configuration. */
     @Nullable private WarmUpConfiguration warmUpCfg;
 
+    /** Memory allocator. */
+    @Nullable private MemoryAllocator memoryAllocator = null;
+
     /**
      * Gets data region name.
      *
@@ -244,6 +248,25 @@ public DataRegionConfiguration setSwapPath(String swapPath) {
         return this;
     }
 
+    /**
+     * @return Memory allocator instance.
+     */
+    @Nullable public MemoryAllocator getMemoryAllocator() {
+        return memoryAllocator;
+    }
+
+    /**
+     * Sets memory allocator. If not specified, default, based on {@code Unsafe} allocator will be used.
+     *
+     * @param allocator Memory allocator instance.
+     * @return {@code this} for chaining.
+     */
+    public DataRegionConfiguration setMemoryAllocator(MemoryAllocator allocator) {
+        memoryAllocator = allocator;
+
+        return this;
+    }
+
     /**
      * Gets memory pages eviction mode. If {@link DataPageEvictionMode#DISABLED} is used (default) then an out of
      * memory exception will be thrown if the memory region usage, defined by this data region, goes beyond its

modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java

@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteExperimental;
+import org.apache.ignite.mem.MemoryAllocator;
 import org.apache.ignite.mxbean.MetricsMxBean;
 import org.jetbrains.annotations.Nullable;
 
@@ -337,6 +338,9 @@ public class DataStorageConfiguration implements Serializable {
     /** Minimum size of wal archive folder, in bytes. */
     private long minWalArchiveSize = HALF_MAX_WAL_ARCHIVE_SIZE;
 
+    /** Default memory allocator for all data regions. */
+    @Nullable private MemoryAllocator memoryAllocator = null;
+
     /**
      * Creates valid durable memory configuration with all default values.
      */
@@ -1364,6 +1368,27 @@ public DataStorageConfiguration setMinWalArchiveSize(long walArchiveMinSize) {
         return this;
     }
 
+    /**
+     * @return Memory allocator instance.
+     */
+    @Nullable public MemoryAllocator getMemoryAllocator() {
+        return memoryAllocator;
+    }
+
+    /**
+     * Sets default memory allocator for all memory regions. If not specified, default, based on {@code Unsafe}
+     * allocator will be used. Allocator can be overrided for data region using
+     * {@link DataRegionConfiguration#setMemoryAllocator(MemoryAllocator)}
+     *
+     * @param allocator Memory allocator instance.
+     * @return {@code this} for chaining.
+     */
+    public DataStorageConfiguration setMemoryAllocator(MemoryAllocator allocator) {
+        memoryAllocator = allocator;
+
+        return this;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DataStorageConfiguration.class, this);

modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryAllocator.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.mem.unsafe;
+
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.mem.MemoryAllocator;
+
+/** */
+public class UnsafeMemoryAllocator implements MemoryAllocator {
+    /** {@inheritDoc} */
+    @Override public long allocateMemory(long size) {
+        return GridUnsafe.allocateMemory(size);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void freeMemory(long addr) {
+        GridUnsafe.freeMemory(addr);
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java

@@ -25,8 +25,9 @@
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.UnsafeChunk;
-import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.mem.MemoryAllocator;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Memory provider implementation based on unsafe memory access.
@@ -49,10 +50,22 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider {
     /** */
     private int used = 0;
 
+    /** */
+    private final MemoryAllocator allocator;
+
+    /**
+     * @param log Ignite logger to use.
+     */
+    public UnsafeMemoryProvider(@Nullable IgniteLogger log) {
+        this(log, null);
+    }
+
     /**
      * @param log Ignite logger to use.
+     * @param allocator Memory allocator. If {@code null}, default {@link UnsafeMemoryAllocator} will be used.
      */
-    public UnsafeMemoryProvider(IgniteLogger log) {
+    public UnsafeMemoryProvider(@Nullable IgniteLogger log, @Nullable MemoryAllocator allocator) {
+        this.allocator = allocator == null ? new UnsafeMemoryAllocator() : allocator;
         this.log = log;
     }
 
@@ -70,20 +83,21 @@ public UnsafeMemoryProvider(IgniteLogger log) {
 
     /** {@inheritDoc} */
     @Override public void shutdown(boolean deallocate) {
+        if (!deallocate) {
+            used = 0;
+
+            return;
+        }
+
         if (regions != null) {
             for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) {
                 DirectMemoryRegion chunk = it.next();
 
-                if (deallocate) {
-                    GridUnsafe.freeMemory(chunk.address());
+                allocator.freeMemory(chunk.address());
 
-                    // Safety.
-                    it.remove();
-                }
+                // Safety.
+                it.remove();
             }
-
-            if (!deallocate)
-                used = 0;
         }
     }
 
@@ -100,7 +114,7 @@ public UnsafeMemoryProvider(IgniteLogger log) {
         long ptr;
 
         try {
-            ptr = GridUnsafe.allocateMemory(chunkSize);
+            ptr = allocator.allocateMemory(chunkSize);
         }
         catch (IllegalArgumentException e) {
             String msg = "Failed to allocate next memory chunk: " + U.readableSize(chunkSize, true) +

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java

@@ -1276,7 +1276,11 @@ private DataRegion initMemory(
         boolean trackable,
         PageReadWriteManager pmPageMgr
     ) throws IgniteCheckedException {
-        PageMemory pageMem = createPageMemory(createOrReuseMemoryProvider(plcCfg), memCfg, plcCfg, memMetrics, trackable, pmPageMgr);
+        if (plcCfg.getMemoryAllocator() == null)
+            plcCfg.setMemoryAllocator(memCfg.getMemoryAllocator());
+
+        PageMemory pageMem = createPageMemory(createOrReuseMemoryProvider(plcCfg), memCfg, plcCfg, memMetrics,
+            trackable, pmPageMgr);
 
         return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
     }
@@ -1315,7 +1319,7 @@ private DirectMemoryProvider createMemoryProvider(DataRegionConfiguration plcCfg
         File allocPath = buildAllocPath(plcCfg);
 
         return allocPath == null ?
-            new UnsafeMemoryProvider(log) :
+            new UnsafeMemoryProvider(log, plcCfg.getMemoryAllocator()) :
             new MappedFileMemoryProvider(
                 log,
                 allocPath);

modules/core/src/main/java/org/apache/ignite/mem/MemoryAllocator.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.mem;
+
+/**
+ * Base interface for offheap memory allocator.
+ */
+public interface MemoryAllocator {
+    /**
+     * @param size Size of allocated memory.
+     *
+     * @return Pointer to memory or {@code 0} if failed.
+     */
+    public long allocateMemory(long size);
+
+    /**
+     * Deallocates memory.
+     *
+     * @param addr Address of memory.
+     */
+    public void freeMemory(long addr);
+}