Skip to content

Commit

Permalink
IGNITE-15922 Implement NUMA-aware allocator
Browse files Browse the repository at this point in the history
  • Loading branch information
ivandasch committed Nov 19, 2021
1 parent 5255324 commit f4d4cd6
Show file tree
Hide file tree
Showing 24 changed files with 1,550 additions and 8 deletions.
15 changes: 12 additions & 3 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ matrix:
include:
- language: java
os: linux
dist: xenial
dist: bionic
before_install:
- sudo apt-get update
- sudo apt-get -y install libnuma-dev
install: skip
jdk: openjdk8
before_script:
Expand All @@ -32,7 +35,10 @@ matrix:

- language: java
os: linux
dist: xenial
dist: bionic
before_install:
- sudo apt-get update
- sudo apt-get -y install libnuma-dev
install: skip
jdk: openjdk11
before_script:
Expand All @@ -58,7 +64,10 @@ matrix:
- language: java
name: "Check test suites"
os: linux
dist: xenial
dist: bionic
before_install:
- sudo apt-get update
- sudo apt-get -y install libnuma-dev
install: skip
jdk: openjdk8
script: mvn test -Pcheck-test-suites,all-java,all-scala,scala -B -V
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.mem.MemoryAllocator;
import org.apache.ignite.mxbean.DataRegionMetricsMXBean;
import org.apache.ignite.mxbean.MetricsMxBean;
import org.jetbrains.annotations.Nullable;
Expand Down Expand Up @@ -152,6 +153,9 @@ public final class DataRegionConfiguration implements Serializable {
/** Warm-up configuration. */
@Nullable private WarmUpConfiguration warmUpCfg;

/** Memory allocator. */
@Nullable private MemoryAllocator memoryAllocator = null;

/**
* Gets data region name.
*
Expand Down Expand Up @@ -244,6 +248,25 @@ public DataRegionConfiguration setSwapPath(String swapPath) {
return this;
}

/**
* @return Memory allocator instance.
*/
@Nullable public MemoryAllocator getMemoryAllocator() {
return memoryAllocator;
}

/**
* Sets memory allocator.
*
* @param allocator Memory allocator instance.
* @return {@code this} for chaining.
*/
public DataRegionConfiguration setMemoryAllocator(MemoryAllocator allocator) {
memoryAllocator = allocator;

return this;
}

/**
* Gets memory pages eviction mode. If {@link DataPageEvictionMode#DISABLED} is used (default) then an out of
* memory exception will be thrown if the memory region usage, defined by this data region, goes beyond its
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ignite.internal.mem.unsafe;

import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.mem.MemoryAllocator;

/** */
public class UnsafeMemoryAllocator implements MemoryAllocator {
/** {@inheritDoc} */
@Override public long allocateMemory(long size) {
return GridUnsafe.allocateMemory(size);
}

/** {@inheritDoc} */
@Override public void freeMemory(long addr) {
GridUnsafe.freeMemory(addr);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.mem.UnsafeChunk;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.mem.MemoryAllocator;
import org.jetbrains.annotations.Nullable;

/**
* Memory provider implementation based on unsafe memory access.
Expand All @@ -49,10 +50,22 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider {
/** */
private int used = 0;

/** */
private final MemoryAllocator allocator;

/**
* @param log Ignite logger to use.
*/
public UnsafeMemoryProvider(@Nullable IgniteLogger log) {
this(log, null);
}

/**
* @param log Ignite logger to use.
* @param allocator Memory allocator. If {@code null}, default {@link UnsafeMemoryAllocator} will be used.
*/
public UnsafeMemoryProvider(IgniteLogger log) {
public UnsafeMemoryProvider(@Nullable IgniteLogger log, @Nullable MemoryAllocator allocator) {
this.allocator = allocator == null ? new UnsafeMemoryAllocator() : allocator;
this.log = log;
}

Expand All @@ -75,7 +88,7 @@ public UnsafeMemoryProvider(IgniteLogger log) {
DirectMemoryRegion chunk = it.next();

if (deallocate) {
GridUnsafe.freeMemory(chunk.address());
allocator.freeMemory(chunk.address());

// Safety.
it.remove();
Expand All @@ -100,7 +113,7 @@ public UnsafeMemoryProvider(IgniteLogger log) {
long ptr;

try {
ptr = GridUnsafe.allocateMemory(chunkSize);
ptr = allocator.allocateMemory(chunkSize);
}
catch (IllegalArgumentException e) {
String msg = "Failed to allocate next memory chunk: " + U.readableSize(chunkSize, true) +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1315,7 +1315,7 @@ private DirectMemoryProvider createMemoryProvider(DataRegionConfiguration plcCfg
File allocPath = buildAllocPath(plcCfg);

return allocPath == null ?
new UnsafeMemoryProvider(log) :
new UnsafeMemoryProvider(log, plcCfg.getMemoryAllocator()) :
new MappedFileMemoryProvider(
log,
allocPath);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ignite.mem;

/** */
public interface MemoryAllocator {
/**
* @param size Size of allocated memory.
*
* @return Pointer to memory.
*/
long allocateMemory(long size);

/**
* Deallocates memory.
*
* @param addr Address of memory.
*/
void freeMemory(long addr);
}
178 changes: 178 additions & 0 deletions modules/numa-allocator/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
<?xml version="1.0" encoding="UTF-8"?>

<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<!--
POM file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-parent</artifactId>
<version>1</version>
<relativePath>../../parent</relativePath>
</parent>

<artifactId>ignite-numa-allocator</artifactId>
<version>${revision}</version>
<url>http://ignite.apache.org</url>

<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>ignite-core</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>ignite-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>ignite-tools</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>${spring.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.thoughtworks.xstream</groupId>
<artifactId>xstream</artifactId>
<version>1.4.8</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<id>build-cpp</id>
<phase>generate-sources</phase>
<configuration>
<tasks>
<mkdir dir="${project.build.directory}/cppbuild"/>
<exec os="Linux" executable="cmake" dir="${project.build.directory}/cppbuild" failonerror="true">
<arg line="-DCMAKE_BUILD_TYPE=Release"/>
<arg line="${basedir}/src/main/cpp"/>
</exec>
<exec os="Linux" executable="make" dir="${project.build.directory}/cppbuild" failonerror="true">
<arg line="VERBOSE=1"/>
<arg line="-j4"/>
</exec>
<copy failonerror="false"
file="${project.build.directory}/cppbuild/libnuma_alloc.so"
tofile="${project.build.directory}/classes/org/apache/ignite/internal/mem/linux/amd64/libnuma_alloc.so"/>
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
<execution>
<id>clean-native</id>
<phase>clean</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<echo>Builddir: ${project.build.directory}</echo>
<echo>Basedir: ${basedir}</echo>
<delete dir="${project.build.directory}/cppbuild"/>
</target>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-libs</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<excludeTransitive>false</excludeTransitive>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
Loading

0 comments on commit f4d4cd6

Please sign in to comment.