Skip to content

Commit 749b623

Browse files
authored
Introduce jvm.options.d for customizing JVM options (#51882)
This commit introduces the ability to override JVM options by adding custom JVM options files to a jvm.options.d directory. This simplifies administration of Elasticsearch by not requiring administrators to keep the root jvm.options file in sync with changes that we make to the root jvm.options file. Instead, they are not expected to modify this file but instead supply their own in jvm.options.d. In Docker installations, this means they can bind mount this directory in. In future versions of Elasticsearch, we can consider removing the root jvm.options file (instead, providing all options there as system JVM options).
1 parent 5936896 commit 749b623

File tree

18 files changed

+241
-111
lines changed

18 files changed

+241
-111
lines changed

distribution/archives/build.gradle

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ task createPluginsDir(type: EmptyDirTask) {
4444
dir = "${pluginsDir}"
4545
dirMode = 0755
4646
}
47+
ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d')
48+
task createJvmOptionsDir(type: EmptyDirTask) {
49+
dir = "${jvmOptionsDir}"
50+
dirMode = 0750
51+
}
4752

4853
CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
4954
return copySpec {
@@ -55,6 +60,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
5560
dirMode 0750
5661
fileMode 0660
5762
with configFiles(distributionType, oss, jdk)
63+
from {
64+
dirMode 0750
65+
jvmOptionsDir.getParent()
66+
}
5867
}
5968
into('bin') {
6069
with binFiles(distributionType, oss, jdk)
@@ -94,7 +103,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
94103

95104
// common config across all zip/tar
96105
tasks.withType(AbstractArchiveTask) {
97-
dependsOn createLogsDir, createPluginsDir
106+
dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir
98107
String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1)
99108
destinationDir = file("${subdir}/build/distributions")
100109
baseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}"

distribution/docker/src/docker/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ ${source_elasticsearch}
3030
RUN tar zxf /opt/${elasticsearch} --strip-components=1
3131
RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \
3232
&& sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
33-
RUN mkdir -p config data logs
34-
RUN chmod 0775 config data logs
33+
RUN mkdir -p config config/jvm.options.d data logs
34+
RUN chmod 0775 config config/jvm.options.d data logs
3535
COPY config/elasticsearch.yml config/log4j2.properties config/
3636
RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
3737

distribution/packages/build.gradle

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,9 @@ void addProcessFilesTask(String type, boolean oss, boolean jdk) {
7979
mkdir "${packagingFiles}/var/lib/elasticsearch"
8080
mkdir "${packagingFiles}/usr/share/elasticsearch/plugins"
8181

82-
// bare empty dir for /etc/elasticsearch
82+
// bare empty dir for /etc/elasticsearch and /etc/elasticsearch/jvm.options.d
8383
mkdir "${packagingFiles}/elasticsearch"
84+
mkdir "${packagingFiles}/elasticsearch/jvm.options.d"
8485
}
8586
}
8687
}
@@ -197,6 +198,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
197198
includeEmptyDirs true
198199
createDirectoryEntry true
199200
include("elasticsearch") // empty dir, just to add directory entry
201+
include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry
200202
}
201203
from("${packagingFiles}/etc/elasticsearch") {
202204
into('/etc/elasticsearch')

distribution/packages/src/common/scripts/postrm

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ else
1717
fi
1818

1919
REMOVE_DIRS=false
20+
REMOVE_JVM_OPTIONS_DIRECTORY=false
2021
REMOVE_USER_AND_GROUP=false
2122

2223
case "$1" in
@@ -27,6 +28,8 @@ case "$1" in
2728
;;
2829

2930
purge)
31+
REMOVE_DIRS=true
32+
REMOVE_JVM_OPTIONS_DIRECTORY=true
3033
REMOVE_USER_AND_GROUP=true
3134
;;
3235
failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear)
@@ -80,6 +83,20 @@ if [ "$REMOVE_DIRS" = "true" ]; then
8083
rmdir --ignore-fail-on-non-empty /var/lib/elasticsearch
8184
fi
8285

86+
# delete the jvm.options.d directory if and only if empty
87+
if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
88+
rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}/jvm.options.d"
89+
fi
90+
91+
# delete the jvm.options.d directory if we are purging
92+
if [ "$REMOVE_JVM_OPTIONS_DIRECTORY" = "true" ]; then
93+
if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
94+
echo -n "Deleting jvm.options.d directory..."
95+
rm -rf "${ES_PATH_CONF}/jvm.options.d"
96+
echo " OK"
97+
fi
98+
fi
99+
83100
# delete the conf directory if and only if empty
84101
if [ -d "${ES_PATH_CONF}" ]; then
85102
rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}"

distribution/packages/src/deb/lintian/elasticsearch

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ missing-dep-on-jarwrapper
1616
# we prefer to not make our config and log files world readable
1717
non-standard-file-perm etc/default/elasticsearch 0660 != 0644
1818
non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
19+
non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
1920
non-standard-file-perm etc/elasticsearch/*
2021
non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
2122
non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755

distribution/src/bin/elasticsearch

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,7 @@ then
4646
fi
4747
fi
4848

49-
ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
50-
ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"`
49+
ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
5150

5251
# manual parsing to find out, if process should be detached
5352
if [[ $DAEMONIZE = false ]]; then

distribution/src/bin/elasticsearch-service.bat

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
115115
if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
116116

117117
@setlocal
118-
for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
118+
for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
119119
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
120120

121121
if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
@@ -167,15 +167,15 @@ for %%a in ("%ES_JAVA_OPTS:;=","%") do (
167167
@endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS% & set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS%
168168

169169
if "%JVM_MS%" == "" (
170-
echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
170+
echo minimum heap size not set; configure using -Xms via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
171171
goto:eof
172172
)
173173
if "%JVM_MX%" == "" (
174-
echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
174+
echo maximum heap size not set; configure using -Xmx via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
175175
goto:eof
176176
)
177177
if "%JVM_SS%" == "" (
178-
echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
178+
echo thread stack size not set; configure using -Xss via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
179179
goto:eof
180180
)
181181
set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS:"=%

distribution/src/bin/elasticsearch.bat

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,8 @@ if not defined ES_TMPDIR (
7272
for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set ES_TMPDIR=%%a
7373
)
7474

75-
set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
7675
@setlocal
77-
for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
76+
for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
7877
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
7978

8079
if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

Lines changed: 76 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
import java.io.InputStreamReader;
2828
import java.io.Reader;
2929
import java.nio.charset.StandardCharsets;
30+
import java.nio.file.DirectoryStream;
3031
import java.nio.file.Files;
32+
import java.nio.file.Path;
3133
import java.nio.file.Paths;
3234
import java.util.ArrayList;
3335
import java.util.Arrays;
@@ -43,6 +45,7 @@
4345
import java.util.regex.Matcher;
4446
import java.util.regex.Pattern;
4547
import java.util.stream.Collectors;
48+
import java.util.stream.StreamSupport;
4649

4750
/**
4851
* Parses JVM options from a file and prints a single line with all JVM options to standard output.
@@ -53,82 +56,90 @@ final class JvmOptionsParser {
5356
* The main entry point. The exit code is 0 if the JVM options were successfully parsed, otherwise the exit code is 1. If an improperly
5457
* formatted line is discovered, the line is output to standard error.
5558
*
56-
* @param args the args to the program which should consist of a single option, the path to the JVM options
59+
* @param args the args to the program which should consist of a single option, the path to ES_PATH_CONF
5760
*/
5861
public static void main(final String[] args) throws InterruptedException, IOException {
5962
if (args.length != 1) {
60-
throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args));
61-
}
62-
final List<String> jvmOptions = new ArrayList<>();
63-
final SortedMap<Integer, String> invalidLines = new TreeMap<>();
64-
try (
65-
InputStream is = Files.newInputStream(Paths.get(args[0]));
66-
Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
67-
BufferedReader br = new BufferedReader(reader)
68-
) {
69-
parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, new JvmOptionConsumer() {
70-
@Override
71-
public void accept(final String jvmOption) {
72-
jvmOptions.add(jvmOption);
73-
}
74-
}, new InvalidLineConsumer() {
75-
@Override
76-
public void accept(final int lineNumber, final String line) {
77-
invalidLines.put(lineNumber, line);
78-
}
79-
});
63+
throw new IllegalArgumentException("expected one argument specifying path to ES_PATH_CONF but was " + Arrays.toString(args));
8064
}
8165

82-
if (invalidLines.isEmpty()) {
83-
// now append the JVM options from ES_JAVA_OPTS
84-
final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
85-
if (environmentJvmOptions != null) {
86-
jvmOptions.addAll(
87-
Arrays.stream(environmentJvmOptions.split("\\s+"))
88-
.filter(Predicate.not(String::isBlank))
89-
.collect(Collectors.toUnmodifiableList())
90-
);
66+
final ArrayList<Path> jvmOptionsFiles = new ArrayList<>();
67+
jvmOptionsFiles.add(Paths.get(args[0], "jvm.options"));
68+
69+
final Path jvmOptionsDirectory = Paths.get(args[0], "jvm.options.d");
70+
71+
if (Files.isDirectory(jvmOptionsDirectory)) {
72+
try (
73+
DirectoryStream<Path> jvmOptionsDirectoryStream = Files.newDirectoryStream(Paths.get(args[0], "jvm.options.d"), "*.options")
74+
) {
75+
// collect the matching JVM options files after sorting them by Path::compareTo
76+
StreamSupport.stream(jvmOptionsDirectoryStream.spliterator(), false).sorted().forEach(jvmOptionsFiles::add);
9177
}
92-
final Map<String, String> substitutions = new HashMap<>();
93-
substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
94-
if (null != System.getenv("ES_PATH_CONF")) {
95-
substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
78+
}
79+
80+
final List<String> jvmOptions = new ArrayList<>();
81+
82+
for (final Path jvmOptionsFile : jvmOptionsFiles) {
83+
final SortedMap<Integer, String> invalidLines = new TreeMap<>();
84+
try (
85+
InputStream is = Files.newInputStream(jvmOptionsFile);
86+
Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
87+
BufferedReader br = new BufferedReader(reader)
88+
) {
89+
parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, jvmOptions::add, invalidLines::put);
9690
}
97-
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
98-
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
99-
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
100-
final List<String> finalJvmOptions = new ArrayList<>(
101-
systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
102-
);
103-
finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
104-
finalJvmOptions.addAll(substitutedJvmOptions);
105-
finalJvmOptions.addAll(ergonomicJvmOptions);
106-
final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
107-
Launchers.outPrintln(spaceDelimitedJvmOptions);
108-
Launchers.exit(0);
109-
} else {
110-
final String errorMessage = String.format(
111-
Locale.ROOT,
112-
"encountered [%d] error%s parsing [%s]",
113-
invalidLines.size(),
114-
invalidLines.size() == 1 ? "" : "s",
115-
args[0]
116-
);
117-
Launchers.errPrintln(errorMessage);
118-
int count = 0;
119-
for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
120-
count++;
121-
final String message = String.format(
91+
if (invalidLines.isEmpty() == false) {
92+
final String errorMessage = String.format(
12293
Locale.ROOT,
123-
"[%d]: encountered improperly formatted JVM option line [%s] on line number [%d]",
124-
count,
125-
entry.getValue(),
126-
entry.getKey()
94+
"encountered [%d] error%s parsing [%s]",
95+
invalidLines.size(),
96+
invalidLines.size() == 1 ? "" : "s",
97+
jvmOptionsFile
12798
);
128-
Launchers.errPrintln(message);
99+
Launchers.errPrintln(errorMessage);
100+
int count = 0;
101+
for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
102+
count++;
103+
final String message = String.format(
104+
Locale.ROOT,
105+
"[%d]: encountered improperly formatted JVM option in [%s] on line number [%d]: [%s]",
106+
count,
107+
jvmOptionsFile,
108+
entry.getKey(),
109+
entry.getValue()
110+
);
111+
Launchers.errPrintln(message);
112+
}
113+
Launchers.exit(1);
129114
}
130-
Launchers.exit(1);
131115
}
116+
117+
// now append the JVM options from ES_JAVA_OPTS
118+
final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
119+
if (environmentJvmOptions != null) {
120+
jvmOptions.addAll(
121+
Arrays.stream(environmentJvmOptions.split("\\s+"))
122+
.filter(Predicate.not(String::isBlank))
123+
.collect(Collectors.toUnmodifiableList())
124+
);
125+
}
126+
final Map<String, String> substitutions = new HashMap<>();
127+
substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
128+
if (null != System.getenv("ES_PATH_CONF")) {
129+
substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
130+
}
131+
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
132+
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
133+
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
134+
final List<String> finalJvmOptions = new ArrayList<>(
135+
systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
136+
);
137+
finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
138+
finalJvmOptions.addAll(substitutedJvmOptions);
139+
finalJvmOptions.addAll(ergonomicJvmOptions);
140+
final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
141+
Launchers.outPrintln(spaceDelimitedJvmOptions);
142+
Launchers.exit(0);
132143
}
133144

134145
static List<String> substitutePlaceholders(final List<String> jvmOptions, final Map<String, String> substitutions) {

docs/reference/setup/important-settings/heap-size.asciidoc

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ caches, but the less memory it leaves available for the operating system to use
4949
for the filesystem cache. Also, larger heaps can cause longer garbage
5050
collection pauses.
5151

52-
Here are examples of how to set the heap size via the jvm.options file:
52+
Here is an example of how to set the heap size via a `jvm.options.d/` file:
5353

5454
[source,txt]
5555
------------------
@@ -60,8 +60,7 @@ Here are examples of how to set the heap size via the jvm.options file:
6060
<2> Set the maximum heap size to 2g.
6161

6262
It is also possible to set the heap size via an environment variable. This can
63-
be done by commenting out the `Xms` and `Xmx` settings in the
64-
<<jvm-options,`jvm.options`>> file and setting these values via `ES_JAVA_OPTS`:
63+
be done by setting these values via `ES_JAVA_OPTS`:
6564

6665
[source,sh]
6766
------------------

0 commit comments

Comments
 (0)