Skip to content

Commit

Permalink
TIKA-3441 -- prevent infinite loop on failure to bind to a port
Browse files Browse the repository at this point in the history
  • Loading branch information
@tballison
tballison committed Jun 9, 2021
1 parent 499e1c9 commit fd98eee
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 26 deletions.
60 changes: 45 additions & 15 deletions tika-server/src/main/java/org/apache/tika/server/TikaServerCli.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.net.BindException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -46,6 +47,7 @@
import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
import org.apache.cxf.rs.security.cors.CrossOriginResourceSharingFilter;
import org.apache.cxf.service.factory.ServiceConstructionException;
import org.apache.cxf.transport.common.gzip.GZIPInInterceptor;
import org.apache.cxf.transport.common.gzip.GZIPOutInterceptor;
import org.apache.tika.Tika;
Expand Down Expand Up @@ -85,6 +87,8 @@
public class TikaServerCli {


public static final int BIND_EXCEPTION = 42;

//used in spawn-child mode
private static final long DEFAULT_MAX_FILES = 100000;

Expand Down Expand Up @@ -177,10 +181,28 @@ private static void execute(String[] args) throws Exception {
}
}
}
executeLegacy(line, options);
try {
executeLegacy(line, options);
} catch (ServiceConstructionException e) {
if (isBindException(e)) {
LOG.warn("failed to bind to port", e);
System.exit(BIND_EXCEPTION);
}
throw e;
}
}
}

private static boolean isBindException(Throwable e) {
if (e == null) {
return false;
}
if (e instanceof BindException) {
return true;
}
return isBindException(e.getCause());
}

private static String[] stripChildArgs(String[] args) {
List<String> ret = new ArrayList<>();
for (int i = 0; i < args.length; i++) {
Expand Down Expand Up @@ -288,27 +310,16 @@ private static void executeLegacy(CommandLine line, Options options) throws Exce
String serverId = line.hasOption("i") ? line.getOptionValue("i") : UUID.randomUUID().toString();
LOG.debug("SERVER ID:" +serverId);
ServerStatus serverStatus;
//this is used in a forked process to write status to the forking process
//and to check status from forking process
//it will be null if running in legacy no fork mode
//if this is a child process
if (line.hasOption("child")) {
serverStatus = new ServerStatus(serverId, Integer.parseInt(line.getOptionValue("numRestarts")),
false);
//redirect!!!
InputStream in = System.in;
System.setIn(new ByteArrayInputStream(new byte[0]));
System.setOut(System.err);

long maxFiles = DEFAULT_MAX_FILES;
if (line.hasOption("maxFiles")) {
maxFiles = Long.parseLong(line.getOptionValue("maxFiles"));
}

ServerTimeouts serverTimeouts = configureServerTimeouts(line);
String childStatusFile = line.getOptionValue("childStatusFile");
Thread serverThread =
new Thread(new ServerStatusWatcher(serverStatus, in,
Paths.get(childStatusFile), maxFiles, serverTimeouts));

serverThread.start();
} else {
serverStatus = new ServerStatus(serverId, 0, true);
}
Expand Down Expand Up @@ -380,6 +391,25 @@ private static void executeLegacy(CommandLine line, Options options) throws Exce
if (line.hasOption("metrics")) {
MetricsHelper.registerPostStart(sf, server);
}

//start the forked server thread after the server has started
if (line.hasOption("child")) {
long maxFiles = DEFAULT_MAX_FILES;
if (line.hasOption("maxFiles")) {
maxFiles = Long.parseLong(line.getOptionValue("maxFiles"));
}

ServerTimeouts serverTimeouts = configureServerTimeouts(line);
String childStatusFile = line.getOptionValue("childStatusFile");
InputStream in = System.in;
System.setIn(new ByteArrayInputStream(new byte[0]));

Thread serverThread =
new Thread(new ServerStatusWatcher(serverStatus, in,
Paths.get(childStatusFile), maxFiles, serverTimeouts));

serverThread.start();
}
LOG.info("Started Apache Tika server at {}", url);
}

Expand Down
55 changes: 44 additions & 11 deletions tika-server/src/main/java/org/apache/tika/server/TikaServerWatchDog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

package org.apache.tika.server;

import org.apache.tika.io.MappedBufferCleaner;
import org.apache.tika.utils.ProcessUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -28,17 +27,16 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.net.BindException;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
Expand Down Expand Up @@ -72,23 +70,23 @@ public void execute(String[] args, ServerTimeouts serverTimeouts) throws Excepti
startPingTimer(serverTimeouts);
try {
int restarts = 0;
childProcess = new ChildProcess(args, restarts, serverTimeouts);
childProcess = startChildProcess(args, restarts, serverTimeouts);
setChildStatus(CHILD_STATUS.RUNNING);
while (true) {
if (!childProcess.ping()) {
LOG.debug("bad ping, initializing");
LOG.info("bad ping. restarting.");
restarts++;
if (serverTimeouts.getMaxRestarts() > -1 && restarts >= serverTimeouts.getMaxRestarts()) {
LOG.warn("hit max restarts: "+restarts+". Stopping now");
break;
}
setChildStatus(CHILD_STATUS.INITIALIZING);
lastPing = null;
childProcess.close();
LOG.debug("About to restart the child process");
childProcess = new ChildProcess(args, restarts, serverTimeouts);
LOG.info("About to restart the child process");
childProcess = startChildProcess(args, restarts, serverTimeouts);
LOG.info("Successfully restarted child process -- {} restarts so far)", restarts);
setChildStatus(CHILD_STATUS.RUNNING);
if (serverTimeouts.getMaxRestarts() > -1 && restarts >= serverTimeouts.getMaxRestarts()) {
LOG.warn("hit max restarts: "+restarts+". Stopping now");
break;
}
}
Thread.sleep(serverTimeouts.getPingPulseMillis());
}
Expand All @@ -104,6 +102,28 @@ public void execute(String[] args, ServerTimeouts serverTimeouts) throws Excepti
}
}

private ChildProcess startChildProcess(String[] args, int restarts,
ServerTimeouts serverTimeouts) throws Exception {
int consecutiveRestarts = 0;
//if there's a bind exception, retry for 5 seconds to give the OS
//a chance to release the port
int maxBind = 5;
while (consecutiveRestarts < maxBind) {
try {
return new ChildProcess(args, restarts, serverTimeouts);
} catch (BindException e) {
LOG.warn("WatchDog observes bind exception on retry {}. " +
"Will retry {} times.", consecutiveRestarts, maxBind);
consecutiveRestarts++;
Thread.sleep(1000);
if (consecutiveRestarts > maxBind) {
throw e;
}
}
}
throw new RuntimeException("Couldn't start child process");
}

private String[] addIdIfMissing(String[] args) {
for (String arg : args) {
//id is already specified, leave the array as is
Expand Down Expand Up @@ -272,6 +292,9 @@ private ChildProcess(String[] args, int numRestarts, ServerTimeouts serverTimeou
}
if (!process.isAlive()) {
close();
if (process.exitValue() == TikaServerCli.BIND_EXCEPTION) {
throw new BindException("couldn't bind");
}
throw new RuntimeException("Failed to start child process -- child is not alive");
}
if (!Files.exists(childStatusFile)) {
Expand Down Expand Up @@ -448,14 +471,24 @@ public void run() {

private static synchronized void destroyChildForcibly(Process process) {
process = process.destroyForcibly();

try {
boolean destroyed = process.waitFor(60, TimeUnit.SECONDS);
if (! destroyed) {
LOG.error("Child process still alive after 60 seconds. " +
"Shutting down the parent.");
System.exit(1);
}
try {
int exitValue = process.exitValue();
LOG.info("Forked (child) process shut down with exit value: {}", exitValue);
} catch (IllegalThreadStateException e) {
LOG.error("Child process still alive when trying to read exit value. " +
"Shutting down the parent.");
System.exit(1);
}
} catch (InterruptedException e) {
LOG.warn("interrupted exception while trying to destroy the forked process");
//swallow
}
}
Expand Down

0 comments on commit fd98eee

Please sign in to comment.