Skip to content

Commit

Permalink
Merge pull request #276 from internetarchive/jdk11-upgrade-jetty
Browse files Browse the repository at this point in the history
JDK11 support: upgrade to Jetty 9.4.19, Restlet 2.4.0 and drop JDK 7 support
nlevitt authored Aug 7, 2019

Verified

This commit was signed with the committer’s verified signature.
wezm Wesley Moore
2 parents a1f2c95 + 97b9c3c commit cefad96
Showing 39 changed files with 426 additions and 474 deletions.
7 changes: 2 additions & 5 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -6,9 +6,6 @@ matrix:
include:
- jdk: oraclejdk8
dist: trusty
- jdk: openjdk7
dist: trusty
env: PROJECTS='--projects commons,modules,engine'
- jdk: openjdk8
- jdk: openjdk11
allow_failures:
@@ -22,14 +19,14 @@ before_install:
- "export _JAVA_OPTIONS=-Xmx1500m"
- "echo _JAVA_OPTIONS=$_JAVA_OPTIONS"

install: mvn dependency:resolve -B -V $PROJECTS
install: mvn dependency:resolve -B -V

cache:
directories:
- $HOME/.m2

script:
- travis_wait 30 mvn install $PROJECTS
- travis_wait 30 mvn install

after_failure:
- cat */target/surefire-reports/*.txt
2 changes: 1 addition & 1 deletion commons/src/test/resources/log4j.xml
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>

<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>

2 changes: 1 addition & 1 deletion contrib/src/main/resources/log4j.xml
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>

<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>

2 changes: 1 addition & 1 deletion contrib/src/test/resources/log4j.xml
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>

<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>

2 changes: 1 addition & 1 deletion dist/src/main/conf/logging.properties
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@
# ...and even less from the too-chatty-with-WARNINGs HttpClient library...
org.apache.commons.httpclient.level = SEVERE
org.restlet.Component.LogFilter.level = SEVERE
org.mortbay.log.level = SEVERE
org.eclipse.jetty.log.level = SEVERE
# ...but INFO for our classes, which reserve FINE/FINER/FINEST for bulk/trivia...
org.archive.level = INFO

2 changes: 1 addition & 1 deletion dist/src/test/resources/log4j.xml
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
<level value="ERROR" />
</logger>

<logger name="org.mortbay.log">
<logger name="org.eclipse.jetty">
<level value="ERROR" />
</logger>

60 changes: 33 additions & 27 deletions engine/pom.xml
Original file line number Diff line number Diff line change
@@ -25,44 +25,50 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<!-- we force the version of jetty rather than allowing restlet
to include it transitively as we need at least 9.4.12 for jdk11 -->
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.4.19.v20190610</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
<version>9.4.19.v20190610</version>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-ajp</artifactId>
<version>6.1.26</version>
<scope>compile</scope>
</dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<version>9.4.19.v20190610</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.restlet</groupId>
<artifactId>org.restlet</artifactId>
<version>1.1.10</version>
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet.ext.jetty</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-client</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.noelios.restlet</groupId>
<artifactId>com.noelios.restlet</artifactId>
<version>1.1.10</version>
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet.ext.xml</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>com.noelios.restlet</groupId>
<artifactId>com.noelios.restlet.ext.jetty</artifactId>
<version>1.1.10</version>
<!-- for HTTP Digest auth -->
<groupId>org.restlet.jse</groupId>
<artifactId>org.restlet.ext.crypto</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
33 changes: 20 additions & 13 deletions engine/src/main/java/org/archive/crawler/Heritrix.java
Original file line number Diff line number Diff line change
@@ -57,10 +57,11 @@
import org.archive.util.ArchiveUtils;
import org.archive.util.KeyTool;
import org.restlet.Component;
import org.restlet.Guard;
import org.restlet.Server;
import org.restlet.data.ChallengeScheme;
import org.restlet.data.Protocol;
import org.restlet.security.ChallengeAuthenticator;
import org.restlet.security.MapVerifier;


/**
@@ -212,7 +213,7 @@ public void instanceMain(String[] args)
"mailto, clsid, res, file, rtsp, about");
}

String maxFormSize = "org.mortbay.jetty.Request.maxFormContentSize";
String maxFormSize = "org.eclipse.jetty.server.Request.maxFormContentSize";
if (System.getProperty(maxFormSize) == null) {
System.setProperty(maxFormSize, "52428800");
}
@@ -334,24 +335,30 @@ public void instanceMain(String[] args)
try {
engine = new Engine(jobsDir);
component = new Component();

if(bindHosts.isEmpty()) {
// listen all addresses
setupServer(port, null, keystorePath, keystorePassword, keyPassword);
setupServer(component, port, null, keystorePath, keystorePassword, keyPassword);
} else {
// bind only to declared addresses, or just 'localhost'
for(String address : bindHosts) {
setupServer(port, address, keystorePath, keystorePassword, keyPassword);
setupServer(component, port, address, keystorePath, keystorePassword, keyPassword);
}
}
component.getClients().add(Protocol.FILE);
component.getClients().add(Protocol.CLAP);
Guard guard = new RateLimitGuard(null,
component.getClients().add(Protocol.CLAP);

MapVerifier verifier = new MapVerifier();
verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray());

ChallengeAuthenticator guard = new RateLimitGuard(component.getContext().createChildContext(),
ChallengeScheme.HTTP_DIGEST, "Authentication Required");
guard.getSecrets().put(authLogin, authPassword.toCharArray());
component.getDefaultHost().attach(guard);
guard.setVerifier(verifier);
guard.setNext(new EngineApplication(engine));

component.getDefaultHost().attach(guard);
component.start();

startupOut.println("engine listening at port "+port);
startupOut.println("operator login set per " +
((aOption.startsWith("@")) ? "file "+aOption : "command-line"));
@@ -457,16 +464,16 @@ protected void useAdhocKeystore(PrintStream startupOut) {

/**
* Create an HTTPS restlet Server instance matching the given parameters.
*
*
* @param component
* @param port
* @param address
* @param keystorePath
* @param keystorePassword
* @param keyPassword
*/
protected void setupServer(int port, String address, String keystorePath, String keystorePassword, String keyPassword) {
Server server = new Server(Protocol.HTTPS,address,port,null);
component.getServers().add(server);
protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword) {
Server server = component.getServers().add(Protocol.HTTPS, address, port);
server.getContext().getParameters().add("keystorePath", keystorePath);
server.getContext().getParameters().add("keystorePassword", keystorePassword);
server.getContext().getParameters().add("keyPassword", keyPassword);
45 changes: 2 additions & 43 deletions engine/src/main/java/org/archive/crawler/restlet/BaseResource.java
Original file line number Diff line number Diff line change
@@ -19,55 +19,14 @@

package org.archive.crawler.restlet;

import java.util.List;

import org.restlet.Context;
import org.restlet.data.MediaType;
import org.restlet.data.Preference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Resource;
import org.restlet.resource.Variant;
import org.restlet.resource.ServerResource;

/**
* Abstract {@code Resource} with common shared functionality.
*
* @author nlevitt
*/
public abstract class BaseResource extends Resource {

public BaseResource(Context ctx, Request req, Response res) {
super(ctx, req, res);
}

/**
* If client can accept text/html, always prefer it. WebKit-based browsers
* claim to want application/xml, but we don't want to give it to them. See
* <a href="https://webarchive.jira.com/browse/HER-1603">https://webarchive.jira.com/browse/HER-1603</a>
*/
public Variant getPreferredVariant() {
boolean addExplicitTextHtmlPreference = false;

for (Preference<MediaType> mediaTypePreference: getRequest().getClientInfo().getAcceptedMediaTypes()) {
if (mediaTypePreference.getMetadata().equals(MediaType.TEXT_HTML)) {
mediaTypePreference.setQuality(Float.MAX_VALUE);
addExplicitTextHtmlPreference = false;
break;
} else if (mediaTypePreference.getMetadata().includes(MediaType.TEXT_HTML)) {
addExplicitTextHtmlPreference = true;
}
}

if (addExplicitTextHtmlPreference) {
List<Preference<MediaType>> acceptedMediaTypes = getRequest().getClientInfo().getAcceptedMediaTypes();
acceptedMediaTypes.add(new Preference<MediaType>(MediaType.TEXT_HTML, Float.MAX_VALUE));
getRequest().getClientInfo().setAcceptedMediaTypes(acceptedMediaTypes);
}


return super.getPreferredVariant();
}

public abstract class BaseResource extends ServerResource {
protected String getStaticRef(String resource) {
String rootRef = getRequest().getRootRef().toString();
return rootRef + "/engine/static/" + resource;
Original file line number Diff line number Diff line change
@@ -19,35 +19,31 @@

package org.archive.crawler.restlet;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.archive.crawler.restlet.models.BeansModel;
import org.archive.crawler.restlet.models.ViewModel;
import org.archive.spring.PathSharingContext;
import org.archive.util.TextUtils;
import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.representation.EmptyRepresentation;
import org.restlet.representation.Representation;
import org.restlet.representation.WriterRepresentation;
import org.restlet.resource.ResourceException;
import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import org.restlet.representation.Variant;
import org.springframework.beans.BeanWrapperImpl;
import org.springframework.beans.BeansException;

@@ -69,12 +65,12 @@ public class BeanBrowseResource extends JobRelatedResource {
protected PathSharingContext appCtx;
protected String beanPath;
private Configuration _templateConfiguration;

public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException {
super(ctx, req, res);

@Override
public void init(Context ctx, Request req, Response res) throws ResourceException {
super.init(ctx, req, res);
getVariants().add(new Variant(MediaType.TEXT_HTML));
getVariants().add(new Variant(MediaType.APPLICATION_XML));
setModifiable(true); // accept POSTs
appCtx = cj.getJobContext();
beanPath = (String)req.getAttributes().get("beanPath");
if (beanPath!=null) {
@@ -99,13 +95,14 @@ public Configuration getTemplateConfiguration(){
return _templateConfiguration;
}

public void acceptRepresentation(Representation entity) throws ResourceException {
@Override
protected Representation post(Representation entity, Variant variant) throws ResourceException {
if (appCtx == null) {
throw new ResourceException(404);
}

// copy op?
Form form = getRequest().getEntityAsForm();
Form form = new Form(entity);
beanPath = form.getFirstValue("beanPath");

String newVal = form.getFirstValue("newVal");
@@ -122,6 +119,7 @@ public void acceptRepresentation(Representation entity) throws ResourceException
ref.setPath(getBeansRefPath());
ref.addSegment(beanPath);
getResponse().redirectSeeOther(ref);
return new EmptyRepresentation();
}

public String getBeansRefPath() {
@@ -137,7 +135,8 @@ public String getBeansRefPath() {
return path;
}

public Representation represent(Variant variant) throws ResourceException {
@Override
public Representation get(Variant variant) throws ResourceException {
if (appCtx == null) {
throw new ResourceException(404);
}
@@ -209,8 +208,8 @@ protected BeansModel makeDataModel(){
for(String name: appCtx.getBeanDefinitionNames()) {
addPresentableNestedNames(nestedNames, appCtx.getBean(name), alreadyWritten);
}
return new BeansModel(cj.getShortName(),

return new BeansModel(cj.getShortName(),
new Reference(getRequest().getResourceRef().getBaseRef(), "..").getTargetRef().toString(),
beanPath,
bean,
Loading

0 comments on commit cefad96

Please sign in to comment.