diff --git a/.travis.yml b/.travis.yml index 20978aa4d..c3f0a33a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,6 @@ matrix: include: - jdk: oraclejdk8 dist: trusty - - jdk: openjdk7 - dist: trusty - env: PROJECTS='--projects commons,modules,engine' - jdk: openjdk8 - jdk: openjdk11 allow_failures: @@ -22,14 +19,14 @@ before_install: - "export _JAVA_OPTIONS=-Xmx1500m" - "echo _JAVA_OPTIONS=$_JAVA_OPTIONS" -install: mvn dependency:resolve -B -V $PROJECTS +install: mvn dependency:resolve -B -V cache: directories: - $HOME/.m2 script: - - travis_wait 30 mvn install $PROJECTS + - travis_wait 30 mvn install after_failure: - cat */target/surefire-reports/*.txt diff --git a/commons/src/test/resources/log4j.xml b/commons/src/test/resources/log4j.xml index 475b130e2..e04f34ab8 100644 --- a/commons/src/test/resources/log4j.xml +++ b/commons/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/contrib/src/main/resources/log4j.xml b/contrib/src/main/resources/log4j.xml index 475b130e2..e04f34ab8 100644 --- a/contrib/src/main/resources/log4j.xml +++ b/contrib/src/main/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/contrib/src/test/resources/log4j.xml b/contrib/src/test/resources/log4j.xml index 475b130e2..e04f34ab8 100644 --- a/contrib/src/test/resources/log4j.xml +++ b/contrib/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/dist/src/main/conf/logging.properties b/dist/src/main/conf/logging.properties index f83723961..094cb1762 100644 --- a/dist/src/main/conf/logging.properties +++ b/dist/src/main/conf/logging.properties @@ -5,7 +5,7 @@ # ...and even less from the too-chatty-with-WARNINGs HttpClient library... org.apache.commons.httpclient.level = SEVERE org.restlet.Component.LogFilter.level = SEVERE -org.mortbay.log.level = SEVERE +org.eclipse.jetty.log.level = SEVERE # ...but INFO for our classes, which reserve FINE/FINER/FINEST for bulk/trivia... org.archive.level = INFO diff --git a/dist/src/test/resources/log4j.xml b/dist/src/test/resources/log4j.xml index 475b130e2..e04f34ab8 100644 --- a/dist/src/test/resources/log4j.xml +++ b/dist/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/engine/pom.xml b/engine/pom.xml index 5e4a7f44c..9051e9e0b 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -25,44 +25,50 @@ ${project.version} compile + - org.mortbay.jetty - jetty - 6.1.26 - compile + org.eclipse.jetty + jetty-server + 9.4.19.v20190610 - org.mortbay.jetty + org.eclipse.jetty jetty-util - 6.1.26 - compile + 9.4.19.v20190610 - org.mortbay.jetty - jetty-sslengine - 6.1.26 - compile - - - org.mortbay.jetty - jetty-ajp - 6.1.26 - compile - + org.eclipse.jetty + jetty-servlet + 9.4.19.v20190610 + test + + + org.restlet.jse + org.restlet + 2.4.0 + - org.restlet - org.restlet - 1.1.10 + org.restlet.jse + org.restlet.ext.jetty + 2.4.0 + + + org.eclipse.jetty + jetty-client + + - com.noelios.restlet - com.noelios.restlet - 1.1.10 + org.restlet.jse + org.restlet.ext.xml + 2.4.0 - com.noelios.restlet - com.noelios.restlet.ext.jetty - 1.1.10 + + org.restlet.jse + org.restlet.ext.crypto + 2.4.0 joda-time diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java index 114950722..ee0fcf79c 100644 --- a/engine/src/main/java/org/archive/crawler/Heritrix.java +++ b/engine/src/main/java/org/archive/crawler/Heritrix.java @@ -57,10 +57,11 @@ import org.archive.util.ArchiveUtils; import org.archive.util.KeyTool; import org.restlet.Component; -import org.restlet.Guard; import org.restlet.Server; import org.restlet.data.ChallengeScheme; import org.restlet.data.Protocol; +import org.restlet.security.ChallengeAuthenticator; +import org.restlet.security.MapVerifier; /** @@ -212,7 +213,7 @@ public void instanceMain(String[] args) "mailto, clsid, res, file, rtsp, about"); } - String maxFormSize = "org.mortbay.jetty.Request.maxFormContentSize"; + String maxFormSize = "org.eclipse.jetty.server.Request.maxFormContentSize"; if (System.getProperty(maxFormSize) == null) { System.setProperty(maxFormSize, "52428800"); } @@ -334,24 +335,30 @@ public void instanceMain(String[] args) try { engine = new Engine(jobsDir); component = new Component(); - + if(bindHosts.isEmpty()) { // listen all addresses - setupServer(port, null, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, null, keystorePath, keystorePassword, keyPassword); } else { // bind only to declared addresses, or just 'localhost' for(String address : bindHosts) { - setupServer(port, address, keystorePath, keystorePassword, keyPassword); + setupServer(component, port, address, keystorePath, keystorePassword, keyPassword); } } component.getClients().add(Protocol.FILE); - component.getClients().add(Protocol.CLAP); - Guard guard = new RateLimitGuard(null, + component.getClients().add(Protocol.CLAP); + + MapVerifier verifier = new MapVerifier(); + verifier.getLocalSecrets().put(authLogin, authPassword.toCharArray()); + + ChallengeAuthenticator guard = new RateLimitGuard(component.getContext().createChildContext(), ChallengeScheme.HTTP_DIGEST, "Authentication Required"); - guard.getSecrets().put(authLogin, authPassword.toCharArray()); - component.getDefaultHost().attach(guard); + guard.setVerifier(verifier); guard.setNext(new EngineApplication(engine)); + + component.getDefaultHost().attach(guard); component.start(); + startupOut.println("engine listening at port "+port); startupOut.println("operator login set per " + ((aOption.startsWith("@")) ? "file "+aOption : "command-line")); @@ -457,16 +464,16 @@ protected void useAdhocKeystore(PrintStream startupOut) { /** * Create an HTTPS restlet Server instance matching the given parameters. - * + * + * @param component * @param port * @param address * @param keystorePath * @param keystorePassword * @param keyPassword */ - protected void setupServer(int port, String address, String keystorePath, String keystorePassword, String keyPassword) { - Server server = new Server(Protocol.HTTPS,address,port,null); - component.getServers().add(server); + protected void setupServer(Component component, int port, String address, String keystorePath, String keystorePassword, String keyPassword) { + Server server = component.getServers().add(Protocol.HTTPS, address, port); server.getContext().getParameters().add("keystorePath", keystorePath); server.getContext().getParameters().add("keystorePassword", keystorePassword); server.getContext().getParameters().add("keyPassword", keyPassword); diff --git a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java index 1ebc80efb..72ae434b2 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java @@ -19,55 +19,14 @@ package org.archive.crawler.restlet; -import java.util.List; - -import org.restlet.Context; -import org.restlet.data.MediaType; -import org.restlet.data.Preference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Resource; -import org.restlet.resource.Variant; +import org.restlet.resource.ServerResource; /** * Abstract {@code Resource} with common shared functionality. * * @author nlevitt */ -public abstract class BaseResource extends Resource { - - public BaseResource(Context ctx, Request req, Response res) { - super(ctx, req, res); - } - - /** - * If client can accept text/html, always prefer it. WebKit-based browsers - * claim to want application/xml, but we don't want to give it to them. See - * https://webarchive.jira.com/browse/HER-1603 - */ - public Variant getPreferredVariant() { - boolean addExplicitTextHtmlPreference = false; - - for (Preference mediaTypePreference: getRequest().getClientInfo().getAcceptedMediaTypes()) { - if (mediaTypePreference.getMetadata().equals(MediaType.TEXT_HTML)) { - mediaTypePreference.setQuality(Float.MAX_VALUE); - addExplicitTextHtmlPreference = false; - break; - } else if (mediaTypePreference.getMetadata().includes(MediaType.TEXT_HTML)) { - addExplicitTextHtmlPreference = true; - } - } - - if (addExplicitTextHtmlPreference) { - List> acceptedMediaTypes = getRequest().getClientInfo().getAcceptedMediaTypes(); - acceptedMediaTypes.add(new Preference(MediaType.TEXT_HTML, Float.MAX_VALUE)); - getRequest().getClientInfo().setAcceptedMediaTypes(acceptedMediaTypes); - } - - - return super.getPreferredVariant(); - } - +public abstract class BaseResource extends ServerResource { protected String getStaticRef(String resource) { String rootRef = getRequest().getRootRef().toString(); return rootRef + "/engine/static/" + resource; diff --git a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java index b09d70af7..5ae0c8975 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java @@ -19,16 +19,12 @@ package org.archive.crawler.restlet; -import java.io.File; import java.io.IOException; -import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.io.Writer; import java.net.URLDecoder; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.Set; @@ -36,18 +32,18 @@ import org.archive.crawler.restlet.models.BeansModel; import org.archive.crawler.restlet.models.ViewModel; import org.archive.spring.PathSharingContext; -import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import org.springframework.beans.BeanWrapperImpl; import org.springframework.beans.BeansException; @@ -69,12 +65,12 @@ public class BeanBrowseResource extends JobRelatedResource { protected PathSharingContext appCtx; protected String beanPath; private Configuration _templateConfiguration; - - public BeanBrowseResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); - setModifiable(true); // accept POSTs appCtx = cj.getJobContext(); beanPath = (String)req.getAttributes().get("beanPath"); if (beanPath!=null) { @@ -99,13 +95,14 @@ public Configuration getTemplateConfiguration(){ return _templateConfiguration; } - public void acceptRepresentation(Representation entity) throws ResourceException { + @Override + protected Representation post(Representation entity, Variant variant) throws ResourceException { if (appCtx == null) { throw new ResourceException(404); } // copy op? - Form form = getRequest().getEntityAsForm(); + Form form = new Form(entity); beanPath = form.getFirstValue("beanPath"); String newVal = form.getFirstValue("newVal"); @@ -122,6 +119,7 @@ public void acceptRepresentation(Representation entity) throws ResourceException ref.setPath(getBeansRefPath()); ref.addSegment(beanPath); getResponse().redirectSeeOther(ref); + return new EmptyRepresentation(); } public String getBeansRefPath() { @@ -137,7 +135,8 @@ public String getBeansRefPath() { return path; } - public Representation represent(Variant variant) throws ResourceException { + @Override + public Representation get(Variant variant) throws ResourceException { if (appCtx == null) { throw new ResourceException(404); } @@ -209,8 +208,8 @@ protected BeansModel makeDataModel(){ for(String name: appCtx.getBeanDefinitionNames()) { addPresentableNestedNames(nestedNames, appCtx.getBean(name), alreadyWritten); } - - return new BeansModel(cj.getShortName(), + + return new BeansModel(cj.getShortName(), new Reference(getRequest().getResourceRef().getBaseRef(), "..").getTargetRef().toString(), beanPath, bean, diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index a4ea7996b..232dbdd2d 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -30,8 +30,8 @@ import org.restlet.data.CharacterSet; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.resource.CharacterRepresentation; -import org.restlet.resource.FileRepresentation; +import org.restlet.representation.CharacterRepresentation; +import org.restlet.representation.FileRepresentation; /** * Representation wrapping a FileRepresentation, displaying its contents @@ -40,9 +40,9 @@ * @author gojomo */ public class EditRepresentation extends CharacterRepresentation { - protected FileRepresentation fileRepresentation; + protected FileRepresentation fileRepresentation; protected EnhDirectoryResource dirResource; - + public EditRepresentation(FileRepresentation representation, EnhDirectoryResource resource) { super(MediaType.TEXT_HTML); fileRepresentation = representation; diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java index 6c7793f71..a82c30261 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineApplication.java @@ -26,19 +26,19 @@ import org.archive.crawler.framework.Engine; import org.archive.util.TextUtils; import org.restlet.Application; -import org.restlet.Directory; -import org.restlet.Redirector; import org.restlet.Restlet; -import org.restlet.Router; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.Status; -import org.restlet.resource.Representation; -import org.restlet.resource.StringRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.StringRepresentation; +import org.restlet.resource.Directory; +import org.restlet.routing.Redirector; +import org.restlet.routing.Router; +import org.restlet.routing.Template; import org.restlet.service.StatusService; -import org.restlet.util.Template; /** * Restlet Application for a Heritrix crawl 'Engine', which is aware of @@ -56,7 +56,8 @@ public EngineApplication(Engine engine) { setStatusService(new EngineStatusService()); } - public synchronized Restlet createRoot() { + @Override + public Restlet createInboundRoot() { Router router = new Router(getContext()); router.attach("/",new Redirector(null,"/engine",Redirector.MODE_CLIENT_TEMPORARY)); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index b18d9f3dc..dabd7f3ea 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -30,21 +30,24 @@ import org.archive.crawler.restlet.models.EngineModel; import org.archive.crawler.restlet.models.ViewModel; import org.restlet.Context; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; +import static org.restlet.data.MediaType.APPLICATION_XML; + /** * Restlet Resource representing an Engine that may be used * to assemble, launch, monitor, and manage crawls. @@ -56,11 +59,12 @@ public class EngineResource extends BaseResource { private Configuration _templateConfiguration; - public EngineResource(Context ctx, Request req, Response res) { - super(ctx, req, res); - setModifiable(true); + + @Override + public void init(Context ctx, Request req, Response res) { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); - getVariants().add(new Variant(MediaType.APPLICATION_XML)); + getVariants().add(new Variant(APPLICATION_XML)); Configuration tmpltCfg = new Configuration(); tmpltCfg.setClassForTemplateLoading(this.getClass(),""); @@ -74,10 +78,12 @@ public void setTemplateConfiguration(Configuration tmpltCfg) { public Configuration getTemplateConfiguration(){ return _templateConfiguration; } - public Representation represent(Variant variant) throws ResourceException { + + @Override + protected Representation get(Variant variant) throws ResourceException { Representation representation; - if (variant.getMediaType() == MediaType.APPLICATION_XML) { - representation = new WriterRepresentation(MediaType.APPLICATION_XML) { + if (variant.getMediaType() == APPLICATION_XML) { + representation = new WriterRepresentation(APPLICATION_XML) { public void write(Writer writer) throws IOException { XmlMarshaller.marshalDocument(writer, "engine", makeDataModel()); } @@ -93,10 +99,10 @@ public void write(Writer writer) throws IOException { representation.setCharacterSet(CharacterSet.UTF_8); return representation; } - + @Override - public void acceptRepresentation(Representation entity) throws ResourceException { - Form form = getRequest().getEntityAsForm(); + protected Representation post(Representation entity, Variant variant) throws ResourceException { + Form form = new Form(entity); String action = form.getFirstValue("action"); if("rescan".equals(action)) { getEngine().findJobConfigs(); @@ -182,6 +188,7 @@ public void acceptRepresentation(Representation entity) throws ResourceException } // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); + return new EmptyRepresentation(); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java index ea7bb79d6..bb26b16ba 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectory.java @@ -26,10 +26,12 @@ import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.restlet.Context; -import org.restlet.Handler; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.data.Status; +import org.restlet.resource.Directory; +import org.restlet.resource.ServerResource; /** * Enhanced version of Restlet Directory, which allows the local @@ -39,11 +41,11 @@ * * @author gojomo */ -public abstract class EnhDirectory extends org.restlet.Directory { +public abstract class EnhDirectory extends Directory { protected IOFileFilter editFilter = FileFilterUtils.falseFileFilter(); protected IOFileFilter pageFilter = FileFilterUtils.falseFileFilter(); protected IOFileFilter tailFilter = FileFilterUtils.falseFileFilter(); - + public EnhDirectory(Context context, Reference rootLocalReference) { super(context, rootLocalReference); // TODO Auto-generated constructor stub @@ -55,21 +57,27 @@ public EnhDirectory(Context context, String rootUri) { } @Override - public Handler findTarget(Request request, Response response) { - Handler retVal; - synchronized(this) { + public void handle(Request request, Response response) { + synchronized (this) { Reference oldRef = getRootRef(); setRootRef(determineRootRef(request)); try { - retVal = new EnhDirectoryResource(this, request, response); - } catch (IOException ioe) { - getLogger().log(Level.WARNING, - "Unable to find the directory's resource", ioe); - retVal = null; + super.handle(request, response); + } finally { + setRootRef(oldRef); + } + + // XXX: FileRepresentation.isAvailable() returns false for empty files generating status 204 No Content + // which confuses browsers. Force it back it 200 OK. + if (response.getStatus() == Status.SUCCESS_NO_CONTENT) { + response.setStatus(Status.SUCCESS_OK); } - setRootRef(oldRef); } - return retVal; + } + + @Override + public ServerResource create(Request request, Response response) { + return new EnhDirectoryResource(); } protected abstract Reference determineRootRef(Request request); diff --git a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java index 5f7c0b5e0..18c593543 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EnhDirectoryResource.java @@ -23,22 +23,18 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import org.apache.commons.io.FileUtils; -import org.restlet.data.CharacterSet; -import org.restlet.data.Form; -import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.data.Status; -import org.restlet.resource.FileRepresentation; -import org.restlet.resource.Representation; +import org.restlet.data.*; +import org.restlet.engine.local.DirectoryServerResource; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.FileRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.Variant; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; - -import com.noelios.restlet.local.DirectoryResource; /** * Enhanced version of Restlet DirectoryResource, adding ability to @@ -46,20 +42,15 @@ * * @author gojomo */ -public class EnhDirectoryResource extends DirectoryResource { - - public EnhDirectoryResource(EnhDirectory directory, Request request, Response response) throws IOException { - super(directory, request, response); - } - - /** +public class EnhDirectoryResource extends DirectoryServerResource { + /** * Add EditRepresentation as a variant when appropriate. * - * @see com.noelios.restlet.local.DirectoryResource#getVariants() + * @see org.restlet.engine.local.DirectoryServerResource#getVariants() */ @Override public List getVariants() { - List variants = super.getVariants(); + List variants = new LinkedList<>(super.getVariants(Method.GET)); Form f = getRequest().getResourceRef().getQueryAsForm(); String format = f.getFirstValue("format"); if("textedit".equals(format)) { @@ -73,7 +64,7 @@ public List getVariants() { } catch (Exception e) { throw new RuntimeException(e); } - variants = super.getVariants(); + variants = new LinkedList<>(super.getVariants(Method.GET)); } // wrap FileRepresentations in EditRepresentations ListIterator iter = variants.listIterator(); @@ -119,18 +110,18 @@ public List getVariants() { } protected EnhDirectory getEnhDirectory() { - return (EnhDirectory)getDirectory(); + return (EnhDirectory) getDirectory(); } - /** + /** * Accept a POST used to edit or create a file. * - * @see org.restlet.resource.Resource#acceptRepresentation(org.restlet.resource.Representation) + * @see org.restlet.resource.ServerResource#post(Representation, Variant) */ - public void acceptRepresentation(Representation entity) - throws ResourceException { + @Override + protected Representation post(Representation entity, Variant variant) throws ResourceException { // TODO: only allowPost on valid targets - Form form = getRequest().getEntityAsForm(); + Form form = new Form(entity); String newContents = form.getFirstValue("contents"); EditRepresentation er; try { @@ -152,6 +143,6 @@ public void acceptRepresentation(Representation entity) Reference ref = getRequest().getOriginalRef().clone(); /// ref.setQuery(null); getResponse().redirectSeeOther(ref); - + return new EmptyRepresentation(); } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/Flash.java b/engine/src/main/java/org/archive/crawler/restlet/Flash.java index a43d6c9ec..441408663 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/Flash.java +++ b/engine/src/main/java/org/archive/crawler/restlet/Flash.java @@ -29,10 +29,10 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.RandomUtils; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.Cookie; import org.restlet.data.CookieSetting; -import org.restlet.data.Request; -import org.restlet.data.Response; import org.restlet.util.Series; /** diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index ae967a0a9..4dfe548c7 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -40,8 +40,8 @@ import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.resource.ResourceException; import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanWrapperImpl; @@ -58,18 +58,19 @@ public abstract class JobRelatedResource extends BaseResource { private final static Logger LOGGER = Logger.getLogger(JobRelatedResource.class.getName()); - protected CrawlJob cj; + protected CrawlJob cj; protected IdentityHashMap beanToNameMap; - - public JobRelatedResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) { + super.init(ctx, req, res); cj = getEngine().getJob((String)req.getAttributes().get("job")); - if(cj==null) { + if(cj == null) { throw new ResourceException(404); } } - + protected Engine getEngine() { return ((EngineApplication)getApplication()).getEngine(); } @@ -95,13 +96,13 @@ protected void addPresentableNestedNames(Collection namedBeans, Object o return; } - - Reference baseRef = getRequest().getResourceRef().getBaseRef(); + + Reference baseRef = getRequest().getResourceRef().getBaseRef(); if (baseRef.getPath().endsWith("beans")) { baseRef.setPath(baseRef.getPath() + "/"); } - + if (getBeanToNameMap().containsKey(obj)) { // this object is itself a named bean Map bean = new LinkedHashMap(); diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index d3b0a295c..5b4ab6c74 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -38,12 +38,13 @@ import org.restlet.data.CharacterSet; import org.restlet.data.Form; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.EmptyRepresentation; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.ObjectWrapper; @@ -68,10 +69,10 @@ public class JobResource extends BaseResource { protected CrawlJob cj; - public JobResource(Context ctx, Request req, Response res) + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); - setModifiable(true); + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); cj = getEngine().getJob( @@ -88,7 +89,9 @@ public void setTemplateConfiguration(Configuration tmpltCfg) { public Configuration getTemplateConfiguration(){ return _templateConfiguration; } - public Representation represent(Variant variant) throws ResourceException { + + @Override + public Representation get(Variant variant) throws ResourceException { if (cj == null) { throw new ResourceException(404); } @@ -184,19 +187,18 @@ protected Engine getEngine() { } @Override - public void acceptRepresentation(Representation entity) + public Representation post(Representation entity, Variant variant) throws ResourceException { if (cj == null) { throw new ResourceException(404); } // copy op? - Form form = null; - form = getRequest().getEntityAsForm(); + Form form = new Form(entity); String copyTo = form.getFirstValue("copyTo"); if (copyTo != null) { copyJob(copyTo, "on".equals(form.getFirstValue("asProfile"))); - return; + return new EmptyRepresentation(); } AlertHandler.ensureStaticInitialization(); AlertThreadGroup.setThreadLogger(cj.getJobLogger()); @@ -241,6 +243,7 @@ public void acceptRepresentation(Representation entity) // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); + return new EmptyRepresentation(); } protected void copyJob(String copyTo, boolean asProfile) diff --git a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java index 363ee674e..aabd0d468 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/PagedRepresentation.java @@ -38,8 +38,8 @@ import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.resource.CharacterRepresentation; -import org.restlet.resource.FileRepresentation; +import org.restlet.representation.CharacterRepresentation; +import org.restlet.representation.FileRepresentation; /** * Representation wrapping a FileRepresentation, displaying its contents @@ -50,7 +50,7 @@ public class PagedRepresentation extends CharacterRepresentation { // passed-in at construction /** wrapped FileRepresentation **/ - protected FileRepresentation fileRepresentation; + protected FileRepresentation fileRepresentation; /** wrapped EnhDirectoryResource; used to formulate self-links **/ protected EnhDirectoryResource dirResource; @@ -124,7 +124,7 @@ protected void loadLines() throws IOException { /** * Write the paged HTML. * - * @see org.restlet.resource.Representation#write(java.io.Writer) + * @see org.restlet.representation.Representation#write(java.io.Writer) */ @Override public void write(Writer writer) throws IOException { diff --git a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java index d4d304ca7..042fd2ea6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java +++ b/engine/src/main/java/org/archive/crawler/restlet/RateLimitGuard.java @@ -18,21 +18,21 @@ */ package org.archive.crawler.restlet; -import java.util.Collection; import java.util.logging.Logger; import org.restlet.Context; -import org.restlet.Guard; +import org.restlet.Request; +import org.restlet.Response; import org.restlet.data.ChallengeScheme; -import org.restlet.data.Request; +import org.restlet.security.ChallengeAuthenticator; /** - * Guard that slows and logs failed authentication attempts, to make + * ChallengeAuthenticator that slows and logs failed authentication attempts, to make * brute-force guessing attacks less feasible. * * @author gojomo */ -public class RateLimitGuard extends Guard { +public class RateLimitGuard extends ChallengeAuthenticator { private static final int MIN_MS_BETWEEN_ATTEMPTS = 6000; private static final Logger logger = Logger.getLogger(RateLimitGuard.class.getName()); @@ -43,14 +43,10 @@ public RateLimitGuard(Context context, ChallengeScheme scheme, String realm) thr super(context, scheme, realm); } - public RateLimitGuard(Context context, String realm, Collection baseUris, String serverKey) { - super(context, realm, baseUris, serverKey); - } - @Override - public synchronized int authenticate(Request request) { - int retVal = super.authenticate(request); - if(retVal == AUTHENTICATION_INVALID) { + protected boolean authenticate(Request request, Response response) { + boolean succeeded = super.authenticate(request, response); + if (!succeeded) { logger.warning("authentication failure "+request); // wait until at least LAG has passed from last failure // holding object lock the whole time, so no other checks @@ -64,8 +60,8 @@ public synchronized int authenticate(Request request) { // ignore } } - lastFailureTime = now + sleepMs; + lastFailureTime = now + sleepMs; } - return retVal; + return succeeded; } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java index 3ccc87945..5468bf553 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ReportGenResource.java @@ -23,12 +23,12 @@ import org.restlet.Context; import org.restlet.data.MediaType; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.Representation; +import org.restlet.representation.StringRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.StringRepresentation; -import org.restlet.resource.Variant; +import org.restlet.representation.Variant; /** * Restlet Resource which generates fresh reports and then redirects @@ -38,24 +38,26 @@ */ public class ReportGenResource extends JobRelatedResource { protected String reportClass; - - public ReportGenResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_PLAIN)); reportClass = (String)req.getAttributes().get("reportClass"); } - public Representation represent(Variant variant) throws ResourceException { + @Override + protected Representation get(Variant variant) throws ResourceException { // generate report if (cj == null || cj.getCrawlController() == null) { throw new ResourceException(500); } - File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); + File f = cj.getCrawlController().getStatisticsTracker().writeReportFile(reportClass); if (f==null) { throw new ResourceException(500); } // redirect - String relative = JobResource.getHrefPath(f,cj); + String relative = JobResource.getHrefPath(f, cj); if(relative!=null) { getResponse().redirectSeeOther("../"+relative+"?m="+f.lastModified()); return new StringRepresentation(""); diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index 5a8bfa6bc..58fee0f91 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -41,12 +41,12 @@ import org.restlet.data.Form; import org.restlet.data.MediaType; import org.restlet.data.Reference; -import org.restlet.data.Request; -import org.restlet.data.Response; -import org.restlet.resource.Representation; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.representation.Representation; +import org.restlet.representation.WriterRepresentation; import org.restlet.resource.ResourceException; -import org.restlet.resource.Variant; -import org.restlet.resource.WriterRepresentation; +import org.restlet.representation.Variant; import freemarker.template.Configuration; import freemarker.template.ObjectWrapper; @@ -81,10 +81,10 @@ public int compare(ScriptEngineFactory sef1, ScriptEngineFactory sef2) { protected String chosenEngine = FACTORIES.isEmpty() ? "" : FACTORIES.getFirst().getNames().get(0); private Configuration _templateConfiguration; - - public ScriptResource(Context ctx, Request req, Response res) throws ResourceException { - super(ctx, req, res); - setModifiable(true); + + @Override + public void init(Context ctx, Request req, Response res) throws ResourceException { + super.init(ctx, req, res); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); @@ -92,7 +92,7 @@ public ScriptResource(Context ctx, Request req, Response res) throws ResourceExc tmpltCfg.setClassForTemplateLoading(this.getClass(),""); tmpltCfg.setObjectWrapper(ObjectWrapper.BEANS_WRAPPER); setTemplateConfiguration(tmpltCfg); - + scriptingConsole = new ScriptingConsole(cj); } public void setTemplateConfiguration(Configuration tmpltCfg) { @@ -105,8 +105,8 @@ public Configuration getTemplateConfiguration(){ private ScriptingConsole scriptingConsole; @Override - public void acceptRepresentation(Representation entity) throws ResourceException { - Form form = getRequest().getEntityAsForm(); + public Representation post(Representation entity, Variant variant) throws ResourceException { + Form form = new Form(entity); chosenEngine = form.getFirstValue("engine"); String script = form.getFirstValue("script"); if(StringUtils.isBlank(script)) { @@ -121,10 +121,11 @@ public void acceptRepresentation(Representation entity) throws ResourceException //TODO: log script, results somewhere; job log INFO? - getResponse().setEntity(represent()); + return get(variant); } - - public Representation represent(Variant variant) throws ResourceException { + + @Override + public Representation get(Variant variant) throws ResourceException { Representation representation; if (variant.getMediaType() == MediaType.APPLICATION_XML) { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { diff --git a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java index 11098b382..176204c67 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java +++ b/engine/src/main/java/org/archive/crawler/restlet/XmlMarshaller.java @@ -36,7 +36,7 @@ import javax.xml.bind.annotation.XmlType; import org.apache.commons.lang.StringUtils; -import org.restlet.util.XmlWriter; +import org.restlet.ext.xml.XmlWriter; import org.xml.sax.SAXException; /** diff --git a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java index 138f47a10..7f7d9e6be 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/CheckpointSelfTest.java @@ -22,10 +22,10 @@ import java.io.IOException; import org.archive.crawler.framework.CrawlJob; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; /** @@ -85,7 +85,7 @@ protected void startHttpServer() throws Exception { private Server makeHttpServer(int port) throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost(HOST); sc.setPort(port); server.addConnector(sc); diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java index 63d4b764b..6e04f56a9 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormAuthSelfTest.java @@ -19,19 +19,20 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; + import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; /** * Test form-based authentication @@ -60,17 +61,17 @@ protected void verify() throws Exception { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java index 510fb8395..332f6835d 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/FormLoginSelfTest.java @@ -19,20 +19,20 @@ package org.archive.crawler.selftest; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; + import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; - /** * Test form-based authentication * @@ -60,17 +60,17 @@ protected void verify() throws Exception { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java index a0f68464f..6ad5ebd29 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/HttpAuthSelfTest.java @@ -24,17 +24,16 @@ import java.util.HashSet; import java.util.Set; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.servlet.ServletHandler; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; /** * Test HTTP basic authentication @@ -73,27 +72,27 @@ protected void startHttpServer() throws Exception { ConstraintMapping cm = new ConstraintMapping(); cm.setConstraint(constraint); cm.setPathSpec("/basic/*"); + + UserStore userStore = new UserStore(); + userStore.addUser("Mr. Happy Pants", new Password("xyzzy"), new String[]{"rule"}); + HashLoginService loginService = new HashLoginService("Hyrule"); + loginService.setUserStore(userStore); - HashUserRealm realm = new HashUserRealm(); - realm.setName("Hyrule"); - realm.put("Mr. Happy Pants", "xyzzy"); - realm.addUserToRole("Mr. Happy Pants", "user"); - - SecurityHandler securityHandler = new SecurityHandler(); - securityHandler.setUserRealm(realm); + ConstraintSecurityHandler securityHandler = new ConstraintSecurityHandler(); + securityHandler.setLoginService(loginService); securityHandler.setConstraintMappings(new ConstraintMapping[]{cm}); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { securityHandler, rhandler, servletHandler, diff --git a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java index 0771360c0..7f09dfbe6 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java +++ b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java @@ -38,12 +38,12 @@ import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; /** * Base class for 'self tests', integrations tests formatted as unit @@ -182,7 +182,7 @@ protected void stopHttpServer() throws Exception { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index 8d880bf34..ceea95170 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -48,12 +48,12 @@ protected void verifyWarcStats() { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); assertEquals(13, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); + assertEquals(7501, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(2133, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(5368, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); assertEquals(0, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); } @@ -66,17 +66,17 @@ protected void verifySourceStats() throws Exception { sourceStats = stats.getSourceStats("http://127.0.0.1:7777/a.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(2942l, (long) sourceStats.get("novel")); + assertEquals(2133l, (long) sourceStats.get("novel")); assertEquals(3l, (long) sourceStats.get("novelCount")); - assertEquals(2942l, (long) sourceStats.get("warcNovelContentBytes")); + assertEquals(2133l, (long) sourceStats.get("warcNovelContentBytes")); assertEquals(3l, (long) sourceStats.get("warcNovelUrls")); sourceStats = stats.getSourceStats("http://localhost:7777/b.html"); assertNotNull(sourceStats); assertEquals(4, sourceStats.keySet().size()); - assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); + assertEquals(5368l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); - assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); + assertEquals(5368l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); } diff --git a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java index 489253456..d70ddc0bb 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/UserAgentSelfTest.java @@ -20,14 +20,14 @@ package org.archive.crawler.selftest; import org.archive.util.ArchiveUtils; -import org.mortbay.jetty.Handler; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.DefaultHandler; -import org.mortbay.jetty.handler.HandlerList; -import org.mortbay.jetty.handler.ResourceHandler; -import org.mortbay.jetty.servlet.ServletHandler; -import org.mortbay.jetty.servlet.ServletHolder; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.ResourceHandler; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; /** * @author pjack @@ -54,17 +54,17 @@ protected void verify() throws Exception { protected void startHttpServer() throws Exception { Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath()); - ServletHandler servletHandler = new ServletHandler(); + ServletHandler servletHandler = new ServletHandler(); HandlerList handlers = new HandlerList(); - handlers.setHandlers(new Handler[] { + handlers.setHandlers(new Handler[] { rhandler, servletHandler, new DefaultHandler() }); diff --git a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java index 7d3ce5761..39ca0bb62 100644 --- a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java +++ b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java @@ -44,20 +44,20 @@ import org.archive.net.UURIFactory; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.NCSARequestLog; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.handler.RequestLogHandler; -import org.mortbay.jetty.security.Authenticator; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.FormAuthenticator; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.servlet.HashSessionManager; -import org.mortbay.jetty.servlet.SessionHandler; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.security.authentication.FormAuthenticator; +import org.eclipse.jetty.server.NCSARequestLog; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.handler.RequestLogHandler; +import org.eclipse.jetty.server.session.DefaultSessionCache; +import org.eclipse.jetty.server.session.NullSessionDataStore; +import org.eclipse.jetty.server.session.SessionCache; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; /* Somewhat redundant to org.archive.crawler.selftest.FormAuthSelfTest, but * the code is written, it's easier to run in eclipse, and no doubt tests @@ -211,12 +211,9 @@ protected static class FormAuthTestHandler extends SessionHandler { public FormAuthTestHandler() { super(); } - + @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (target.endsWith("/set-cookie")) { response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); } @@ -247,16 +244,16 @@ protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, constraintMapping.setConstraint(constraint); constraintMapping.setPathSpec("/auth/*"); - SecurityHandler authWrapper = new SecurityHandler(); + UserStore userStore = new UserStore(); + userStore.addUser(login, new Password(password), new String[]{role}); + + HashLoginService loginService = new HashLoginService(realm); + loginService.setUserStore(userStore); + + ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler(); authWrapper.setAuthenticator(authenticator); - authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); - authWrapper.setUserRealm(new HashUserRealm(realm) { - { - put(login, password); - addUserToRole(login, role); - } - }); + authWrapper.setLoginService(loginService); return authWrapper; } @@ -265,7 +262,7 @@ protected void startHttpServers() throws Exception { // server for form auth Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7779); server.addConnector(sc); @@ -277,16 +274,17 @@ protected void startHttpServers() throws Exception { requestLogHandler.setRequestLog(requestLog); handlers.addHandler(requestLogHandler); - FormAuthenticator formAuthenticatrix = new FormAuthenticator(); - formAuthenticatrix.setLoginPage("/login.html"); - + FormAuthenticator formAuthenticatrix = new FormAuthenticator("/login.html", null, false); + SecurityHandler authWrapper = makeAuthWrapper(formAuthenticatrix, FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN, FORM_AUTH_PASSWORD); authWrapper.setHandler(handlers); SessionHandler sessionHandler = new SessionHandler(); - sessionHandler.setSessionManager(new HashSessionManager()); + SessionCache cache = new DefaultSessionCache(sessionHandler); + cache.setSessionDataStore(new NullSessionDataStore()); + sessionHandler.setSessionCache(cache); sessionHandler.setHandler(authWrapper); server.setHandler(sessionHandler); diff --git a/engine/src/test/resources/log4j.xml b/engine/src/test/resources/log4j.xml index 475b130e2..e04f34ab8 100644 --- a/engine/src/test/resources/log4j.xml +++ b/engine/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/modules/pom.xml b/modules/pom.xml index 0115535c4..ff3816aed 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -35,23 +35,15 @@ 1.6.3 - org.mortbay.jetty - jetty-util - 6.1.26 - test + org.eclipse.jetty + jetty-server + 9.4.19.v20190610 - org.mortbay.jetty - jetty-sslengine - 6.1.26 - test + org.eclipse.jetty + jetty-security + 9.4.19.v20190610 - - org.mortbay.jetty - jetty - 6.1.26 - test - org.littleshoot littleproxy diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 4ea94d22c..257c85024 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -43,18 +43,17 @@ import org.archive.spring.ConfigPath; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.SessionHandler; -import org.mortbay.log.Log; import com.google.common.io.Files; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.ssl.SslContextFactory; public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { @@ -64,9 +63,7 @@ public TestHandler() { } @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (request.getParameter("name") != null) { Cookie cookie = new javax.servlet.http.Cookie(request.getParameter("name"), request.getParameter("value")); @@ -114,7 +111,7 @@ public static Server startHttpServer() throws Exception { server.setHandler(new TestHandler()); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); @@ -133,11 +130,18 @@ public static Server startHttpServer() throws Exception { "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - SslSocketConnector ssc = new SslSocketConnector(); + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); + sslContextFactory.setKeyStorePath(keystoreFile.getPath()); + + HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + ServerConnector ssc = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); ssc.setHost("127.0.0.1"); ssc.setPort(7443); - ssc.setKeyPassword(KEYSTORE_PASSWORD); - ssc.setKeystore(keystoreFile.getPath()); server.addConnector(sc); server.addConnector(ssc); @@ -345,7 +349,7 @@ protected void testExplicitDomain(AbstractCookieStore cookieStore) throws URIExc CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -379,7 +383,7 @@ protected void testExplicitDomainWithLeadingDot(AbstractCookieStore cookieStore) CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=.example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=.example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=.example.com\r\n")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -413,7 +417,7 @@ protected void testRejectDomain(AbstractCookieStore cookieStore) throws URIExcep CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&domain=somethingelse.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=somethingelse.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=somethingelse.com\r\n")); // check fetch of original domain has no cookie curi = makeCrawlURI("http://example.com:7777/"); @@ -434,7 +438,7 @@ protected void testRejectDomain(AbstractCookieStore cookieStore) throws URIExcep curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=BAR.example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=bar.example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=bar.example.com\r\n")); // check fetch of original domain has no cookie curi = makeCrawlURI("http://foo.example.com:7777/"); @@ -463,7 +467,7 @@ protected void testSubdomainParentDomain(AbstractCookieStore cookieStore) throws CrawlURI curi = makeCrawlURI("http://FOO.example.com:7777/?name=foo&value=bar&domain=example.com"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar;Domain=example.com\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar; Domain=example.com\r\n")); curi = makeCrawlURI("http://FOO.example.com:7777/"); fetcher().process(curi); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 0b159abec..08b7c4a63 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -30,27 +30,21 @@ import org.archive.modules.ProcessorTestBase; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.NCSARequestLog; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Response; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.handler.RequestLogHandler; -import org.mortbay.jetty.security.Authenticator; -import org.mortbay.jetty.security.BasicAuthenticator; -import org.mortbay.jetty.security.Constraint; -import org.mortbay.jetty.security.ConstraintMapping; -import org.mortbay.jetty.security.DigestAuthenticator; -import org.mortbay.jetty.security.HashUserRealm; -import org.mortbay.jetty.security.SecurityHandler; -import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.SessionHandler; -import org.mortbay.log.Log; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.security.*; +import org.eclipse.jetty.security.authentication.BasicAuthenticator; +import org.eclipse.jetty.security.authentication.DigestAuthenticator; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.security.Constraint; +import org.eclipse.jetty.util.security.Password; +import org.eclipse.jetty.util.ssl.SslContextFactory; public class FetchHTTPTest extends ProcessorTestBase { @@ -125,12 +119,13 @@ protected static class TestHandler extends SessionHandler { public TestHandler() { super(); } - + @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + + // echo the remote host back to the client so tests can reference it + response.setHeader("Client-Host", request.getRemoteHost()); + if (target.endsWith("/set-cookie")) { response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); } @@ -221,9 +216,7 @@ public void handle(String target, HttpServletRequest request, } protected static Map httpServers; - protected static Request lastRequest = null; - protected static Response lastResponse = null; - + protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, final String role, String realm, final String login, final String password) { @@ -235,16 +228,15 @@ protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, constraintMapping.setConstraint(constraint); constraintMapping.setPathSpec("/auth/*"); - SecurityHandler authWrapper = new SecurityHandler(); + ConstraintSecurityHandler authWrapper = new ConstraintSecurityHandler(); authWrapper.setAuthenticator(authenticator); authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); - authWrapper.setUserRealm(new HashUserRealm(realm) { - { - put(login, password); - addUserToRole(login, role); - } - }); + UserStore userStore = new UserStore(); + userStore.addUser(login, new Password(password), new String[] {role}); + HashLoginService loginService = new HashLoginService(realm); + loginService.setUserStore(userStore); + authWrapper.setLoginService(loginService); return authWrapper; } @@ -259,24 +251,10 @@ public static Map startHttpServers() throws Exception { HashMap servers = new HashMap(); - HandlerCollection handlers = new HandlerCollection(); - handlers.addHandler(new TestHandler()); - RequestLogHandler requestLogHandler = new RequestLogHandler(); - NCSARequestLog requestLog = new NCSARequestLog() { - @Override - public void log(Request request, Response response) { - super.log(request, response); - lastRequest = request; - lastResponse = response; - } - }; - requestLogHandler.setRequestLog(requestLog); - handlers.addHandler(requestLogHandler); - // server for basic auth Server server = new Server(); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); @@ -284,6 +262,8 @@ public void log(Request request, Response response) { SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(), BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN, BASIC_AUTH_PASSWORD); + HandlerCollection handlers = new HandlerCollection(); + handlers.addHandler(new TestHandler()); authWrapper.setHandler(handlers); server.setHandler(authWrapper); @@ -299,16 +279,23 @@ public void log(Request request, Response response) { "-storepass", KEYSTORE_PASSWORD, "-keypass", KEYSTORE_PASSWORD, "-alias", "jetty", - "-genkey", + "-genkey", "-keyalg", "RSA", "-dname", "CN=127.0.0.1", "-validity","3650"}); // 10 yr validity - - SslSocketConnector ssc = new SslSocketConnector(); + + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePassword(KEYSTORE_PASSWORD); + sslContextFactory.setKeyStorePath(keystoreFile.getPath()); + + HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + ServerConnector ssc = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); ssc.setHost("127.0.0.1"); ssc.setPort(7443); - ssc.setKeyPassword(KEYSTORE_PASSWORD); - ssc.setKeystore(keystoreFile.getPath()); server.addConnector(ssc); @@ -317,7 +304,7 @@ public void log(Request request, Response response) { // server for digest auth server = new Server(); - sc = new SocketConnector(); + sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7778); server.addConnector(sc); @@ -325,7 +312,9 @@ public void log(Request request, Response response) { authWrapper = makeAuthWrapper(new DigestAuthenticator(), DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN, DIGEST_AUTH_PASSWORD); - authWrapper.setHandler(handlers); + HandlerCollection handlers2 = new HandlerCollection(); + handlers2.addHandler(new TestHandler()); + authWrapper.setHandler(handlers2); server.setHandler(authWrapper); server.start(); @@ -333,7 +322,7 @@ public void log(Request request, Response response) { return servers; } - + protected static void ensureHttpServers() throws Exception { if (httpServers == null) { httpServers = startHttpServers(); @@ -362,9 +351,4 @@ protected void tearDown() throws Exception { } }; } - - public static Request getLastRequest() { - return lastRequest; - } - } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 43de8044e..6ca6728cf 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -157,7 +157,7 @@ protected void runDefaultChecks(CrawlURI curi, String... exclusionsArray) assertEquals(DEFAULT_PAYLOAD_STRING, curi.getRecorder().getContentReplayCharSequence().toString()); if (!exclusions.contains("httpBindAddress")) { - assertEquals("127.0.0.1", FetchHTTPTest.getLastRequest().getRemoteAddr()); + assertTrue(rawResponseString(curi).contains("Client-Host: 127.0.0.1\r\n")); } if (!exclusions.contains("nonFatalFailuresIsEmpty")) { @@ -262,7 +262,7 @@ public void testBasicAuth() throws Exception { // check that we got the expected response and the fetcher did its thing assertEquals(401, curi.getFetchStatus()); - assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); + assertEquals("basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); assertTrue(curi.getCredentials().contains(basicAuthCredential)); assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic")); @@ -406,8 +406,8 @@ public void tryHttpBindAddress(String addr) throws Exception { fetcher().process(curi); // the client bind address isn't recorded anywhere in heritrix as - // far as i can tell, so we get it this way... - assertEquals(addr, FetchHTTPTest.getLastRequest().getRemoteAddr()); + // far as i can tell, so we get the server to echo it back to us... + assertTrue(rawResponseString(curi).contains("Client-Host: " + addr + "\r\n")); runDefaultChecks(curi, "httpBindAddress"); } @@ -780,7 +780,9 @@ public void testLaxUrlEncoding() throws Exception { fetcher().process(curi); // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertTrue(httpRequestString(curi).startsWith("GET /99% HTTP/1.0\r\n")); - runDefaultChecks(curi, "requestLine"); + // jetty 9 rejects requests with paths like this with 400 Bad Request + // so we can't run these checks anymore + //runDefaultChecks(curi, "requestLine"); } public void testTwoQuestionMarks() throws Exception { diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java index ea2980e06..5e40d6642 100644 --- a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -70,11 +70,11 @@ import org.archive.util.Base32; import org.archive.util.Recorder; import org.archive.util.TmpDirTestCase; -import org.mortbay.jetty.Request; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.bio.SocketConnector; -import org.mortbay.jetty.handler.HandlerCollection; -import org.mortbay.jetty.servlet.SessionHandler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.session.SessionHandler; public class ContentDigestHistoryTest extends TmpDirTestCase { @@ -237,7 +237,6 @@ public void testWarcDedupe() throws Exception { fetcher.process(curi1); assertEquals(200, curi1.getFetchStatus()); - assertEquals(141, curi1.getContentSize()); assertEquals(expectedDigest, curi1.getContentDigestSchemeString()); assertFalse(curi1.hasContentDigestHistory()); @@ -260,7 +259,6 @@ public void testWarcDedupe() throws Exception { fetcher.process(curi2); assertEquals(200, curi1.getFetchStatus()); - assertEquals(141, curi1.getContentSize()); assertEquals(expectedDigest, curi1.getContentDigestSchemeString()); assertFalse(curi2.hasContentDigestHistory()); @@ -305,7 +303,6 @@ public void testWarcDedupe() throws Exception { assertTrue(recordIterator.hasNext()); record = recordIterator.next(); assertEquals(WARCRecordType.response.toString(), record.getHeader().getHeaderValue(HEADER_KEY_TYPE)); - assertEquals("141", record.getHeader().getHeaderValue(CONTENT_LENGTH)); assertEquals(expectedDigest, record.getHeader().getHeaderValue(HEADER_KEY_PAYLOAD_DIGEST)); assertEquals(curi1.getUURI().toString(), record.getHeader().getHeaderValue(HEADER_KEY_URI)); assertEquals(payloadRecordIdWithBrackets, record.getHeader().getHeaderValue(HEADER_KEY_ID)); @@ -374,10 +371,7 @@ protected Server newHttpServer() throws Exception { HandlerCollection handlers = new HandlerCollection(); handlers.addHandler(new SessionHandler(){ @Override - public void handle(String target, HttpServletRequest request, - HttpServletResponse response, int dispatch) throws IOException, - ServletException { - + public void doHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); @@ -388,7 +382,7 @@ public void handle(String target, HttpServletRequest request, Server server = new Server(); server.setHandler(handlers); - SocketConnector sc = new SocketConnector(); + ServerConnector sc = new ServerConnector(server); sc.setHost("127.0.0.1"); sc.setPort(7777); server.addConnector(sc); diff --git a/modules/src/test/resources/log4j.xml b/modules/src/test/resources/log4j.xml index 475b130e2..e04f34ab8 100644 --- a/modules/src/test/resources/log4j.xml +++ b/modules/src/test/resources/log4j.xml @@ -11,7 +11,7 @@ - + diff --git a/pom.xml b/pom.xml index 4a9fac234..2cf5e5648 100644 --- a/pom.xml +++ b/pom.xml @@ -373,8 +373,8 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html maven-compiler-plugin 3.3 - 1.7 - 1.7 + 1.8 + 1.8 UTF-8