From 34832398b15c454f6e587286f061a94ed3700090 Mon Sep 17 00:00:00 2001 From: Andrew Rouse Date: Sat, 11 Feb 2017 00:16:28 +0000 Subject: [PATCH 1/7] Fix ordering of expected and actual arguments Also includes a tiny bit of reformatting --- .../fat/java/com/ibm/ws/lars/rest/ApiTest.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/server/src/fat/java/com/ibm/ws/lars/rest/ApiTest.java b/server/src/fat/java/com/ibm/ws/lars/rest/ApiTest.java index 99fc2e2..607324c 100644 --- a/server/src/fat/java/com/ibm/ws/lars/rest/ApiTest.java +++ b/server/src/fat/java/com/ibm/ws/lars/rest/ApiTest.java @@ -90,7 +90,7 @@ public class ApiTest { @Parameters(name = "{0}") public static Collection data() { return Arrays.asList(new Object[][] { { Protocol.HTTP }, - { Protocol.HTTPS } }); + { Protocol.HTTPS } }); } public ApiTest(Protocol protocol) { @@ -897,8 +897,7 @@ public void testGetAllAssetsWithSearch() throws IOException, InvalidJsonAssetExc // Testing phrases Asset testAsset9 = addLittleAsset(new String[] { "bill", "ben", "jack", "jill", "name", "a long name which can be searched" }); - Asset testAsset10 = addLittleAsset(new String[] - { "bill", "ben", "jack", "jill", "name", "long", "description", "name", "tags", "which can" }); + Asset testAsset10 = addLittleAsset(new String[] { "bill", "ben", "jack", "jill", "name", "long", "description", "name", "tags", "which can" }); // This should get just 9, as it is searching for a phrase String searchQuery = URLEncoder.encode("\"long name which\"", StandardCharsets.UTF_8.name()); @@ -988,20 +987,20 @@ public void testGetAllAssetsPagination() throws Exception { Asset asset4 = addLittleAsset("name", "asset4"); AssetList page1 = repository.getAllAssets("offset=0&limit=2"); - assertEquals("Wrong number of assets on page 1", page1.size(), 2); + assertEquals("Wrong number of assets on page 1", 2, page1.size()); AssetList page2 = repository.getAllAssets("offset=2&limit=2"); - assertEquals("Wrong number of assets on page 2", page2.size(), 2); + assertEquals("Wrong number of assets on page 2", 2, page2.size()); AssetList page3 = repository.getAllAssets("offset=4&limit=2"); - assertEquals("Wrong number of assets on page 3", page3.size(), 0); + assertEquals("Wrong number of assets on page 3", 0, page3.size()); assertThat(collatePages(page1, page2, page3), containsInAnyOrder(asset1, asset2, asset3, asset4)); page1 = repository.getAllAssets("offset=0&name=asset2|asset3|asset4&limit=2"); - assertEquals("Wrong number of assets on page 1", page1.size(), 2); + assertEquals("Wrong number of assets on page 1", 2, page1.size()); page2 = repository.getAllAssets("offset=2&name=asset2|asset3|asset4&limit=2"); - assertEquals("Wrong number of assets on page 2", page2.size(), 1); + assertEquals("Wrong number of assets on page 2", 1, page2.size()); page3 = repository.getAllAssets("offset=4&name=asset2|asset3|asset4&limit=2"); - assertEquals("Wrong number of assets on page 3", page3.size(), 0); + assertEquals("Wrong number of assets on page 3", 0, page3.size()); assertThat(collatePages(page1, page2, page3), containsInAnyOrder(asset2, asset3, asset4)); } From 5ff8ba3d3e6ecfdc849c2fdabaf723d5648de444 Mon Sep 17 00:00:00 2001 From: Andrew Rouse Date: Mon, 13 Feb 2017 13:56:25 +0000 Subject: [PATCH 2/7] Update copyright year for new files --- cli-client/.settings/org.eclipse.jdt.ui.prefs | 2 +- client-lib-tests/.settings/org.eclipse.jdt.ui.prefs | 2 +- client-lib/.settings/org.eclipse.jdt.ui.prefs | 2 +- server/.settings/org.eclipse.jdt.ui.prefs | 2 +- test-utils/.settings/org.eclipse.jdt.ui.prefs | 2 +- upload-lib/.settings/org.eclipse.jdt.ui.prefs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cli-client/.settings/org.eclipse.jdt.ui.prefs b/cli-client/.settings/org.eclipse.jdt.ui.prefs index 225100c..f68bcc8 100644 --- a/cli-client/.settings/org.eclipse.jdt.ui.prefs +++ b/cli-client/.settings/org.eclipse.jdt.ui.prefs @@ -70,7 +70,7 @@ org.eclipse.jdt.ui.keywordthis=false org.eclipse.jdt.ui.ondemandthreshold=99 org.eclipse.jdt.ui.overrideannotation=true org.eclipse.jdt.ui.staticondemandthreshold=99 -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= sp_cleanup.add_default_serial_version_id=true sp_cleanup.add_generated_serial_version_id=false sp_cleanup.add_missing_annotations=true diff --git a/client-lib-tests/.settings/org.eclipse.jdt.ui.prefs b/client-lib-tests/.settings/org.eclipse.jdt.ui.prefs index 57d45e2..c1252d5 100644 --- a/client-lib-tests/.settings/org.eclipse.jdt.ui.prefs +++ b/client-lib-tests/.settings/org.eclipse.jdt.ui.prefs @@ -70,7 +70,7 @@ org.eclipse.jdt.ui.keywordthis=false org.eclipse.jdt.ui.ondemandthreshold=99 org.eclipse.jdt.ui.overrideannotation=true org.eclipse.jdt.ui.staticondemandthreshold=99 -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= sp_cleanup.add_default_serial_version_id=true sp_cleanup.add_generated_serial_version_id=false sp_cleanup.add_missing_annotations=true diff --git a/client-lib/.settings/org.eclipse.jdt.ui.prefs b/client-lib/.settings/org.eclipse.jdt.ui.prefs index 57d45e2..c1252d5 100644 --- a/client-lib/.settings/org.eclipse.jdt.ui.prefs +++ b/client-lib/.settings/org.eclipse.jdt.ui.prefs @@ -70,7 +70,7 @@ org.eclipse.jdt.ui.keywordthis=false org.eclipse.jdt.ui.ondemandthreshold=99 org.eclipse.jdt.ui.overrideannotation=true org.eclipse.jdt.ui.staticondemandthreshold=99 -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= sp_cleanup.add_default_serial_version_id=true sp_cleanup.add_generated_serial_version_id=false sp_cleanup.add_missing_annotations=true diff --git a/server/.settings/org.eclipse.jdt.ui.prefs b/server/.settings/org.eclipse.jdt.ui.prefs index 225100c..f68bcc8 100644 --- a/server/.settings/org.eclipse.jdt.ui.prefs +++ b/server/.settings/org.eclipse.jdt.ui.prefs @@ -70,7 +70,7 @@ org.eclipse.jdt.ui.keywordthis=false org.eclipse.jdt.ui.ondemandthreshold=99 org.eclipse.jdt.ui.overrideannotation=true org.eclipse.jdt.ui.staticondemandthreshold=99 -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= sp_cleanup.add_default_serial_version_id=true sp_cleanup.add_generated_serial_version_id=false sp_cleanup.add_missing_annotations=true diff --git a/test-utils/.settings/org.eclipse.jdt.ui.prefs b/test-utils/.settings/org.eclipse.jdt.ui.prefs index 225100c..f68bcc8 100644 --- a/test-utils/.settings/org.eclipse.jdt.ui.prefs +++ b/test-utils/.settings/org.eclipse.jdt.ui.prefs @@ -70,7 +70,7 @@ org.eclipse.jdt.ui.keywordthis=false org.eclipse.jdt.ui.ondemandthreshold=99 org.eclipse.jdt.ui.overrideannotation=true org.eclipse.jdt.ui.staticondemandthreshold=99 -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= sp_cleanup.add_default_serial_version_id=true sp_cleanup.add_generated_serial_version_id=false sp_cleanup.add_missing_annotations=true diff --git a/upload-lib/.settings/org.eclipse.jdt.ui.prefs b/upload-lib/.settings/org.eclipse.jdt.ui.prefs index 225100c..f68bcc8 100644 --- a/upload-lib/.settings/org.eclipse.jdt.ui.prefs +++ b/upload-lib/.settings/org.eclipse.jdt.ui.prefs @@ -70,7 +70,7 @@ org.eclipse.jdt.ui.keywordthis=false org.eclipse.jdt.ui.ondemandthreshold=99 org.eclipse.jdt.ui.overrideannotation=true org.eclipse.jdt.ui.staticondemandthreshold=99 -org.eclipse.jdt.ui.text.custom_code_templates= +org.eclipse.jdt.ui.text.custom_code_templates= sp_cleanup.add_default_serial_version_id=true sp_cleanup.add_generated_serial_version_id=false sp_cleanup.add_missing_annotations=true From 4bc992696938ae927f6e043381714488b58ba36b Mon Sep 17 00:00:00 2001 From: Andrew Rouse Date: Mon, 13 Feb 2017 15:05:45 +0000 Subject: [PATCH 3/7] Reformat RepositoryRESTResource Newer version of Eclipse wants to reformat this class when I save, so let's separate the reformatting changes into another commit. --- .../ws/lars/rest/RepositoryRESTResource.java | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java b/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java index 49ed5c0..9806cfc 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java @@ -265,8 +265,7 @@ public Response createAttachmentWithContent(@QueryParam("name") String name, @PathParam("assetId") String assetId, @Context HttpServletRequest request, BufferedInMultiPart inMultiPart, - @Context UriInfo uriInfo - ) throws InvalidJsonAssetException, InvalidIdException, AssetPersistenceException, NonExistentArtefactException { + @Context UriInfo uriInfo) throws InvalidJsonAssetException, InvalidIdException, AssetPersistenceException, NonExistentArtefactException { if (logger.isLoggable(Level.FINE)) { logger.fine("createAttachmentWithContent called, name: " + name + " assetId: " + assetId); @@ -326,8 +325,8 @@ public Response createAttachmentNoContent(@QueryParam("name") String name, @Path("/assets/{assetId}/attachments") @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({ ADMIN_ROLE, USER_ROLE }) - public Response getAttachments(@PathParam("assetId") String assetId, @Context UriInfo uriInfo, @Context SecurityContext sc) - throws InvalidIdException, NonExistentArtefactException, JsonProcessingException { + public Response getAttachments(@PathParam("assetId") String assetId, @Context UriInfo uriInfo, + @Context SecurityContext sc) throws InvalidIdException, NonExistentArtefactException, JsonProcessingException { if (logger.isLoggable(Level.FINE)) { logger.fine("getAttachments called for assetId: " + assetId); } @@ -392,9 +391,7 @@ public Response getAttachmentContent(@PathParam("assetId") String assetId, final InputStream contentInputStream = contentResponse.getContentStream(); StreamingOutput stream = new InputStreamStreamingOutput(contentInputStream); - return Response.ok(stream) - .header("Content-Type", contentResponse.getContentType()) - .build(); + return Response.ok(stream).header("Content-Type", contentResponse.getContentType()).build(); } else { String body = getErrorJson(Response.Status.NOT_FOUND, "Could not find attachment for id " + attachmentId); return Response.status(Response.Status.NOT_FOUND).entity(body).build(); @@ -405,8 +402,7 @@ public Response getAttachmentContent(@PathParam("assetId") String assetId, @Path("/assets/{assetId}/state") @Produces(MediaType.APPLICATION_JSON) @RolesAllowed(ADMIN_ROLE) - public Response updateAssetState(@PathParam("assetId") String assetId, String actionJSON) - throws NonExistentArtefactException, RepositoryResourceLifecycleException { + public Response updateAssetState(@PathParam("assetId") String assetId, String actionJSON) throws NonExistentArtefactException, RepositoryResourceLifecycleException { if (logger.isLoggable(Level.FINE)) { logger.fine("updateAssetState called for assetId: " + assetId + " action: " + actionJSON); } @@ -414,9 +410,7 @@ public Response updateAssetState(@PathParam("assetId") String assetId, String ac Asset.StateAction action = getStateAction(actionJSON); if (action == null) { String error = "Either the supplied JSON was badly formed, or it did not contain a valid 'action' field: " + actionJSON; - return Response.status(Response.Status.BAD_REQUEST) - .entity(getErrorJson(Response.Status.BAD_REQUEST, error)) - .build(); + return Response.status(Response.Status.BAD_REQUEST).entity(getErrorJson(Response.Status.BAD_REQUEST, error)).build(); } assetService.updateAssetState(action, assetId); @@ -426,7 +420,8 @@ public Response updateAssetState(@PathParam("assetId") String assetId, String ac @GET @Path("/assets/{assetId}/assetreviews") @Produces(MediaType.APPLICATION_JSON) - public Response getAssetReviews(@PathParam("assetId") String assetId, @Context UriInfo uriInfo, @Context SecurityContext sc) throws InvalidIdException, NonExistentArtefactException { + public Response getAssetReviews(@PathParam("assetId") String assetId, @Context UriInfo uriInfo, + @Context SecurityContext sc) throws InvalidIdException, NonExistentArtefactException { if (logger.isLoggable(Level.FINE)) { logger.fine("getAssetReviews called with id of '" + assetId + "'"); } From d8d485324d51412bed7e61f0f0d7a7f6975e5e3d Mon Sep 17 00:00:00 2001 From: Andrew Rouse Date: Fri, 10 Feb 2017 22:54:32 +0000 Subject: [PATCH 4/7] Reduce memory usage when returning large result sets This change allows results to be streamed from the database into a response to the client when retrieving multiple assets. It avoids needing to load the whole result set into memory before returning it. To do this we introduce the AssetCursor, which wraps a database result cursor and allows the Rest layer to iterate through the results without needing to know the details of the persistence backend. To allow changes to be made to the information from the database before it is consumed by the rest layer we introduce the AssetOperation, which encapsulates some processing which should be done to the asset before it is returned (e.g. transforming between data types or removing implementation-specific fields) --- .../ibm/ws/lars/rest/PersistenceBeanTest.java | 88 ++++++++------- .../com/ibm/ws/lars/rest/AssetConverter.java | 34 ++++++ .../ibm/ws/lars/rest/AssetCursorWriter.java | 58 ++++++++++ .../ibm/ws/lars/rest/AssetServiceLayer.java | 6 +- .../com/ibm/ws/lars/rest/PersistenceBean.java | 104 +++++++++--------- .../java/com/ibm/ws/lars/rest/Persistor.java | 6 +- .../ws/lars/rest/RepositoryRESTResource.java | 7 +- .../com/ibm/ws/lars/rest/model/Asset.java | 6 +- .../ibm/ws/lars/rest/model/AssetCursor.java | 53 +++++++++ .../ws/lars/rest/model/AssetOperation.java | 25 +++++ .../ws/lars/rest/model/MongoAssetCursor.java | 65 +++++++++++ .../ws/lars/rest/AssetServiceLayerTest.java | 8 +- .../ibm/ws/lars/rest/BasicAssetCursor.java | 64 +++++++++++ .../com/ibm/ws/lars/rest/MemoryPersistor.java | 11 +- 14 files changed, 423 insertions(+), 112 deletions(-) create mode 100644 server/src/main/java/com/ibm/ws/lars/rest/AssetConverter.java create mode 100644 server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java create mode 100644 server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java create mode 100644 server/src/main/java/com/ibm/ws/lars/rest/model/AssetOperation.java create mode 100644 server/src/main/java/com/ibm/ws/lars/rest/model/MongoAssetCursor.java create mode 100644 server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java diff --git a/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java b/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java index 5cbb823..c4db421 100644 --- a/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java +++ b/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java @@ -39,8 +39,6 @@ import java.util.Map; import java.util.logging.Logger; -import mockit.Mocked; - import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -49,7 +47,7 @@ import com.ibm.ws.lars.rest.exceptions.InvalidJsonAssetException; import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.model.Asset; -import com.ibm.ws.lars.rest.model.AssetList; +import com.ibm.ws.lars.rest.model.AssetCursor; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.AttachmentContentMetadata; import com.ibm.ws.lars.testutils.BasicChecks; @@ -58,6 +56,8 @@ import com.mongodb.MongoClient; import com.mongodb.WriteConcern; +import mockit.Mocked; + public class PersistenceBeanTest { // TODO Should the db name be configurable? @@ -116,7 +116,7 @@ public void tearDown() throws Exception { } private void assertEmpty() throws IOException { - AssetList allAssets = persistenceBean.retrieveAllAssets(); + AssetCursor allAssets = persistenceBean.retrieveAllAssets(); assertTrue(allAssets.size() == 0); } @@ -235,14 +235,14 @@ public void testRetrieveAllFiltered() throws Exception { List filters = new ArrayList<>(); filters.add(new AssetFilter("name", Arrays.asList(eq("new name1")))); - AssetList assets = persistenceBean.retrieveAllAssets(filters, null, null, null); + List assets = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertEquals("Should only have got 1 asset back", 1, assets.size()); assertEquals("Got the wrong asset back", asset1.get_id(), assets.get(0).get_id()); List filters2 = new ArrayList<>(); filters2.add(new AssetFilter("layer1.layer1field", Arrays.asList(eq("layer1value")))); - AssetList assets2 = persistenceBean.retrieveAllAssets(filters2, null, null, null); + List assets2 = readAll(persistenceBean.retrieveAllAssets(filters2, null, null, null)); assertEquals("Should have got 2 asset back", 2, assets2.size()); for (Asset retrievedAsset : assets2) { if (!retrievedAsset.get_id().equals(asset1.get_id()) && !retrievedAsset.get_id().equals(asset2.get_id())) { @@ -253,7 +253,7 @@ public void testRetrieveAllFiltered() throws Exception { List filters3 = new ArrayList<>(); filters3.add(new AssetFilter("name", Arrays.asList(eq("new name1"), eq("new name2")))); - AssetList assets3 = persistenceBean.retrieveAllAssets(filters3, null, null, null); + List assets3 = readAll(persistenceBean.retrieveAllAssets(filters3, null, null, null)); assertEquals("Should have got 2 asset back", 2, assets3.size()); for (Asset retrievedAsset : assets3) { if (!retrievedAsset.get_id().equals(asset1.get_id()) && !retrievedAsset.get_id().equals(asset2.get_id())) { @@ -264,7 +264,7 @@ public void testRetrieveAllFiltered() throws Exception { // With a search term as well List filters4 = new ArrayList<>(); filters4.add(new AssetFilter("name", Arrays.asList(eq("new name1"), eq("new name2")))); - AssetList assets4 = persistenceBean.retrieveAllAssets(filters4, "name1", null, null); + List assets4 = readAll(persistenceBean.retrieveAllAssets(filters4, "name1", null, null)); assertEquals("Wrong number of assets retrieved", 1, assets4.size()); Asset retrieved = assets4.get(0); assertEquals("Got the wrong asset back", asset1.get_id(), retrieved.get_id()); @@ -282,13 +282,13 @@ public void testRetrieveAllFiltered2() throws Exception { // Test with an empty set of filters List emptyFilters = Collections.emptyList(); - AssetList assets = persistenceBean.retrieveAllAssets(emptyFilters, null, null, null); + List assets = readAll(persistenceBean.retrieveAllAssets(emptyFilters, null, null, null)); assertEquals("An empty filter should get all assets", 4, assets.size()); List filters = new ArrayList<>(); // test which retrieves no assets filters.add(new AssetFilter("blurgh", Arrays.asList(eq("new name1")))); - AssetList assets2 = persistenceBean.retrieveAllAssets(filters, null, null, null); + List assets2 = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertEquals("Should not have got any assets back", 0, assets2.size()); // test which uses multiple entries in the map @@ -296,7 +296,7 @@ public void testRetrieveAllFiltered2() throws Exception { filters.add(new AssetFilter("name", Arrays.asList(eq("new name1")))); filters.add(new AssetFilter("layer1.layer1field", Arrays.asList(eq("layer1value")))); - AssetList assets3 = persistenceBean.retrieveAllAssets(filters, null, null, null); + List assets3 = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertEquals("Wrong number of assets retrieved", 2, assets3.size()); String id1 = asset1.get_id(); String id2 = asset2.get_id(); @@ -318,7 +318,7 @@ public void testRetrieveAllAssetsNotFiltered() throws InvalidJsonAssetException // Empty filters should get everything List emptyFilters = Collections.emptyList(); - AssetList allAssets = persistenceBean.retrieveAllAssets(emptyFilters, null, null, null); + List allAssets = readAll(persistenceBean.retrieveAllAssets(emptyFilters, null, null, null)); assertEquals("Unexpected number of assets returned", 3, allAssets.size()); List filters; @@ -326,14 +326,14 @@ public void testRetrieveAllAssetsNotFiltered() throws InvalidJsonAssetException // query that should return nothing filters = new ArrayList<>(); filters.add(new AssetFilter("layer1.layer1field", Arrays.asList(neq("layer1value")))); - AssetList emptyAssets = persistenceBean.retrieveAllAssets(filters, null, null, null); + List emptyAssets = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertEquals("Unexpected number of assets returned", 0, emptyAssets.size()); filters.clear(); // basic not filter filters = new ArrayList<>(); filters.add(new AssetFilter("name", Arrays.asList(neq("new name1")))); - AssetList assets1 = persistenceBean.retrieveAllAssets(filters, null, null, null); + List assets1 = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertEquals("Unexpected number of assets returned", 1, assets1.size()); assertEquals("The wrong asset id was retrieved", asset3.get_id(), assets1.get(0).get_id()); @@ -342,7 +342,7 @@ public void testRetrieveAllAssetsNotFiltered() throws InvalidJsonAssetException filters = new ArrayList<>(); filters.add(new AssetFilter("name", Arrays.asList(neq("new name1")))); filters.add(new AssetFilter("layer1.layer1field", Arrays.asList(eq("layer1value2")))); - AssetList assets2 = persistenceBean.retrieveAllAssets(filters, null, null, null); + List assets2 = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertEquals("Unexpected number of assets returned", 1, assets2.size()); assertEquals("The wrong asset id was retrieved", asset4.get_id(), assets2.get(0).get_id()); @@ -350,7 +350,7 @@ public void testRetrieveAllAssetsNotFiltered() throws InvalidJsonAssetException filters = new ArrayList<>(); filters.add(new AssetFilter("name", Arrays.asList(neq("new name1")))); filters.add(new AssetFilter("layer1.layer1field", Arrays.asList(eq("layer1value2")))); - AssetList assets3 = persistenceBean.retrieveAllAssets(filters, "\"new name2\"", null, null); + List assets3 = readAll(persistenceBean.retrieveAllAssets(filters, "\"new name2\"", null, null)); assertEquals("Unexpected number of assets returned", 1, assets3.size()); assertEquals("The wrong asset id was retrieved", asset4.get_id(), assets3.get(0).get_id()); @@ -370,7 +370,7 @@ public void testRetrieveAllAssetsOrFiltered() throws InvalidJsonAssetException { // Empty filters should get everything List emptyFilters = Collections.emptyList(); - AssetList allAssets = persistenceBean.retrieveAllAssets(emptyFilters, null, null, null); + List allAssets = readAll(persistenceBean.retrieveAllAssets(emptyFilters, null, null, null)); assertEquals("Unexpected number of assets returned", 9, allAssets.size()); assertThat(allAssets, containsInAnyOrder(assetsWithIds(asset1, asset2, asset3, asset4, asset5, asset6, asset7, asset8, asset9))); @@ -379,28 +379,28 @@ public void testRetrieveAllAssetsOrFiltered() throws InvalidJsonAssetException { // Simple OR filter filters = new ArrayList<>(); filters.add(new AssetFilter("weather", Arrays.asList(eq("hot"), eq("warm")))); - AssetList result1 = persistenceBean.retrieveAllAssets(filters, null, null, null); + List result1 = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertThat(result1, containsInAnyOrder(assetsWithIds(asset1, asset2, asset3, asset7, asset8, asset9))); // OR with NOT filters = new ArrayList<>(); filters.add(new AssetFilter("weather", Arrays.asList(eq("hot"), eq("warm")))); filters.add(new AssetFilter("ground", Arrays.asList(neq("mountainous")))); - AssetList result2 = persistenceBean.retrieveAllAssets(filters, null, null, null); + List result2 = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertThat(result2, containsInAnyOrder(assetsWithIds(asset1, asset2, asset7, asset8))); // Two ORs filters = new ArrayList<>(); filters.add(new AssetFilter("weather", Arrays.asList(eq("hot"), eq("warm")))); filters.add(new AssetFilter("ground", Arrays.asList(eq("hilly"), eq("mountainous")))); - AssetList result3 = persistenceBean.retrieveAllAssets(filters, null, null, null); + List result3 = readAll(persistenceBean.retrieveAllAssets(filters, null, null, null)); assertThat(result3, containsInAnyOrder(assetsWithIds(asset2, asset3, asset8, asset9))); // OR with NOT and searchTerm filters = new ArrayList<>(); filters.add(new AssetFilter("weather", Arrays.asList(eq("hot"), eq("warm")))); filters.add(new AssetFilter("ground", Arrays.asList(neq("mountainous")))); - AssetList result4 = persistenceBean.retrieveAllAssets(filters, "long", null, null); + List result4 = readAll(persistenceBean.retrieveAllAssets(filters, "long", null, null)); assertThat(result4, containsInAnyOrder(assetsWithIds(asset1, asset2, asset7))); } @@ -445,29 +445,29 @@ public void testPagination() throws Exception { List emptyFilter = Collections.emptyList(); // Test 2 per page - AssetList page1 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(0, 2), null); + List page1 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(0, 2), null)); assertEquals("Wrong number of assets on page 1", 2, page1.size()); - AssetList page2 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(2, 2), null); + List page2 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(2, 2), null)); assertEquals("Wrong number of assets on page 2", 2, page2.size()); - AssetList page3 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(4, 2), null); + List page3 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(4, 2), null)); assertEquals("Wrong number of assets on page 3", 0, page3.size()); assertThat(collatePages(page1, page2, page3), containsInAnyOrder(asset1, asset2, asset3, asset4)); // Test 3 per page - page1 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(0, 3), null); + page1 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(0, 3), null)); assertEquals("Wrong number of assets on page 1", 3, page1.size()); - page2 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(3, 3), null); + page2 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(3, 3), null)); assertEquals("Wrong number of assets on page 2", 1, page2.size()); - page3 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(6, 3), null); + page3 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(6, 3), null)); assertEquals("Wrong number of assets on page 3", 0, page3.size()); assertThat(collatePages(page1, page2, page3), containsInAnyOrder(asset1, asset2, asset3, asset4)); // Test with filter List conditions = new ArrayList<>(); conditions.add(new AssetFilter("name", Arrays.asList(eq("asset2"), eq("asset3"), eq("asset4")))); - page1 = persistenceBean.retrieveAllAssets(conditions, null, new PaginationOptions(0, 2), null); + page1 = readAll(persistenceBean.retrieveAllAssets(conditions, null, new PaginationOptions(0, 2), null)); assertEquals("Wrong number of assets on page 1", 2, page1.size()); - page2 = persistenceBean.retrieveAllAssets(conditions, null, new PaginationOptions(2, 2), null); + page2 = readAll(persistenceBean.retrieveAllAssets(conditions, null, new PaginationOptions(2, 2), null)); assertEquals("Wrong number of assets on page 2", 1, page2.size()); assertThat(collatePages(page1, page2), containsInAnyOrder(asset2, asset3, asset4)); } @@ -481,26 +481,26 @@ public void testSortOptions() throws Exception { List emptyFilter = Collections.emptyList(); - AssetList result = persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("name", ASCENDING)); + List result = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("name", ASCENDING))); assertThat(result, contains(asset1, asset2, asset3, asset4)); - result = persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("name", DESCENDING)); + result = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("name", DESCENDING))); assertThat(result, contains(asset4, asset3, asset2, asset1)); // Missing values should be the "lowest" - result = persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("score", ASCENDING)); + result = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("score", ASCENDING))); assertThat(result, contains(asset1, asset4, asset3, asset2)); - result = persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("score", DESCENDING)); + result = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("score", DESCENDING))); assertThat(result, contains(asset2, asset3, asset4, asset1)); // Sort by something non-existent, the order is undefined but it should return all assets - result = persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("wibble", ASCENDING)); + result = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, null, new SortOptions("wibble", ASCENDING))); assertThat(result, containsInAnyOrder(asset1, asset2, asset3, asset4)); // Test sorting with pagination - AssetList page1 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(0, 2), new SortOptions("name", ASCENDING)); - AssetList page2 = persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(2, 2), new SortOptions("name", ASCENDING)); + List page1 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(0, 2), new SortOptions("name", ASCENDING))); + List page2 = readAll(persistenceBean.retrieveAllAssets(emptyFilter, null, new PaginationOptions(2, 2), new SortOptions("name", ASCENDING))); assertThat(collatePages(page1, page2), contains(asset1, asset2, asset3, asset4)); } @@ -548,11 +548,11 @@ public void testCountAllAssets() throws Exception { * @param lists the AssetLists * @return the collated list */ - private static List collatePages(AssetList... lists) { + private static List collatePages(List... lists) { List result = new ArrayList(); - for (AssetList list : lists) { - for (Asset asset : list) { - result.add(asset); + for (Object list : lists) { + for (Object asset : (Iterable) list) { + result.add((Asset) asset); } } return result; @@ -583,4 +583,12 @@ private static Condition eq(String value) { private static Condition neq(String value) { return new Condition(Operation.NOT_EQUALS, value); } + + private static List readAll(AssetCursor cursor) { + List result = new ArrayList<>(); + while (cursor.hasNext()) { + result.add(cursor.next()); + } + return result; + } } diff --git a/server/src/main/java/com/ibm/ws/lars/rest/AssetConverter.java b/server/src/main/java/com/ibm/ws/lars/rest/AssetConverter.java new file mode 100644 index 0000000..111e936 --- /dev/null +++ b/server/src/main/java/com/ibm/ws/lars/rest/AssetConverter.java @@ -0,0 +1,34 @@ +/******************************************************************************* + * Copyright (c) 2017 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.ibm.ws.lars.rest; + +import java.util.Map; + +import com.fasterxml.jackson.databind.util.StdConverter; +import com.ibm.ws.lars.rest.model.Asset; + +/** + * + */ +public class AssetConverter extends StdConverter> { + + /** {@inheritDoc} */ + @Override + public Map convert(Asset value) { + return value.getProperties(); + } + +} diff --git a/server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java b/server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java new file mode 100644 index 0000000..59d7a86 --- /dev/null +++ b/server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java @@ -0,0 +1,58 @@ +/******************************************************************************* + * Copyright (c) 2017 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.ibm.ws.lars.rest; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyWriter; +import javax.ws.rs.ext.Provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ibm.ws.lars.rest.model.AssetCursor; + +/** + * + */ +@Provider +public class AssetCursorWriter implements MessageBodyWriter { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** {@inheritDoc} */ + @Override + public long getSize(AssetCursor arg0, Class arg1, Type arg2, Annotation[] arg3, MediaType arg4) { + return -1; + } + + /** {@inheritDoc} */ + @Override + public boolean isWriteable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { + return AssetCursor.class.isAssignableFrom(type) && mediaType.isCompatible(MediaType.APPLICATION_JSON_TYPE); + } + + /** {@inheritDoc} */ + @Override + public void writeTo(AssetCursor cursor, Class type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap httpHeaders, + OutputStream entityStream) throws IOException { + MAPPER.writeValue(entityStream, cursor); + } + +} diff --git a/server/src/main/java/com/ibm/ws/lars/rest/AssetServiceLayer.java b/server/src/main/java/com/ibm/ws/lars/rest/AssetServiceLayer.java index 0174f9e..fb9eb5c 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/AssetServiceLayer.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/AssetServiceLayer.java @@ -36,7 +36,7 @@ import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.exceptions.RepositoryException; import com.ibm.ws.lars.rest.model.Asset; -import com.ibm.ws.lars.rest.model.AssetList; +import com.ibm.ws.lars.rest.model.AssetCursor; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.AttachmentContentMetadata; import com.ibm.ws.lars.rest.model.AttachmentContentResponse; @@ -62,14 +62,14 @@ public class AssetServiceLayer { /** * @see Persistor#retrieveAllAssets() */ - public AssetList retrieveAllAssets() { + public AssetCursor retrieveAllAssets() { return persistenceBean.retrieveAllAssets(); } /** * @see Persistor#retrieveAllAssets(Collection,String, PaginationOptions, SortOptions) */ - public AssetList retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions) { + public AssetCursor retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions) { return persistenceBean.retrieveAllAssets(filters, searchTerm, pagination, sortOptions); } diff --git a/server/src/main/java/com/ibm/ws/lars/rest/PersistenceBean.java b/server/src/main/java/com/ibm/ws/lars/rest/PersistenceBean.java index 76212ea..66d9b1b 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/PersistenceBean.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/PersistenceBean.java @@ -37,11 +37,13 @@ import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.exceptions.RepositoryException; import com.ibm.ws.lars.rest.model.Asset; -import com.ibm.ws.lars.rest.model.AssetList; +import com.ibm.ws.lars.rest.model.AssetCursor; +import com.ibm.ws.lars.rest.model.AssetOperation; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.AttachmentContentMetadata; import com.ibm.ws.lars.rest.model.AttachmentContentResponse; import com.ibm.ws.lars.rest.model.AttachmentList; +import com.ibm.ws.lars.rest.model.MongoAssetCursor; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObjectBuilder; @@ -78,7 +80,7 @@ public class PersistenceBean implements Persistor { Arrays.asList(new String[] { "name", "description", "shortDescription", "tags" }); /** The _id field of a MongoDB object */ - private static String ID = "_id"; + public static final String ID = "_id"; private static final String DB_NAME = "mongo/larsDB"; @@ -113,6 +115,25 @@ private static void convertObjectIdToHexString(DBObject obj) { } } + private static AssetOperation CONVERT_OID_TO_HEX = new AssetOperation() { + @Override + public void perform(Asset asset) { + Object objectIdObject = asset.getProperty(ID); + if ((objectIdObject != null) && (objectIdObject instanceof ObjectId)) { + ObjectId objId = (ObjectId) objectIdObject; + String hex = objId.toHexString(); + asset.setProperty(ID, hex); + } + } + }; + + private static AssetOperation REMOVE_SCORE = new AssetOperation() { + @Override + public void perform(Asset asset) { + asset.getProperties().remove("score"); + } + }; + private static void convertHexIdToObjectId(DBObject obj) { Object idObject = obj.get(ID); if ((idObject != null) && (idObject instanceof String)) { @@ -122,29 +143,21 @@ private static void convertHexIdToObjectId(DBObject obj) { } @Override - public AssetList retrieveAllAssets() { - List> mapList = new ArrayList<>(); - - try (DBCursor cursor = getAssetCollection().find()) { - if (logger.isLoggable(Level.FINE)) { - logger.fine("retrieveAllAssets: found " + cursor.count() + " assets."); - } - for (DBObject obj : cursor) { - convertObjectIdToHexString(obj); - // BSON spec says that all keys have to be strings - // so this should be safe. - @SuppressWarnings("unchecked") - Map assetMap = obj.toMap(); - mapList.add(assetMap); - } + public AssetCursor retrieveAllAssets() { + DBCursor cursor = getAssetCollection().find(); + if (logger.isLoggable(Level.FINE)) { + logger.fine("retrieveAllAssets: found " + cursor.count() + " assets."); } - return AssetList.createAssetListFromMaps(mapList); + AssetCursor assetCursor = new MongoAssetCursor(cursor); + assetCursor.addOperation(CONVERT_OID_TO_HEX); + + return assetCursor; } /** {@inheritDoc} */ @Override - public AssetList retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions) { + public AssetCursor retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions) { if (filters.size() == 0 && searchTerm == null && pagination == null && sortOptions == null) { return retrieveAllAssets(); @@ -169,19 +182,13 @@ public AssetList retrieveAllAssets(Collection filters, String searc } } - List results = query(filterObject, sortObject, projectionObject, pagination); - List> assets = new ArrayList>(); - for (DBObject result : results) { - // BSON spec says that all keys have to be strings - // so this should be safe. - @SuppressWarnings("unchecked") - Map resultMap = result.toMap(); - if (textScoreAdded) { - resultMap.remove("score"); - } - assets.add(resultMap); + AssetCursor results = query(filterObject, sortObject, projectionObject, pagination); + + if (textScoreAdded) { + results.addOperation(REMOVE_SCORE); } - return AssetList.createAssetListFromMaps(assets); + + return results; } /** {@inheritDoc} */ @@ -253,7 +260,7 @@ private BasicDBObject createFilterObject(String field, Condition condition) { return new BasicDBObject(field, value); } - private List query(DBObject filterObject, DBObject sortObject, DBObject projectionObject, PaginationOptions pagination) { + private AssetCursor query(DBObject filterObject, DBObject sortObject, DBObject projectionObject, PaginationOptions pagination) { if (logger.isLoggable(Level.FINE)) { logger.fine("query: Querying database with query object " + filterObject); @@ -262,27 +269,24 @@ private List query(DBObject filterObject, DBObject sortObject, DBObjec logger.fine("query: pagination object " + pagination); } - List results = new ArrayList(); - try (DBCursor cursor = getAssetCollection().find(filterObject, projectionObject)) { - if (logger.isLoggable(Level.FINE)) { - logger.fine("query: found " + cursor.count() + " assets."); - } - - if (pagination != null) { - cursor.skip(pagination.getOffset()); - cursor.limit(pagination.getLimit()); - } + DBCursor cursor = getAssetCollection().find(filterObject, projectionObject); + if (logger.isLoggable(Level.FINE)) { + logger.fine("query: found " + cursor.count() + " assets."); + } - if (sortObject != null) { - cursor.sort(sortObject); - } + if (pagination != null) { + cursor.skip(pagination.getOffset()); + cursor.limit(pagination.getLimit()); + } - for (DBObject obj : cursor) { - convertObjectIdToHexString(obj); - results.add(obj); - } + if (sortObject != null) { + cursor.sort(sortObject); } - return results; + + AssetCursor result = new MongoAssetCursor(cursor); + result.addOperation(CONVERT_OID_TO_HEX); + + return result; } private int queryCount(DBObject filterObject) { diff --git a/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java b/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java index 1ce4c20..2adeeaa 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java @@ -24,7 +24,7 @@ import com.ibm.ws.lars.rest.exceptions.InvalidJsonAssetException; import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.model.Asset; -import com.ibm.ws.lars.rest.model.AssetList; +import com.ibm.ws.lars.rest.model.AssetCursor; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.AttachmentContentMetadata; import com.ibm.ws.lars.rest.model.AttachmentContentResponse; @@ -35,7 +35,7 @@ */ public interface Persistor { - public AssetList retrieveAllAssets(); + public AssetCursor retrieveAllAssets(); /** * Retrieve a list of assets, filtered based on the supplied set of filters. See the AssetFilter @@ -56,7 +56,7 @@ public interface Persistor { * @param sortOptions options describing how to sort the results, may be null if the results are * not to be sorted */ - public AssetList retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions); + public AssetCursor retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions); /** * Retrieve the number of assets which match the given set of filters. diff --git a/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java b/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java index 9806cfc..ff6c75d 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/RepositoryRESTResource.java @@ -67,7 +67,7 @@ import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.exceptions.RepositoryException; import com.ibm.ws.lars.rest.model.Asset; -import com.ibm.ws.lars.rest.model.AssetList; +import com.ibm.ws.lars.rest.model.AssetCursor; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.AttachmentContentResponse; import com.ibm.ws.lars.rest.model.AttachmentList; @@ -135,9 +135,8 @@ public Response getAssets(@Context UriInfo info, @Context SecurityContext contex filters.add(ASSET_IS_PUBLISHED); } - AssetList assets = assetService.retrieveAllAssets(filters, params.getSearchTerm(), params.getPagination(), params.getSortOptions()); - String json = assets.toJson(); - return Response.ok(json).build(); + AssetCursor assets = assetService.retrieveAllAssets(filters, params.getSearchTerm(), params.getPagination(), params.getSortOptions()); + return Response.ok(assets).build(); } @HEAD diff --git a/server/src/main/java/com/ibm/ws/lars/rest/model/Asset.java b/server/src/main/java/com/ibm/ws/lars/rest/model/Asset.java index bc86657..d9fb5f6 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/model/Asset.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/model/Asset.java @@ -21,6 +21,8 @@ import java.util.List; import java.util.Map; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.ibm.ws.lars.rest.AssetConverter; import com.ibm.ws.lars.rest.exceptions.InvalidJsonAssetException; import com.ibm.ws.lars.rest.exceptions.RepositoryException; @@ -29,6 +31,7 @@ * service. * */ +@JsonSerialize(converter = AssetConverter.class) public class Asset extends RepositoryObject { public static final String ATTACHMENTS = "attachments"; @@ -307,8 +310,7 @@ public enum StateAction implements PerformAction { public void performAction(Asset asset) throws RepositoryResourceLifecycleException { asset.getState().publish(asset); } - } - , + }, APPROVE("approve") { @Override public void performAction(Asset asset) throws RepositoryResourceLifecycleException { diff --git a/server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java b/server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java new file mode 100644 index 0000000..9037580 --- /dev/null +++ b/server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java @@ -0,0 +1,53 @@ +/******************************************************************************* + * Copyright (c) 2017 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.ibm.ws.lars.rest.model; + +import java.util.Iterator; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +/** + * An iterator over a set of Assets + *

+ * The caller can iterate over a set of Assets, without regard for where they're coming from. + *

+ * For instance, the implementation may be iterating over an in-memory collection, or it may be + * retrieving results from a database and transforming them into assets on demand before returning + * them to the caller. + *

+ * Any {@link AssetOperation}s added to the cursor will be applied to each Asset before it is + * returned. + */ +@JsonSerialize(as = Iterator.class) +public interface AssetCursor extends Iterator { + + /** + * Get the total number of assets which will be returned from this cursor + * + * @return the number of assets + */ + public int size(); + + /** + * Add an {@link AssetOperation} to be applied to each asset before it is returned from this + * cursor + *

+ * Operations are applied in the order they were added to the cursor. + * + * @param op the operation to add + */ + public void addOperation(AssetOperation op); +} diff --git a/server/src/main/java/com/ibm/ws/lars/rest/model/AssetOperation.java b/server/src/main/java/com/ibm/ws/lars/rest/model/AssetOperation.java new file mode 100644 index 0000000..d67fdda --- /dev/null +++ b/server/src/main/java/com/ibm/ws/lars/rest/model/AssetOperation.java @@ -0,0 +1,25 @@ +/******************************************************************************* + * Copyright (c) 2017 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.ibm.ws.lars.rest.model; + +/** + * An operation to be performed on an asset at some later point + * + * @see AssetCursor + */ +public interface AssetOperation { + public void perform(Asset asset); +} diff --git a/server/src/main/java/com/ibm/ws/lars/rest/model/MongoAssetCursor.java b/server/src/main/java/com/ibm/ws/lars/rest/model/MongoAssetCursor.java new file mode 100644 index 0000000..e21a16c --- /dev/null +++ b/server/src/main/java/com/ibm/ws/lars/rest/model/MongoAssetCursor.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (c) 2017 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.ibm.ws.lars.rest.model; + +import java.util.ArrayList; +import java.util.List; + +import com.mongodb.DBCursor; + +/** + * + */ +public class MongoAssetCursor implements AssetCursor { + + private final DBCursor cursor; + private final List operations = new ArrayList<>(); + + public MongoAssetCursor(DBCursor cursor) { + this.cursor = cursor; + } + + @Override + public boolean hasNext() { + return cursor.hasNext(); + } + + @SuppressWarnings("unchecked") + @Override + public Asset next() { + Asset next = Asset.createAssetFromMap(cursor.next().toMap()); + for (AssetOperation op : operations) { + op.perform(next); + } + return next; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public int size() { + return cursor.size(); + } + + @Override + public void addOperation(AssetOperation op) { + operations.add(op); + } + +} diff --git a/server/src/test/java/com/ibm/ws/lars/rest/AssetServiceLayerTest.java b/server/src/test/java/com/ibm/ws/lars/rest/AssetServiceLayerTest.java index e26e89c..368bfc9 100644 --- a/server/src/test/java/com/ibm/ws/lars/rest/AssetServiceLayerTest.java +++ b/server/src/test/java/com/ibm/ws/lars/rest/AssetServiceLayerTest.java @@ -40,7 +40,7 @@ import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.injection.AssetServiceLayerInjection; import com.ibm.ws.lars.rest.model.Asset; -import com.ibm.ws.lars.rest.model.AssetList; +import com.ibm.ws.lars.rest.model.AssetCursor; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.RepositoryObject; import com.ibm.ws.lars.rest.model.RepositoryResourceLifecycleException; @@ -105,7 +105,7 @@ public void lifeCycleTest() throws Exception { Asset gotAsset = service.retrieveAsset(id, dummyUriInfo); assertEquals("The id of the asset has changed", id, gotAsset.get_id()); - AssetList assets = service.retrieveAllAssets(); + AssetCursor assets = service.retrieveAllAssets(); assertEquals("Too many assets", 1, assets.size()); try { @@ -123,13 +123,13 @@ public void lifeCycleTest() throws Exception { Asset simpleGotAsset = service.retrieveAsset(simpleAsset.get_id(), dummyUriInfo); assertEquals("Wrong state", Asset.State.DRAFT, simpleGotAsset.getState()); - AssetList lotsOfAssets = service.retrieveAllAssets(); + AssetCursor lotsOfAssets = service.retrieveAllAssets(); assertEquals("Wrong number of assets found", 2, lotsOfAssets.size()); service.deleteAsset(id); service.deleteAsset(simpleGotAsset.get_id()); - AssetList emptyAssets = service.retrieveAllAssets(); + AssetCursor emptyAssets = service.retrieveAllAssets(); assertEquals("There should be no assets stored", 0, emptyAssets.size()); } diff --git a/server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java b/server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java new file mode 100644 index 0000000..1f16201 --- /dev/null +++ b/server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2017 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.ibm.ws.lars.rest; + +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; + +import com.ibm.ws.lars.rest.model.Asset; +import com.ibm.ws.lars.rest.model.AssetCursor; +import com.ibm.ws.lars.rest.model.AssetOperation; + +/** + * A super simple AssetCursor backed by a collection + */ +public class BasicAssetCursor implements AssetCursor { + + Iterator> assets; + int count; + + public BasicAssetCursor(Collection> states) { + count = states.size(); + assets = states.iterator(); + } + + @Override + public boolean hasNext() { + return assets.hasNext(); + } + + @Override + public Asset next() { + return new Asset(assets.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public int size() { + return count; + } + + @Override + public void addOperation(AssetOperation filter) { + + } + +} diff --git a/server/src/test/java/com/ibm/ws/lars/rest/MemoryPersistor.java b/server/src/test/java/com/ibm/ws/lars/rest/MemoryPersistor.java index a19687d..c939db5 100644 --- a/server/src/test/java/com/ibm/ws/lars/rest/MemoryPersistor.java +++ b/server/src/test/java/com/ibm/ws/lars/rest/MemoryPersistor.java @@ -32,7 +32,7 @@ import com.ibm.ws.lars.rest.exceptions.InvalidJsonAssetException; import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.model.Asset; -import com.ibm.ws.lars.rest.model.AssetList; +import com.ibm.ws.lars.rest.model.AssetCursor; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.AttachmentContentMetadata; import com.ibm.ws.lars.rest.model.AttachmentContentResponse; @@ -64,13 +64,13 @@ static synchronized String getNextId() { * @see com.ibm.ws.lars.rest.Persistor#retrieveAllAssets() */ @Override - public AssetList retrieveAllAssets() { + public AssetCursor retrieveAllAssets() { // Note: retrieveAllAssets does *not* set attachments - return AssetList.createAssetListFromMaps(new ArrayList>(assets.values())); + return new BasicAssetCursor(assets.values()); } @Override - public AssetList retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions) { + public AssetCursor retrieveAllAssets(Collection filters, String searchTerm, PaginationOptions pagination, SortOptions sortOptions) { throw new UnsupportedOperationException("Filtering is not supported in this test facade"); } @@ -154,8 +154,7 @@ public AttachmentList findAttachmentsForAsset(String assetId) { * java.lang.String, java.io.InputStream) */ @Override - public AttachmentContentMetadata createAttachmentContent(String name, String contentType, InputStream attachmentContentStream) - throws AssetPersistenceException { + public AttachmentContentMetadata createAttachmentContent(String name, String contentType, InputStream attachmentContentStream) throws AssetPersistenceException { try { String id = getNextId(); From 79ca3485b99618dd316890d94a36b77a1e103261 Mon Sep 17 00:00:00 2001 From: Andrew Rouse Date: Sat, 11 Feb 2017 15:27:36 +0000 Subject: [PATCH 5/7] Move MongoDB support classes to their own package --- .../com/ibm/ws/lars/rest/PersistenceBeanTest.java | 1 + .../src/main/java/com/ibm/ws/lars/rest/Persistor.java | 2 +- .../lars/rest/{model => mongo}/MongoAssetCursor.java | 7 +++++-- .../ibm/ws/lars/rest/{ => mongo}/PersistenceBean.java | 11 ++++++++--- .../ws/lars/rest/PersistenceBeanBasicSearchTest.java | 1 + .../ibm/ws/lars/rest/PersistenceBeanLoggingTest.java | 1 + 6 files changed, 17 insertions(+), 6 deletions(-) rename server/src/main/java/com/ibm/ws/lars/rest/{model => mongo}/MongoAssetCursor.java (86%) rename server/src/main/java/com/ibm/ws/lars/rest/{ => mongo}/PersistenceBean.java (98%) diff --git a/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java b/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java index c4db421..b03cd19 100644 --- a/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java +++ b/server/src/fat/java/com/ibm/ws/lars/rest/PersistenceBeanTest.java @@ -50,6 +50,7 @@ import com.ibm.ws.lars.rest.model.AssetCursor; import com.ibm.ws.lars.rest.model.Attachment; import com.ibm.ws.lars.rest.model.AttachmentContentMetadata; +import com.ibm.ws.lars.rest.mongo.PersistenceBean; import com.ibm.ws.lars.testutils.BasicChecks; import com.ibm.ws.lars.testutils.FatUtils; import com.mongodb.DB; diff --git a/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java b/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java index 2adeeaa..9524e5c 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/Persistor.java @@ -31,7 +31,7 @@ import com.ibm.ws.lars.rest.model.AttachmentList; /** - * + * Interface to a persistent data store for assets and attachments */ public interface Persistor { diff --git a/server/src/main/java/com/ibm/ws/lars/rest/model/MongoAssetCursor.java b/server/src/main/java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java similarity index 86% rename from server/src/main/java/com/ibm/ws/lars/rest/model/MongoAssetCursor.java rename to server/src/main/java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java index e21a16c..893369e 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/model/MongoAssetCursor.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java @@ -13,15 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. *******************************************************************************/ -package com.ibm.ws.lars.rest.model; +package com.ibm.ws.lars.rest.mongo; import java.util.ArrayList; import java.util.List; +import com.ibm.ws.lars.rest.model.Asset; +import com.ibm.ws.lars.rest.model.AssetCursor; +import com.ibm.ws.lars.rest.model.AssetOperation; import com.mongodb.DBCursor; /** - * + * An {@link AssetCursor} implementation which streams Assets from a Mongo {@link DBCursor}. */ public class MongoAssetCursor implements AssetCursor { diff --git a/server/src/main/java/com/ibm/ws/lars/rest/PersistenceBean.java b/server/src/main/java/com/ibm/ws/lars/rest/mongo/PersistenceBean.java similarity index 98% rename from server/src/main/java/com/ibm/ws/lars/rest/PersistenceBean.java rename to server/src/main/java/com/ibm/ws/lars/rest/mongo/PersistenceBean.java index 66d9b1b..36da6fe 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/PersistenceBean.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/mongo/PersistenceBean.java @@ -14,7 +14,7 @@ * limitations under the License. *******************************************************************************/ -package com.ibm.ws.lars.rest; +package com.ibm.ws.lars.rest.mongo; import java.io.InputStream; import java.util.ArrayList; @@ -32,6 +32,12 @@ import org.bson.types.ObjectId; +import com.ibm.ws.lars.rest.AssetFilter; +import com.ibm.ws.lars.rest.Condition; +import com.ibm.ws.lars.rest.PaginationOptions; +import com.ibm.ws.lars.rest.Persistor; +import com.ibm.ws.lars.rest.RepositoryRESTResource; +import com.ibm.ws.lars.rest.SortOptions; import com.ibm.ws.lars.rest.SortOptions.SortOrder; import com.ibm.ws.lars.rest.exceptions.InvalidJsonAssetException; import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; @@ -43,7 +49,6 @@ import com.ibm.ws.lars.rest.model.AttachmentContentMetadata; import com.ibm.ws.lars.rest.model.AttachmentContentResponse; import com.ibm.ws.lars.rest.model.AttachmentList; -import com.ibm.ws.lars.rest.model.MongoAssetCursor; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObjectBuilder; @@ -80,7 +85,7 @@ public class PersistenceBean implements Persistor { Arrays.asList(new String[] { "name", "description", "shortDescription", "tags" }); /** The _id field of a MongoDB object */ - public static final String ID = "_id"; + private static final String ID = "_id"; private static final String DB_NAME = "mongo/larsDB"; diff --git a/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanBasicSearchTest.java b/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanBasicSearchTest.java index 6d9723d..7b947db 100644 --- a/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanBasicSearchTest.java +++ b/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanBasicSearchTest.java @@ -33,6 +33,7 @@ import com.ibm.ws.lars.rest.Condition.Operation; import com.ibm.ws.lars.rest.SortOptions.SortOrder; +import com.ibm.ws.lars.rest.mongo.PersistenceBean; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; import com.mongodb.DBCollection; diff --git a/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanLoggingTest.java b/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanLoggingTest.java index 3d12804..a64d7bd 100644 --- a/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanLoggingTest.java +++ b/server/src/test/java/com/ibm/ws/lars/rest/PersistenceBeanLoggingTest.java @@ -28,6 +28,7 @@ import com.ibm.ws.lars.rest.exceptions.NonExistentArtefactException; import com.ibm.ws.lars.rest.model.Asset; import com.ibm.ws.lars.rest.model.Attachment; +import com.ibm.ws.lars.rest.mongo.PersistenceBean; import com.mongodb.BasicDBObject; import com.mongodb.DBCursor; From eee6c5ac0ccd53998c8d7fd05fb69beb849721db Mon Sep 17 00:00:00 2001 From: Andrew Rouse Date: Mon, 13 Feb 2017 14:15:42 +0000 Subject: [PATCH 6/7] Move AssetList into test src folder With the introduction of AssetCursor, AssetList is now only used for tests. --- .../{main => fat}/java/com/ibm/ws/lars/rest/model/AssetList.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/src/{main => fat}/java/com/ibm/ws/lars/rest/model/AssetList.java (100%) diff --git a/server/src/main/java/com/ibm/ws/lars/rest/model/AssetList.java b/server/src/fat/java/com/ibm/ws/lars/rest/model/AssetList.java similarity index 100% rename from server/src/main/java/com/ibm/ws/lars/rest/model/AssetList.java rename to server/src/fat/java/com/ibm/ws/lars/rest/model/AssetList.java From c331e2cde011974a5f9c8cc23b3f1e9bfd174a39 Mon Sep 17 00:00:00 2001 From: Andrew Rouse Date: Mon, 13 Feb 2017 14:30:29 +0000 Subject: [PATCH 7/7] Ensure AssetCursors are closed when written Make AssetCursor implement Closable, and then read the AssetCursor within a try-with-resources so that it gets closed. --- .../main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java | 5 +++-- .../main/java/com/ibm/ws/lars/rest/model/AssetCursor.java | 3 ++- .../java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java | 5 +++++ .../test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java | 7 +++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java b/server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java index 59d7a86..33d292f 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/AssetCursorWriter.java @@ -52,7 +52,8 @@ public boolean isWriteable(Class type, Type genericType, Annotation[] annotat @Override public void writeTo(AssetCursor cursor, Class type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap httpHeaders, OutputStream entityStream) throws IOException { - MAPPER.writeValue(entityStream, cursor); + try (AssetCursor cursorToBeClosed = cursor) { + MAPPER.writeValue(entityStream, cursorToBeClosed); + } } - } diff --git a/server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java b/server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java index 9037580..61beed4 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/model/AssetCursor.java @@ -15,6 +15,7 @@ *******************************************************************************/ package com.ibm.ws.lars.rest.model; +import java.io.Closeable; import java.util.Iterator; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -32,7 +33,7 @@ * returned. */ @JsonSerialize(as = Iterator.class) -public interface AssetCursor extends Iterator { +public interface AssetCursor extends Iterator, Closeable { /** * Get the total number of assets which will be returned from this cursor diff --git a/server/src/main/java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java b/server/src/main/java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java index 893369e..08ccfb3 100644 --- a/server/src/main/java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java +++ b/server/src/main/java/com/ibm/ws/lars/rest/mongo/MongoAssetCursor.java @@ -65,4 +65,9 @@ public void addOperation(AssetOperation op) { operations.add(op); } + @Override + public void close() { + cursor.close(); + } + } diff --git a/server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java b/server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java index 1f16201..519e077 100644 --- a/server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java +++ b/server/src/test/java/com/ibm/ws/lars/rest/BasicAssetCursor.java @@ -15,6 +15,7 @@ *******************************************************************************/ package com.ibm.ws.lars.rest; +import java.io.IOException; import java.util.Collection; import java.util.Iterator; import java.util.Map; @@ -61,4 +62,10 @@ public void addOperation(AssetOperation filter) { } + /** {@inheritDoc} */ + @Override + public void close() throws IOException { + // Do nothing + } + }