From e281f54b4f28a0a169cfdcfa836f885252fff82d Mon Sep 17 00:00:00 2001 From: Sourav Roy Date: Thu, 9 May 2024 21:30:28 +0100 Subject: [PATCH] mongodb bulk insert --- .../db2rest/mongo/rest/MongoController.java | 8 ++ .../db2rest/mongo/rest/api/MongoRestApi.java | 7 ++ .../rest/mongo/MongoDBControllerTest.java | 119 +++++++++++++++--- .../testdata/BULK_CREATE_ACTOR_REQUEST.json | 14 +++ .../mongo/repository/MongoRepository.java | 55 +++++++- 5 files changed, 183 insertions(+), 20 deletions(-) create mode 100644 api-rest/src/test/resources/testdata/BULK_CREATE_ACTOR_REQUEST.json diff --git a/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/MongoController.java b/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/MongoController.java index 71d3b0d6..c2284762 100644 --- a/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/MongoController.java +++ b/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/MongoController.java @@ -3,6 +3,7 @@ import com.homihq.db2rest.config.Db2RestConfigProperties; import com.homihq.db2rest.core.dto.CountResponse; +import com.homihq.db2rest.core.dto.CreateBulkResponse; import com.homihq.db2rest.core.dto.CreateResponse; import com.homihq.db2rest.core.dto.DeleteResponse; import com.homihq.db2rest.core.dto.ExistsResponse; @@ -42,6 +43,13 @@ public CreateResponse save(String collectionName, } + @Override + public CreateBulkResponse saveAll(String collectionName, + List includeFields, + List> dataList) { + return mongoRepository.saveAll(collectionName, includeFields, dataList); + } + @Override public UpdateResponse patch(String collectionName, Map data, String filter) { log.debug("Filter - {}", filter); diff --git a/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/api/MongoRestApi.java b/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/api/MongoRestApi.java index aac85757..fd8cc5ce 100644 --- a/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/api/MongoRestApi.java +++ b/api-rest/src/main/java/com/homihq/db2rest/mongo/rest/api/MongoRestApi.java @@ -1,6 +1,7 @@ package com.homihq.db2rest.mongo.rest.api; import com.homihq.db2rest.core.dto.CountResponse; +import com.homihq.db2rest.core.dto.CreateBulkResponse; import com.homihq.db2rest.core.dto.CreateResponse; import com.homihq.db2rest.core.dto.DeleteResponse; import com.homihq.db2rest.core.dto.ExistsResponse; @@ -25,6 +26,12 @@ CreateResponse save(@PathVariable String collectionName, @RequestParam(name = "fields", required = false) List includeFields, @RequestBody Map data); + @ResponseStatus(HttpStatus.CREATED) + @PostMapping("/{collectionName}/bulk") + CreateBulkResponse saveAll(@PathVariable String collectionName, + @RequestParam(name = "fields", required = false) List includeFields, + @RequestBody List> dataList); + @ResponseStatus(HttpStatus.OK) @PatchMapping("/{collectionName}") UpdateResponse patch(@PathVariable String collectionName, diff --git a/api-rest/src/test/java/com/homihq/db2rest/rest/mongo/MongoDBControllerTest.java b/api-rest/src/test/java/com/homihq/db2rest/rest/mongo/MongoDBControllerTest.java index 99848b12..8b9bc633 100644 --- a/api-rest/src/test/java/com/homihq/db2rest/rest/mongo/MongoDBControllerTest.java +++ b/api-rest/src/test/java/com/homihq/db2rest/rest/mongo/MongoDBControllerTest.java @@ -28,12 +28,16 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; +import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.notNullValue; import static org.springframework.data.mongodb.core.query.Query.query; import static org.springframework.http.MediaType.APPLICATION_JSON; import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.document; @@ -67,6 +71,9 @@ class MongoDBControllerTest extends MongodbContainerConfiguration { @GivenJsonResource("/testdata/CREATE_ACTOR_REQUEST.json") Map CREATE_ACTOR_REQUEST; + @GivenJsonResource("/testdata/BULK_CREATE_ACTOR_REQUEST.json") + List> BULK_CREATE_ACTOR_REQUEST; + @GivenJsonResource("/testdata/UPDATE_ACTOR_REQUEST.json") Map UPDATE_ACTOR_REQUEST; @@ -103,6 +110,86 @@ void create() throws Exception { @Test @Order(2) + @DisplayName("Create an actor with with include fields") + void createWithIncludeFields() throws Exception { + mockMvc.perform(post("/Sakila_actors") + .contentType(APPLICATION_JSON) + .accept(APPLICATION_JSON) + .param("fields", "FirstName, LastName") + .content(objectMapper.writeValueAsString(CREATE_ACTOR_REQUEST)) + ) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.row", equalTo(1))) + .andExpect(jsonPath("$.keys.timestamp").exists()) + .andDo(document("mongodb-create-an-actor-with-include-fields")); + + mongoTemplate.remove(query(Criteria + .where("FirstName") + .is("KEVIN")), + "Sakila_actors"); + } + + @Test + @Order(3) + @DisplayName("Create many actors") + void createBulk() throws Exception { + mockMvc.perform(post("/Sakila_actors/bulk") + .contentType(APPLICATION_JSON) + .accept(APPLICATION_JSON) + .content(objectMapper.writeValueAsString(BULK_CREATE_ACTOR_REQUEST)) + ) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.rows").isArray()) + .andExpect(jsonPath("$.rows", hasSize(2))) + .andExpect(jsonPath("$.rows", hasItem(1))) + + .andExpect(jsonPath("$.keys").isArray()) + .andExpect(jsonPath("$.keys", hasSize(2))) + .andExpect(jsonPath("$.keys", allOf(notNullValue()))) + .andDo(document("mongodb-bulk-create-actors")); + + mongoTemplate.remove(query(Criteria + .where("FirstName") + .is("VIVIEN")), + "Sakila_actors"); + mongoTemplate.remove(query(Criteria + .where("FirstName") + .is("CUBA")), + "Sakila_actors"); + } + + @Test + @Order(4) + @DisplayName("Create many actors with include fields") + void createBulkWithIncludeFields() throws Exception { + mockMvc.perform(post("/Sakila_actors/bulk") + .contentType(APPLICATION_JSON) + .accept(APPLICATION_JSON) + .param("fields", "FirstName, LastName") + .content(objectMapper.writeValueAsString(BULK_CREATE_ACTOR_REQUEST)) + ) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.rows").isArray()) + .andExpect(jsonPath("$.rows", hasSize(2))) + .andExpect(jsonPath("$.rows", hasItem(1))) + + .andExpect(jsonPath("$.keys").isArray()) + .andExpect(jsonPath("$.keys", hasSize(2))) + .andExpect(jsonPath("$.keys", allOf(notNullValue()))) + .andDo(document("mongodb-bulk-create-actors-with-include-fields")); + + mongoTemplate.remove(query(Criteria + .where("FirstName") + .is("VIVIEN")), + "Sakila_actors"); + mongoTemplate.remove(query(Criteria + .where("FirstName") + .is("CUBA")), + "Sakila_actors"); + } + + @Test + @Order(5) @DisplayName("Update an existing Actor") void updateExistingActor() throws Exception { mockMvc.perform(patch("/Sakila_actors") @@ -117,7 +204,7 @@ void updateExistingActor() throws Exception { } @Test - @Order(3) + @Order(6) @DisplayName("Update a non-existing Actor") void updateNonExistingActor() throws Exception { mockMvc.perform(patch("/Sakila_actors") @@ -132,7 +219,7 @@ void updateNonExistingActor() throws Exception { } @Test - @Order(4) + @Order(7) @DisplayName("Update non-existing Collection") void updateNonExistingCollection() throws Exception { mockMvc.perform(patch("/unknown_collection") @@ -147,7 +234,7 @@ void updateNonExistingCollection() throws Exception { } @Test - @Order(5) + @Order(8) @DisplayName("Update multiple Actors") void updateExistingActors() throws Exception { mockMvc.perform(patch("/Sakila_actors") @@ -162,7 +249,7 @@ void updateExistingActors() throws Exception { } @Test - @Order(6) + @Order(9) @DisplayName("Test find all actors - all fields") void findAllActors() throws Exception { mockMvc.perform(get("/Sakila_actors") @@ -175,7 +262,7 @@ void findAllActors() throws Exception { } @Test - @Order(7) + @Order(10) @DisplayName("Test find all actors - 2 fields") void findAllActorsWithTwoFields() throws Exception { mockMvc.perform(get("/Sakila_actors") @@ -192,7 +279,7 @@ void findAllActorsWithTwoFields() throws Exception { } @Test - @Order(8) + @Order(11) @DisplayName("Test find all actors with filter") void findAllActorsWithFilter() throws Exception { mockMvc.perform(get("/Sakila_actors") @@ -209,7 +296,7 @@ void findAllActorsWithFilter() throws Exception { } @Test - @Order(9) + @Order(12) @DisplayName("Test find all actors with sorting") void findAllActorsWithSorting() throws Exception { mockMvc.perform(get("/Sakila_actors") @@ -227,7 +314,7 @@ void findAllActorsWithSorting() throws Exception { } @Test - @Order(10) + @Order(13) @DisplayName("Test find all actors with pagination") void findAllActorsWithPagination() throws Exception { mockMvc.perform(get("/Sakila_actors") @@ -247,7 +334,7 @@ void findAllActorsWithPagination() throws Exception { } @Test - @Order(11) + @Order(14) @DisplayName("Find one actor - 2 fields") void findOneActor() throws Exception { mockMvc.perform(get("/Sakila_actors/one") @@ -263,7 +350,7 @@ void findOneActor() throws Exception { } @Test - @Order(12) + @Order(15) @DisplayName("Total number of Actors") void countAll() throws Exception { mockMvc.perform(get("/Sakila_actors/count") @@ -275,7 +362,7 @@ void countAll() throws Exception { } @Test - @Order(13) + @Order(16) @DisplayName("Number of Actors by LastName") void countActorsByLastName() throws Exception { mockMvc.perform(get("/Sakila_actors/count") @@ -288,7 +375,7 @@ void countActorsByLastName() throws Exception { } @Test - @Order(14) + @Order(17) @DisplayName("Actor exists By FirstName") void existsByLastName() throws Exception { mockMvc.perform(get("/Sakila_actors/exists") @@ -301,7 +388,7 @@ void existsByLastName() throws Exception { } @Test - @Order(15) + @Order(18) @DisplayName("Actor exists By unknown name") void existsByUnknownName() throws Exception { mockMvc.perform(get("/Sakila_actors/exists") @@ -314,7 +401,7 @@ void existsByUnknownName() throws Exception { } @Test - @Order(16) + @Order(19) @DisplayName("Actor exists throws exception when 'filter' is not present") void existsThrowsException() throws Exception { mockMvc.perform(get("/Sakila_actors/exists") @@ -327,7 +414,7 @@ void existsThrowsException() throws Exception { } @Test - @Order(17) + @Order(20) @DisplayName("Delete all documents while allowSafeDelete=true") void delete_all_documents_with_allow_safe_delete_true() throws Exception { mockMvc.perform(delete("/Sakila_actors") @@ -339,7 +426,7 @@ void delete_all_documents_with_allow_safe_delete_true() throws Exception { } @Test - @Order(18) + @Order(21) @DisplayName("Delete an actor") void delete_single_record() throws Exception { mockMvc.perform(delete("/Sakila_actors") diff --git a/api-rest/src/test/resources/testdata/BULK_CREATE_ACTOR_REQUEST.json b/api-rest/src/test/resources/testdata/BULK_CREATE_ACTOR_REQUEST.json new file mode 100644 index 00000000..81a1673e --- /dev/null +++ b/api-rest/src/test/resources/testdata/BULK_CREATE_ACTOR_REQUEST.json @@ -0,0 +1,14 @@ +[ + { + "FirstName" : "VIVIEN", + "LastName" : "BERGEN", + "phone" : "XXX-XXXX-XXX", + "address" : "XXXXXXXXXXXXXXXXXXXXXXXXXXXX" + }, + { + "FirstName" : "CUBA", + "LastName" : "OLIVIER", + "phone" : "XXX-XXXX-XXX", + "address" : "XXXXXXXXXXXXXXXXXXXXXXXXXXXX" + } +] \ No newline at end of file diff --git a/mongo-support/src/main/java/com/homihq/db2test/mongo/repository/MongoRepository.java b/mongo-support/src/main/java/com/homihq/db2test/mongo/repository/MongoRepository.java index 5cec1a24..64fb6035 100644 --- a/mongo-support/src/main/java/com/homihq/db2test/mongo/repository/MongoRepository.java +++ b/mongo-support/src/main/java/com/homihq/db2test/mongo/repository/MongoRepository.java @@ -1,6 +1,7 @@ package com.homihq.db2test.mongo.repository; import com.homihq.db2rest.core.dto.CountResponse; +import com.homihq.db2rest.core.dto.CreateBulkResponse; import com.homihq.db2rest.core.dto.CreateResponse; import com.homihq.db2rest.core.dto.DeleteResponse; import com.homihq.db2rest.core.dto.ExistsResponse; @@ -10,13 +11,21 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.bson.Document; +import org.bson.types.ObjectId; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; + +import static java.util.stream.Collectors.toUnmodifiableMap; +import static org.springframework.util.CollectionUtils.isEmpty; @Slf4j @RequiredArgsConstructor @@ -24,15 +33,53 @@ public class MongoRepository { private final MongoTemplate mongoTemplate; - public CreateResponse save(String collectionName, List includedFields, Map data) { + Map insertableData; + if (!isEmpty(includedFields)) { + Set subsetKeys = new HashSet<>(includedFields); + insertableData = filterMapBySubsetKeys(data, subsetKeys); + } else { + insertableData = data; + } + var insertableDocument = convertToDocument(insertableData); + var savedDocument = mongoTemplate.save(insertableDocument, collectionName); + return new CreateResponse(1, savedDocument.getObjectId("_id")); + } + + public CreateBulkResponse saveAll(String collectionName, List includedFields, + List> dataList) { + List> insertableDataList; + if (!isEmpty(includedFields)) { + Set subsetKeys = new HashSet<>(includedFields); + insertableDataList = dataList.stream() + .map(data -> filterMapBySubsetKeys(data, subsetKeys)) + .toList(); + } else { + insertableDataList = dataList; + } + List insertableDocumentList = insertableDataList.stream() + .map(this::convertToDocument) + .toList(); + Collection savedDocuments = mongoTemplate.insert(insertableDocumentList, collectionName); + List rows = Collections.nCopies(savedDocuments.size(), 1); + List keys = savedDocuments.stream().map(doc -> doc.getObjectId("_id")).toList(); + return new CreateBulkResponse(rows.stream() + .mapToInt(Integer::intValue) + .toArray(), keys); + } + + private Map filterMapBySubsetKeys(Map source, Set subsetKeys) { + return source.entrySet().stream() + .filter(entry -> subsetKeys.contains(entry.getKey())) + .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private Document convertToDocument(Map data) { var document = new Document(); data.forEach(document::append); - var savedDocument = mongoTemplate.save(document, collectionName); - return new CreateResponse(1, savedDocument.getObjectId("_id")); - + return document; } public UpdateResponse patch(Query query, String collectionName, Map data) {