From c7b348007a34a684ce63bae2de8db9395e8c2b59 Mon Sep 17 00:00:00 2001 From: Nikesh kumar Date: Mon, 12 Feb 2024 12:36:12 +0530 Subject: [PATCH] feat(rest): create new enpoint to upload component csv file. Signed-off-by: Nikesh kumar --- .../src/docs/asciidoc/api-guide.adoc | 1 + .../src/docs/asciidoc/importExport.adoc | 70 ++++++++ .../importexport/ImportExportController.java | 116 ++++++++++++++ .../Sw360ImportExportService.java | 150 ++++++++++++++++++ .../resourceserver/restdocs/ApiSpecTest.java | 1 + .../restdocs/ImportExportSpec.java | 112 +++++++++++++ 6 files changed, 450 insertions(+) create mode 100644 rest/resource-server/src/docs/asciidoc/importExport.adoc create mode 100644 rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/ImportExportController.java create mode 100644 rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/Sw360ImportExportService.java create mode 100644 rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ImportExportSpec.java diff --git a/rest/resource-server/src/docs/asciidoc/api-guide.adoc b/rest/resource-server/src/docs/asciidoc/api-guide.adoc index 31feb10efe..25313e3959 100644 --- a/rest/resource-server/src/docs/asciidoc/api-guide.adoc +++ b/rest/resource-server/src/docs/asciidoc/api-guide.adoc @@ -307,3 +307,4 @@ include::obligations.adoc[] include::moderationRequests.adoc[] include::fossology.adoc[] include::schedule.adoc[] +include::importExport.adoc[] diff --git a/rest/resource-server/src/docs/asciidoc/importExport.adoc b/rest/resource-server/src/docs/asciidoc/importExport.adoc new file mode 100644 index 0000000000..3132c2f7fe --- /dev/null +++ b/rest/resource-server/src/docs/asciidoc/importExport.adoc @@ -0,0 +1,70 @@ +// Copyright Siemens AG, 2024. Part of the SW360 Portal Project. +// +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// + +[[resources-importExport]] +=== ImportExport + +The ImportExport resource is used to get and upload request. + +[[upload-component]] +==== Upload component csv file. + +A `POST` request help to upload the component csv file. + +[red]#Request parameter# +|=== +|Parameter |Description + +|componentFile +|Upload the component csv file. +|=== + +===== Example request +include::{snippets}/should_document_upload_component_file/curl-request.adoc[] + +===== Example response +include::{snippets}/should_document_upload_component_file/http-response.adoc[] + +[[upload-release]] +==== Upload release link csv file. + +A `POST` request help to upload the release csv file. + +[red]#Request parameter# +|=== +|Parameter |Description + +|releaseFile +|Upload the release link csv file. +|=== + +===== Example request +include::{snippets}/should_document_upload_release_link_file/curl-request.adoc[] + +===== Example response +include::{snippets}/should_document_upload_release_link_file/http-response.adoc[] + +[[upload-component-attachment]] +==== Upload component attachment file. + +A `POST` request help to upload the component attachment file. + +[red]#Request parameter# +|=== +|Parameter |Description + +|component +|Upload the component attachment file. +|=== + +===== Example request +include::{snippets}/should_document_upload_component_attachment_file/curl-request.adoc[] + +===== Example response +include::{snippets}/should_document_upload_component_attachment_file/http-response.adoc[] \ No newline at end of file diff --git a/rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/ImportExportController.java b/rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/ImportExportController.java new file mode 100644 index 0000000000..e312291fb5 --- /dev/null +++ b/rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/ImportExportController.java @@ -0,0 +1,116 @@ +/* + * Copyright Siemens AG, 2024-2025. + * Part of the SW360 Portal Project. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.sw360.rest.resourceserver.importexport; + +import static org.springframework.hateoas.server.mvc.WebMvcLinkBuilder.linkTo; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.thrift.TException; +import org.eclipse.sw360.datahandler.common.CommonUtils; +import org.eclipse.sw360.datahandler.thrift.RequestStatus; +import org.eclipse.sw360.datahandler.thrift.RequestSummary; +import org.eclipse.sw360.datahandler.thrift.users.User; +import org.eclipse.sw360.rest.resourceserver.core.RestControllerHelper; +import org.eclipse.sw360.rest.resourceserver.project.ProjectController; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.rest.webmvc.BasePathAwareController; +import org.springframework.data.rest.webmvc.RepositoryLinksResource; +import org.springframework.hateoas.server.RepresentationModelProcessor; +import org.springframework.http.HttpStatus.Series; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.HttpMediaTypeNotAcceptableException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@BasePathAwareController +@RequiredArgsConstructor(onConstructor = @__(@Autowired)) +@RestController +@SecurityRequirement(name = "tokenAuth") +public class ImportExportController implements RepresentationModelProcessor { + public static final String IMPORTEXPORT_URL = "/importExport"; + + private static final MediaType form = null; + + @NonNull + private final RestControllerHelper restControllerHelper; + + @NonNull + private final Sw360ImportExportService importExportService; + + @Override + public RepositoryLinksResource process(RepositoryLinksResource resource) { + resource.add(linkTo(ImportExportController.class).slash("api/importExport").withRel("importExport")); + return resource; + } + + @Operation( + summary = "Upload component csv file.", + description = "Upload component csv file.", + tags = {"Component"} + ) + @RequestMapping(value = IMPORTEXPORT_URL + "/uploadComponent", method = RequestMethod.POST, consumes = {MediaType.MULTIPART_MIXED_VALUE, MediaType.MULTIPART_FORM_DATA_VALUE}, produces = { MediaType.APPLICATION_JSON_VALUE }) + public ResponseEntity uploadComponentCsv( + @Parameter(description = "The component csv file to be uploaded.") + @RequestParam("componentFile") MultipartFile file, HttpServletRequest request, HttpServletResponse response) throws TException,ServletException,IOException{ + + User sw360User = restControllerHelper.getSw360UserFromAuthentication(); + RequestSummary requestSummary =importExportService.uploadComponent(sw360User, file, request, response); + return ResponseEntity.ok(requestSummary); + } + + @Operation( + summary = "release link file.", + description = "release link file.", + tags = {"Release"} + ) + @RequestMapping(value = IMPORTEXPORT_URL + "/uploadRelease", method = RequestMethod.POST, consumes = {MediaType.MULTIPART_MIXED_VALUE, MediaType.MULTIPART_FORM_DATA_VALUE}, produces = { MediaType.APPLICATION_JSON_VALUE }) + public ResponseEntity uploadReleaseCsv( + @Parameter(description = "The release csv file to be uploaded.") + @RequestParam("releaseFile") MultipartFile file, HttpServletRequest request, HttpServletResponse response) throws TException,ServletException,IOException{ + + User sw360User = restControllerHelper.getSw360UserFromAuthentication(); + RequestSummary requestSummary =importExportService.uploadReleaseLink(sw360User, file, request); + return ResponseEntity.ok(requestSummary); + } + + @Operation( + summary = "component attachment file.", + description = "component attachment file.", + tags = {"Component"} + ) + @RequestMapping(value = IMPORTEXPORT_URL + "/componentAttachment", method = RequestMethod.POST, consumes = {MediaType.MULTIPART_MIXED_VALUE, MediaType.MULTIPART_FORM_DATA_VALUE}, produces = { MediaType.APPLICATION_JSON_VALUE }) + public ResponseEntity uploadComponentAttachment( + @Parameter(description = "The component attachment csv file to be uploaded.") + @RequestParam("attachmentFile") MultipartFile file, HttpServletRequest request, HttpServletResponse response) throws TException,ServletException,IOException{ + + User sw360User = restControllerHelper.getSw360UserFromAuthentication(); + RequestSummary requestSummary =importExportService.uploadComponentAttachment(sw360User, file, request); + return ResponseEntity.ok(requestSummary); + } +} diff --git a/rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/Sw360ImportExportService.java b/rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/Sw360ImportExportService.java new file mode 100644 index 0000000000..2d54e4c36c --- /dev/null +++ b/rest/resource-server/src/main/java/org/eclipse/sw360/rest/resourceserver/importexport/Sw360ImportExportService.java @@ -0,0 +1,150 @@ +/* + * Copyright Siemens AG, 2024-2025. + * Part of the SW360 Portal Project. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + +package org.eclipse.sw360.rest.resourceserver.importexport; + +import static org.eclipse.sw360.datahandler.common.ImportCSV.readAsCSVRecords; +import static org.eclipse.sw360.importer.ComponentImportUtils.convertCSVRecordsToCompCSVRecords; +import static org.eclipse.sw360.importer.ComponentImportUtils.convertCSVRecordsToComponentAttachmentCSVRecords; +import static org.eclipse.sw360.importer.ComponentImportUtils.convertCSVRecordsToReleaseLinkCSVRecords; +import static org.eclipse.sw360.importer.ComponentImportUtils.writeAttachmentsToDatabase; +import static org.eclipse.sw360.importer.ComponentImportUtils.writeReleaseLinksToDatabase; +import static org.eclipse.sw360.importer.ComponentImportUtils.writeToDatabase; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; +import javax.servlet.http.Part; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.THttpClient; +import org.apache.thrift.transport.TTransportException; +import org.apache.tomcat.util.http.fileupload.FileItem; +import org.apache.tomcat.util.http.fileupload.RequestContext; +import org.apache.tomcat.util.http.fileupload.disk.DiskFileItemFactory; +import org.apache.tomcat.util.http.fileupload.servlet.ServletFileUpload; +import org.eclipse.sw360.datahandler.permissions.PermissionUtils; +import org.eclipse.sw360.datahandler.thrift.RequestSummary; +import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentService; +import org.eclipse.sw360.datahandler.thrift.components.ComponentService; +import org.eclipse.sw360.datahandler.thrift.users.User; +import org.eclipse.sw360.datahandler.thrift.users.UserGroup; +import org.eclipse.sw360.datahandler.thrift.vendors.VendorService; +import org.eclipse.sw360.importer.ComponentAttachmentCSVRecord; +import org.eclipse.sw360.importer.ComponentCSVRecord; +import org.eclipse.sw360.importer.ReleaseLinkCSVRecord; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.google.common.base.Joiner; +import com.google.common.collect.FluentIterable; + +import lombok.RequiredArgsConstructor; + +@SuppressWarnings("serial") +@Service +@RequiredArgsConstructor(onConstructor = @__(@Autowired)) +public class Sw360ImportExportService { + @Value("${sw360.thrift-server-url:http://localhost:8080}") + private String thriftServerUrl; + + @JsonInclude + public RequestSummary uploadComponent(User sw360User, MultipartFile file, HttpServletRequest request, HttpServletResponse response) throws IOException, TException, ServletException { + if (!PermissionUtils.isUserAtLeast(UserGroup.ADMIN, sw360User)) { + throw new RuntimeException("Unable to upload component csv file. User is not admin"); + } + List releaseRecords = getCSVFromRequest(request, "file"); + FluentIterable compCSVRecords = convertCSVRecordsToCompCSVRecords(releaseRecords); + ComponentService.Iface sw360ComponentClient = getThriftComponentClient(); + VendorService.Iface sw360VendorClient = getThriftVendorClient(); + AttachmentService.Iface sw360AttachmentClient = getThriftAttachmentClient(); + RequestSummary requestSummary = writeToDatabase(compCSVRecords, sw360ComponentClient, sw360VendorClient, sw360AttachmentClient, sw360User); + return requestSummary; + } + + private List getCSVFromRequest(HttpServletRequest request, String fileUploadFormId) throws IOException, TException, ServletException { + final InputStream stream = getInputStreamFromRequest(request, fileUploadFormId); + return readAsCSVRecords(stream); + } + + private InputStream getInputStreamFromRequest(HttpServletRequest request, String fileUploadFormId) throws IOException, ServletException { + Collection parts = request.getParts(); + + for (Part part : parts) { + if (!part.getName().equals(fileUploadFormId)) { + return part.getInputStream(); + } + } + + throw new IOException("File not found in the request with the specified field name."); + } + + private ComponentService.Iface getThriftComponentClient() throws TTransportException { + THttpClient thriftClient = new THttpClient(thriftServerUrl + "/components/thrift"); + TProtocol protocol = new TCompactProtocol(thriftClient); + return new ComponentService.Client(protocol); + } + + private VendorService.Iface getThriftVendorClient() throws TTransportException { + THttpClient thriftClient = new THttpClient(thriftServerUrl + "/vendors/thrift"); + TProtocol protocol = new TCompactProtocol(thriftClient); + return new VendorService.Client(protocol); + } + + private AttachmentService.Iface getThriftAttachmentClient() throws TTransportException { + THttpClient thriftClient = new THttpClient(thriftServerUrl + "/attachments/thrift"); + TProtocol protocol = new TCompactProtocol(thriftClient); + return new AttachmentService.Client(protocol); + } + + public RequestSummary uploadReleaseLink(User sw360User, MultipartFile file, HttpServletRequest request) throws IOException, TException, ServletException{ + if (!PermissionUtils.isUserAtLeast(UserGroup.ADMIN, sw360User)) { + throw new RuntimeException("Unable to upload component csv file. User is not admin"); + } + List releaseLinkRecords = getCSVFromRequest(request, "file"); + FluentIterable csvRecords = convertCSVRecordsToReleaseLinkCSVRecords(releaseLinkRecords); + ComponentService.Iface sw360ComponentClient = getThriftComponentClient(); + final RequestSummary requestSummary = writeReleaseLinksToDatabase(csvRecords, sw360ComponentClient, sw360User); + return requestSummary; + } + + public RequestSummary uploadComponentAttachment(User sw360User, MultipartFile file, HttpServletRequest request) throws IOException, TException, ServletException{ + if (!PermissionUtils.isUserAtLeast(UserGroup.ADMIN, sw360User)) { + throw new RuntimeException("Unable to upload component attachment csv file. User is not admin"); + } + List attachmentRecords = getCSVFromRequest(request, "file"); + FluentIterable compCSVRecords = convertCSVRecordsToComponentAttachmentCSVRecords(attachmentRecords); + ComponentService.Iface sw360ComponentClient = getThriftComponentClient(); + AttachmentService.Iface sw360AttachmentClient = getThriftAttachmentClient(); + final RequestSummary requestSummary = writeAttachmentsToDatabase(compCSVRecords, sw360User, sw360ComponentClient, sw360AttachmentClient); + return requestSummary; + } +} diff --git a/rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ApiSpecTest.java b/rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ApiSpecTest.java index ab953728f2..4e5c6b19ab 100644 --- a/rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ApiSpecTest.java +++ b/rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ApiSpecTest.java @@ -184,6 +184,7 @@ public void should_document_index() throws Exception { linkWithRel("sw360:moderationRequests").description("The <>"), linkWithRel("sw360:fossology").description("The <>"), linkWithRel("sw360:schedule").description("The <>"), + linkWithRel("sw360:importExport").description("The <>"), linkWithRel("curies").description("The Curies for documentation"), linkWithRel("profile").description("The profiles of the REST resources") ), diff --git a/rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ImportExportSpec.java b/rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ImportExportSpec.java new file mode 100644 index 0000000000..539aa74095 --- /dev/null +++ b/rest/resource-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ImportExportSpec.java @@ -0,0 +1,112 @@ +/* + * Copyright Siemens AG, 2024-2025. + * Part of the SW360 Portal Project. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ + + +package org.eclipse.sw360.rest.resourceserver.restdocs; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.IOException; + +import javax.servlet.ServletException; + +import org.apache.thrift.TException; +import org.eclipse.sw360.datahandler.thrift.RequestStatus; +import org.eclipse.sw360.datahandler.thrift.RequestSummary; +import org.eclipse.sw360.datahandler.thrift.licenses.LicenseType; +import org.eclipse.sw360.datahandler.thrift.users.User; +import org.eclipse.sw360.rest.resourceserver.TestHelper; +import org.eclipse.sw360.rest.resourceserver.importexport.Sw360ImportExportService; +import org.eclipse.sw360.rest.resourceserver.user.Sw360UserService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +@RunWith(SpringJUnit4ClassRunner.class) +public class ImportExportSpec extends TestRestDocsSpecBase { + + @Value("${sw360.test-user-id}") + private String testUserId; + + @Value("${sw360.test-user-password}") + private String testUserPassword; + + @MockBean + private Sw360UserService userServiceMock; + + @MockBean + private Sw360ImportExportService importExportService; + + private RequestSummary requestSummary = new RequestSummary(); + + @Before + @SuppressWarnings("unchecked") + public void before() throws TException, IOException, ServletException { + User sw360User = new User(); + sw360User.setId("123456789"); + sw360User.setEmail("admin@sw360.org"); + sw360User.setFullname("John Doe"); + + requestSummary.setRequestStatus(RequestStatus.SUCCESS); + LicenseType licensetype = new LicenseType(); + licensetype.setId("1234"); + licensetype.setLicenseType("wer"); + licensetype.setLicenseTypeId(123); + licensetype.setType("xyz"); + + given(this.importExportService.uploadComponent(any(), any(), any(),any())).willReturn(requestSummary); + given(this.userServiceMock.getUserByEmailOrExternalId("admin@sw360.org")).willReturn(sw360User); + given(this.importExportService.uploadReleaseLink(any(), any(), any())).willReturn(requestSummary); + given(this.importExportService.uploadComponentAttachment(any(), any(), any())).willReturn(requestSummary); + + } + + @Test + public void should_document_upload_component_file() throws Exception { + MockMultipartFile file = new MockMultipartFile("componentFile","file=@/bom.spdx.rdf".getBytes()); + String accessToken = TestHelper.getAccessToken(mockMvc, testUserId, testUserPassword); + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart("/api/importExport/uploadComponent") + .file(file) + .header("Authorization", "Bearer " + accessToken) + .queryParam("componentFile", "Must need to attach file."); + this.mockMvc.perform(builder).andExpect(status().isOk()).andDo(this.documentationHandler.document()); + } + + @Test + public void should_document_upload_release_link_file() throws Exception { + MockMultipartFile file = new MockMultipartFile("releaseFile","file=@/bom.spdx.rdf".getBytes()); + String accessToken = TestHelper.getAccessToken(mockMvc, testUserId, testUserPassword); + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart("/api/importExport/uploadRelease") + .file(file) + .header("Authorization", "Bearer " + accessToken) + .queryParam("releaseFile", "Must need to attach file."); + this.mockMvc.perform(builder).andExpect(status().isOk()).andDo(this.documentationHandler.document()); + } + + @Test + public void should_document_upload_component_attachment_file() throws Exception { + MockMultipartFile file = new MockMultipartFile("attachmentFile","file=@/bom.spdx.rdf".getBytes()); + String accessToken = TestHelper.getAccessToken(mockMvc, testUserId, testUserPassword); + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart("/api/importExport/componentAttachment") + .file(file) + .header("Authorization", "Bearer " + accessToken) + .queryParam("attachmentFile", "Must need to attach file."); + this.mockMvc.perform(builder).andExpect(status().isOk()).andDo(this.documentationHandler.document()); + } +}