S3 settings functional tests

pom.xml

@@ -75,6 +75,7 @@
         <groovy.version>3.0.14</groovy.version>
         <testcontainers.version>1.17.4</testcontainers.version>
         <mockserver.version>5.14.0</mockserver.version>
+        <amazonaws.version>1.12.600</amazonaws.version>
         <allure.version>2.19.0</allure.version>
         <aspectj.version>1.9.9.1</aspectj.version>
         <bytebuddy.version>1.12.14</bytebuddy.version>
@@ -589,6 +590,18 @@
             <version>${testcontainers.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>${amazonaws.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>localstack</artifactId>
+            <version>${testcontainers.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.mock-server</groupId>
             <artifactId>mockserver-client-java</artifactId>

src/test/groovy/org/prebid/server/functional/service/S3Service.groovy

@@ -0,0 +1,86 @@
+package org.prebid.server.functional.service
+
+import org.prebid.server.functional.model.config.AccountConfig
+import org.prebid.server.functional.model.db.StoredImp
+import org.prebid.server.functional.model.db.StoredRequest
+import org.prebid.server.functional.model.db.StoredResponse
+import org.prebid.server.functional.testcontainers.Dependencies
+import org.prebid.server.functional.util.ObjectMapperWrapper
+import org.testcontainers.containers.localstack.LocalStackContainer
+import org.testcontainers.utility.DockerImageName
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.s3.S3Client
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest
+import software.amazon.awssdk.services.s3.model.PutObjectRequest
+import software.amazon.awssdk.services.s3.model.PutObjectResponse
+import software.amazon.awssdk.core.sync.RequestBody
+
+final class S3Service implements ObjectMapperWrapper {
+
+    private final S3Client s3PbsService
+    private final LocalStackContainer localStackContainer
+
+    static final def DEFAULT_ACCOUNT_DIR = 'account'
+    static final def DEFAULT_IMPS_DIR = 'stored-impressions'
+    static final def DEFAULT_REQUEST_DIR = 'stored-requests'
+    static final def DEFAULT_RESPONSE_DIR = 'stored-responses'
+
+    S3Service(String bucketName) {
+        this.localStackContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:s3-latest"))
+                .withNetwork(Dependencies.network)
+                .withServices(LocalStackContainer.Service.S3)
+        localStackContainer.start()
+        s3PbsService = S3Client.builder()
+                .endpointOverride(localStackContainer.getEndpointOverride(LocalStackContainer.Service.S3))
+                .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(
+                        localStackContainer.getAccessKey(), localStackContainer.getSecretKey())))
+                .region(Region.of(localStackContainer.getRegion()))
+                .build()
+        createBucket(bucketName)
+    }
+
+    String getAccessKeyId() {
+        localStackContainer.accessKey
+    }
+
+    String getSecretKeyId() {
+        localStackContainer.secretKey
+    }
+
+    String getEndpoint() {
+        "http://${localStackContainer.getNetworkAliases().get(0)}:${localStackContainer.getExposedPorts().get(0)}"
+    }
+
+    void createBucket(String bucketName) {
+        CreateBucketRequest createBucketRequest = CreateBucketRequest.builder()
+                .bucket(bucketName)
+                .build()
+        s3PbsService.createBucket(createBucketRequest)
+    }
+
+    PutObjectResponse uploadAccount(String bucketName, AccountConfig account, String fileName = account.id) {
+        uploadFile(bucketName, encode(account), "${DEFAULT_ACCOUNT_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadStoredRequest(String bucketName, StoredRequest storedRequest, String fileName = storedRequest.requestId) {
+        uploadFile(bucketName, encode(storedRequest.requestData), "${DEFAULT_REQUEST_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadStoredResponse(String bucketName, StoredResponse storedRequest, String fileName = storedRequest.responseId) {
+        uploadFile(bucketName, encode(storedRequest.storedAuctionResponse), "${DEFAULT_RESPONSE_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadStoredImp(String bucketName, StoredImp storedImp, String fileName = storedImp.impId) {
+        uploadFile(bucketName, encode(storedImp.impData), "${DEFAULT_IMPS_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadFile(String bucketName, String fileBody, String path) {
+        PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+                .bucket(bucketName)
+                .key(path)
+                .build()
+        s3PbsService.putObject(putObjectRequest, RequestBody.fromString(fileBody))
+    }
+}

src/test/groovy/org/prebid/server/functional/tests/storage/AccountS3Spec.groovy

@@ -0,0 +1,118 @@
+package org.prebid.server.functional.tests.storage
+
+import org.prebid.server.functional.model.AccountStatus
+import org.prebid.server.functional.model.config.AccountConfig
+import org.prebid.server.functional.model.request.auction.BidRequest
+import org.prebid.server.functional.service.PrebidServerException
+import org.prebid.server.functional.service.PrebidServerService
+import org.prebid.server.functional.service.S3Service
+import org.prebid.server.functional.testcontainers.PbsServiceFactory
+import org.prebid.server.functional.util.PBSUtils
+
+import static io.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED
+
+class AccountS3Spec extends StorageBaseSpec {
+
+    protected PrebidServerService s3StorageAccountPbsService = PbsServiceFactory.getService(s3StorageConfig +
+                                                                                            mySqlDisabledConfig +
+                                                                                            ['settings.enforce-valid-account': 'true'])
+
+    def "PBS should process request when active account is present in S3 storage"() {
+        given: "Default BidRequest with account"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Active account config"
+        def account = new AccountConfig(id: accountId, status: AccountStatus.ACTIVE)
+
+        and: "Saved account in AWS S3 storage"
+        S3_SERVICE.uploadAccount(DEFAULT_BUCKET, account)
+
+        when: "PBS processes auction request"
+        def response = s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "Response should contain seatbid"
+        assert response.seatbid.size() == 1
+    }
+
+    def "PBS should throw exception when inactive account is present in S3 storage"() {
+        given: "Default BidRequest with account"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Inactive account config"
+        def account = new AccountConfig(id: accountId, status: AccountStatus.INACTIVE)
+
+        and: "Saved account in AWS S3 storage"
+        S3_SERVICE.uploadAccount(DEFAULT_BUCKET, account)
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Account $accountId is inactive"
+    }
+
+    def "PBS should throw exception when account have different id inside json in S3 storage"() {
+        given: "Default BidRequest with account"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Account config with different accountId"
+        def account = new AccountConfig(id: PBSUtils.randomString, status: AccountStatus.ACTIVE)
+
+        and: "Saved account in AWS S3 storage"
+        S3_SERVICE.uploadAccount(DEFAULT_BUCKET, account, accountId)
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Unauthorized account id: ${accountId}"
+    }
+
+    def "PBS should throw exception when account is invalid in S3 storage json file"() {
+        given: "Default BidRequest"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Saved invalid account in AWS S3 storage"
+        S3_SERVICE.uploadFile(DEFAULT_BUCKET, INVALID_FILE_BODY, "${S3Service.DEFAULT_ACCOUNT_DIR}/${accountId}.json")
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Unauthorized account id: ${accountId}"
+    }
+
+    def "PBS should throw exception when account is not present in S3 storage and valid account enforced"() {
+        given: "Default BidRequest"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Unauthorized account id: ${accountId}"
+    }
+}

src/test/groovy/org/prebid/server/functional/tests/storage/AmpS3Spec.groovy

@@ -0,0 +1,113 @@
+package org.prebid.server.functional.tests.storage
+
+import org.prebid.server.functional.model.db.StoredRequest
+import org.prebid.server.functional.model.request.amp.AmpRequest
+import org.prebid.server.functional.model.request.auction.BidRequest
+import org.prebid.server.functional.model.request.auction.Site
+import org.prebid.server.functional.service.PrebidServerException
+import org.prebid.server.functional.service.S3Service
+import org.prebid.server.functional.util.PBSUtils
+
+import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST
+
+class AmpS3Spec extends StorageBaseSpec {
+
+    def "PBS should take parameters from the stored request on S3 service when it's not specified in the request"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest.tap {
+            site = Site.defaultSite
+            setAccountId(ampRequest.account)
+        }
+
+        and: "Stored request in S3 service"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest)
+        S3_SERVICE.uploadStoredRequest(DEFAULT_BUCKET, storedRequest)
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "Bidder request should contain parameters from the stored request"
+        def bidderRequest = bidder.getBidderRequest(ampStoredRequest.id)
+
+        assert bidderRequest.site?.page == ampStoredRequest.site.page
+        assert bidderRequest.site?.publisher?.id == ampStoredRequest.site.publisher.id
+        assert !bidderRequest.imp[0]?.tagId
+        assert bidderRequest.imp[0]?.banner?.format[0]?.h == ampStoredRequest.imp[0].banner.format[0].h
+        assert bidderRequest.imp[0]?.banner?.format[0]?.w == ampStoredRequest.imp[0].banner.format[0].w
+        assert bidderRequest.regs?.gdpr == ampStoredRequest.regs.ext.gdpr
+    }
+
+    def "PBS should throw exception when trying to take parameters from the stored request on S3 service with invalid id in file"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest.tap {
+            site = Site.defaultSite
+            setAccountId(ampRequest.account)
+        }
+
+        and: "Stored request in S3 service"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest).tap {
+            it.requestId = PBSUtils.randomNumber
+        }
+        S3_SERVICE.uploadStoredRequest(DEFAULT_BUCKET, storedRequest, ampRequest.tagId)
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS should throw request format error"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == BAD_REQUEST.code()
+        assert exception.responseBody == "Invalid request format: Stored request processing failed: " +
+                "No stored request found for id: ${ampRequest.tagId}"
+    }
+
+    def "PBS should throw exception when trying to take parameters from the invalid stored request on S3 service"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest.tap {
+            site = Site.defaultSite
+            setAccountId(ampRequest.account)
+        }
+
+        and: "Stored request in S3 service"
+        S3_SERVICE.uploadFile(DEFAULT_BUCKET, INVALID_FILE_BODY, "${S3Service.DEFAULT_REQUEST_DIR}/${ampRequest.tagId}.json")
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS should throw request format error"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == BAD_REQUEST.code()
+        assert exception.responseBody == "Invalid request format: Stored request processing failed: " +
+                "Can't parse Json for stored request with id ${ampRequest.tagId}"
+    }
+
+    def "PBS should throw an exception when trying to take parameters from stored request on S3 service that do not exist"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS should throw request format error"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == BAD_REQUEST.code()
+        assert exception.responseBody == "Invalid request format: Stored request processing failed: " +
+                "No stored request found for id: ${ampRequest.tagId}"
+    }
+}