diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index a40c98ebb25..620b56d499d 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,5 +1,8 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; import io.vertx.core.Promise; import io.vertx.core.Vertx; import com.maxmind.geoip2.DatabaseReader; @@ -8,36 +11,39 @@ import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; public class DatabaseReaderFactory implements Initializable { - private final String geoLiteCountryUrl; + private final String gcsBucketName; + + private final String geoLiteCountryPath; private final Vertx vertx; + private final Storage storage; + private final AtomicReference databaseReaderRef = new AtomicReference<>(); - public DatabaseReaderFactory(String geoLitCountryUrl, Vertx vertx) { - this.geoLiteCountryUrl = geoLitCountryUrl; + public DatabaseReaderFactory(String gcsBucketName, String geoLiteCountryPath, Vertx vertx, Storage storage) { + this.gcsBucketName = gcsBucketName; + this.geoLiteCountryPath = geoLiteCountryPath; this.vertx = vertx; + this.storage = storage; } @Override public void initialize(Promise initializePromise) { - vertx.executeBlocking(() -> { try { - final URL url = new URL(geoLiteCountryUrl); + final Blob blob = getBlob(); final Path databasePath = Files.createTempFile("GeoLite2-Country", ".mmdb"); - try (InputStream inputStream = url.openStream(); - FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) { - inputStream.transferTo(outputStream); + try (FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) { + outputStream.write(blob.getContent()); } databaseReaderRef.set(new DatabaseReader.Builder(databasePath.toFile()).build()); @@ -49,6 +55,16 @@ public void initialize(Promise initializePromise) { .onComplete(initializePromise); } + private Blob getBlob() { + try { + return Optional.ofNullable(storage.get(gcsBucketName)) + .map(bucket -> bucket.get(geoLiteCountryPath)) + .orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName)); + } catch (StorageException e) { + throw new PreBidException("Error accessing GCS artefact for model: ", e); + } + } + public DatabaseReader getDatabaseReader() { return databaseReaderRef.get(); } diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 959352d1908..4e2da2cc5ce 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -31,8 +31,10 @@ public class GreenbidsRealTimeDataConfiguration { @Bean - DatabaseReaderFactory databaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx) { - return new DatabaseReaderFactory(properties.getGeoLiteCountryPath(), vertx); + DatabaseReaderFactory databaseReaderFactory( + GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) { + return new DatabaseReaderFactory( + properties.getGcsBucketName(), properties.getGeoLiteCountryPath(), vertx, storage); } @Bean diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 3bd3e37b859..e3a2988b112 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Device; +import com.iab.openrtb.request.Geo; import com.iab.openrtb.request.Imp; import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.exception.GeoIp2Exception; @@ -86,12 +87,16 @@ private List extractMessagesForImp( final String ip = Optional.ofNullable(bidRequest.getDevice()) .map(Device::getIp) .orElse(null); - final String countryFromIp = getCountry(ip); + final String country = Optional.ofNullable(bidRequest.getDevice()) + .map(Device::getGeo) + .map(Geo::getCountry) + .orElseGet(() -> getCountry(ip)); + return createThrottlingMessages( bidderNode, impId, greenbidsUserAgent, - countryFromIp, + country, hostname, hourBucket, minuteQuadrant); diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java index 157b70b9474..b7cbac4f46a 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java @@ -12,10 +12,12 @@ import com.iab.openrtb.request.Device; import com.iab.openrtb.request.Imp; import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.exception.GeoIp2Exception; +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.record.Country; import io.vertx.core.Future; import io.vertx.core.Vertx; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -52,12 +54,9 @@ import org.prebid.server.proto.openrtb.ext.request.ExtRequest; import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.net.URL; +import java.net.InetAddress; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; import java.util.List; @@ -66,6 +65,7 @@ import static java.util.function.UnaryOperator.identity; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mock.Strictness.LENIENT; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -88,15 +88,27 @@ public class GreenbidsRealTimeDataProcessedAuctionRequestHookTest { @Mock(strictness = LENIENT) private DatabaseReaderFactory databaseReaderFactory; - @Mock - private DatabaseReader dbReader; + @Mock(strictness = LENIENT) + private DatabaseReader databaseReader; + + @Mock(strictness = LENIENT) + private CountryResponse countryResponse; + + @Mock(strictness = LENIENT) + private Country country; private GreenbidsRealTimeDataProcessedAuctionRequestHook target; @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, GeoIp2Exception { final Storage storage = StorageOptions.newBuilder() .setProjectId("test_project").build().getService(); + + when(country.getName()).thenReturn("United States"); + when(countryResponse.getCountry()).thenReturn(country); + when(databaseReader.country(any(InetAddress.class))).thenReturn(countryResponse); + when(databaseReaderFactory.getDatabaseReader()).thenReturn(databaseReader); + final FilterService filterService = new FilterService(); final OnnxModelRunnerFactory onnxModelRunnerFactory = new OnnxModelRunnerFactory(); final ThrottlingThresholdsFactory throttlingThresholdsFactory = new ThrottlingThresholdsFactory(); @@ -118,7 +130,7 @@ public void setUp() throws IOException { final OnnxModelRunnerWithThresholds onnxModelRunnerWithThresholds = new OnnxModelRunnerWithThresholds( modelCache, thresholdCache); - when(databaseReaderFactory.getDatabaseReader()).thenReturn(dbReader); + final GreenbidsInferenceDataService greenbidsInferenceDataService = new GreenbidsInferenceDataService( databaseReaderFactory, TestBidRequestProvider.MAPPER); @@ -162,7 +174,6 @@ public void callShouldExitEarlyWhenPartnerNotActivatedInBidRequest() { assertThat(result.analyticsTags()).isNull(); } - @Disabled("Broken until dbReader is mocked") @Test public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() throws OrtException, IOException { // given @@ -217,7 +228,6 @@ public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() thr assertThat(fingerprint).isNotNull(); } - @Disabled("Broken until dbReader is mocked") @Test public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() throws OrtException, IOException { // given @@ -278,7 +288,6 @@ public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() thro assertThat(resultBidRequest).usingRecursiveComparison().isEqualTo(expectedBidRequest); } - @Disabled("Broken until dbReader is mocked") @Test public void callShouldFilterBiddersBasedOnModelResults() throws OrtException, IOException { // given