Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Greenbids fix geolookup: host and fetch GCS + mock dbReader UT #3626

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
+58 −26
Open

Greenbids fix geolookup: host and fetch GCS + mock dbReader UT #3626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
fb38c22
Fix geolookup: mock dbReader TU
EvgeniiMunin Dec 13, 2024
5a11d49
fetch mmdb from GCS
EvgeniiMunin Dec 13, 2024
e23fb55
local run dbReader from GCS
EvgeniiMunin Dec 13, 2024
ee640d3
fallback to RTD maxmind to get country
EvgeniiMunin Dec 13, 2024
d902510
fixes review
EvgeniiMunin Dec 16, 2024
8563d47
empty commit check flaky TI
EvgeniiMunin Dec 16, 2024
90d3a2e
empty commit check flaky TI
EvgeniiMunin Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 26 additions & 10 deletions ...rg/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
package org.prebid.server.hooks.modules.greenbids.real.time.data.config;

import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageException;
import io.vertx.core.Promise;
import io.vertx.core.Vertx;
import com.maxmind.geoip2.DatabaseReader;
Expand All @@ -8,36 +11,39 @@

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;

public class DatabaseReaderFactory implements Initializable {

private final String geoLiteCountryUrl;
private final String gcsBucketName;

private final String geoLiteCountryPath;

private final Vertx vertx;

private final Storage storage;

private final AtomicReference<DatabaseReader> databaseReaderRef = new AtomicReference<>();

public DatabaseReaderFactory(String geoLitCountryUrl, Vertx vertx) {
this.geoLiteCountryUrl = geoLitCountryUrl;
public DatabaseReaderFactory(String gcsBucketName, String geoLiteCountryPath, Vertx vertx, Storage storage) {
this.gcsBucketName = gcsBucketName;
this.geoLiteCountryPath = geoLiteCountryPath;
this.vertx = vertx;
this.storage = storage;
}

@Override
public void initialize(Promise<Void> initializePromise) {

vertx.executeBlocking(() -> {
try {
final URL url = new URL(geoLiteCountryUrl);
final Blob blob = getBlob();
final Path databasePath = Files.createTempFile("GeoLite2-Country", ".mmdb");

try (InputStream inputStream = url.openStream();
FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) {
inputStream.transferTo(outputStream);
try (FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) {
outputStream.write(blob.getContent());
}

databaseReaderRef.set(new DatabaseReader.Builder(databasePath.toFile()).build());
Expand All @@ -49,6 +55,16 @@ public void initialize(Promise<Void> initializePromise) {
.onComplete(initializePromise);
}

private Blob getBlob() {
try {
return Optional.ofNullable(storage.get(gcsBucketName))
.map(bucket -> bucket.get(geoLiteCountryPath))
.orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName));
} catch (StorageException e) {
throw new PreBidException("Error accessing GCS artefact for model: ", e);
}
}

public DatabaseReader getDatabaseReader() {
return databaseReaderRef.get();
}
Expand Down
6 changes: 4 additions & 2 deletions ...ver/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@
public class GreenbidsRealTimeDataConfiguration {

@Bean
DatabaseReaderFactory databaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx) {
return new DatabaseReaderFactory(properties.getGeoLiteCountryPath(), vertx);
DatabaseReaderFactory databaseReaderFactory(
GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) {
return new DatabaseReaderFactory(
properties.getGcsBucketName(), properties.getGeoLiteCountryPath(), vertx, storage);
}

@Bean
Expand Down
9 changes: 7 additions & 2 deletions ...bid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.iab.openrtb.request.BidRequest;
import com.iab.openrtb.request.Device;
import com.iab.openrtb.request.Geo;
import com.iab.openrtb.request.Imp;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
Expand Down Expand Up @@ -86,12 +87,16 @@ private List<ThrottlingMessage> extractMessagesForImp(
final String ip = Optional.ofNullable(bidRequest.getDevice())
.map(Device::getIp)
.orElse(null);
final String countryFromIp = getCountry(ip);
final String country = Optional.ofNullable(bidRequest.getDevice())
.map(Device::getGeo)
.map(Geo::getCountry)
.orElseGet(() -> getCountry(ip));

return createThrottlingMessages(
bidderNode,
impId,
greenbidsUserAgent,
countryFromIp,
country,
hostname,
hourBucket,
minuteQuadrant);
Expand Down
33 changes: 21 additions & 12 deletions ...les/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@
import com.iab.openrtb.request.Device;
import com.iab.openrtb.request.Imp;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.record.Country;
import io.vertx.core.Future;
import io.vertx.core.Vertx;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
Expand Down Expand Up @@ -52,12 +54,9 @@
import org.prebid.server.proto.openrtb.ext.request.ExtRequest;
import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.InetAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
Expand All @@ -66,6 +65,7 @@

import static java.util.function.UnaryOperator.identity;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mock.Strictness.LENIENT;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
Expand All @@ -88,15 +88,27 @@ public class GreenbidsRealTimeDataProcessedAuctionRequestHookTest {
@Mock(strictness = LENIENT)
private DatabaseReaderFactory databaseReaderFactory;

@Mock
private DatabaseReader dbReader;
@Mock(strictness = LENIENT)
private DatabaseReader databaseReader;

@Mock(strictness = LENIENT)
private CountryResponse countryResponse;

@Mock(strictness = LENIENT)
private Country country;

private GreenbidsRealTimeDataProcessedAuctionRequestHook target;

@BeforeEach
public void setUp() throws IOException {
public void setUp() throws IOException, GeoIp2Exception {
final Storage storage = StorageOptions.newBuilder()
.setProjectId("test_project").build().getService();

when(country.getName()).thenReturn("United States");
when(countryResponse.getCountry()).thenReturn(country);
when(databaseReader.country(any(InetAddress.class))).thenReturn(countryResponse);
when(databaseReaderFactory.getDatabaseReader()).thenReturn(databaseReader);

final FilterService filterService = new FilterService();
final OnnxModelRunnerFactory onnxModelRunnerFactory = new OnnxModelRunnerFactory();
final ThrottlingThresholdsFactory throttlingThresholdsFactory = new ThrottlingThresholdsFactory();
Expand All @@ -118,7 +130,7 @@ public void setUp() throws IOException {
final OnnxModelRunnerWithThresholds onnxModelRunnerWithThresholds = new OnnxModelRunnerWithThresholds(
modelCache,
thresholdCache);
when(databaseReaderFactory.getDatabaseReader()).thenReturn(dbReader);

final GreenbidsInferenceDataService greenbidsInferenceDataService = new GreenbidsInferenceDataService(
databaseReaderFactory,
TestBidRequestProvider.MAPPER);
Expand Down Expand Up @@ -162,7 +174,6 @@ public void callShouldExitEarlyWhenPartnerNotActivatedInBidRequest() {
assertThat(result.analyticsTags()).isNull();
}

@Disabled("Broken until dbReader is mocked")
@Test
public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() throws OrtException, IOException {
// given
Expand Down Expand Up @@ -217,7 +228,6 @@ public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() thr
assertThat(fingerprint).isNotNull();
}

@Disabled("Broken until dbReader is mocked")
@Test
public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() throws OrtException, IOException {
// given
Expand Down Expand Up @@ -278,7 +288,6 @@ public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() thro
assertThat(resultBidRequest).usingRecursiveComparison().isEqualTo(expectedBidRequest);
}

@Disabled("Broken until dbReader is mocked")
@Test
public void callShouldFilterBiddersBasedOnModelResults() throws OrtException, IOException {
// given
Expand Down
Loading