Greenbids RTD: Add Module (#3242)

extra/bundle/pom.xml

@@ -45,6 +45,11 @@
             <artifactId>pb-response-correction</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.prebid.server.hooks.modules</groupId>
+            <artifactId>greenbids-real-time-data</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 
     <build>

extra/modules/greenbids-real-time-data/pom.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.prebid.server.hooks.modules</groupId>
+        <artifactId>all-modules</artifactId>
+        <version>3.15.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>greenbids-real-time-data</artifactId>
+
+    <name>greenbids-real-time-data</name>
+    <description>Greenbids Real Time Data</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.github.ua-parser</groupId>
+            <artifactId>uap-java</artifactId>
+            <version>1.6.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.microsoft.onnxruntime</groupId>
+            <artifactId>onnxruntime</artifactId>
+            <version>1.16.1</version> <!-- Use the latest available version -->
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>2.41.0</version>
+        </dependency>
+    </dependencies>
+
+</project>

extra/modules/greenbids-real-time-data/src/lombok.config

@@ -0,0 +1 @@
+lombok.anyConstructor.addConstructorProperties = true

extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java

@@ -0,0 +1,55 @@
+package org.prebid.server.hooks.modules.greenbids.real.time.data.config;
+
+import io.vertx.core.Promise;
+import io.vertx.core.Vertx;
+import com.maxmind.geoip2.DatabaseReader;
+import org.prebid.server.exception.PreBidException;
+import org.prebid.server.vertx.Initializable;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class DatabaseReaderFactory implements Initializable {
+
+    private final String geoLiteCountryUrl;
+
+    private final Vertx vertx;
+
+    private final AtomicReference<DatabaseReader> databaseReaderRef = new AtomicReference<>();
+
+    public DatabaseReaderFactory(String geoLitCountryUrl, Vertx vertx) {
+        this.geoLiteCountryUrl = geoLitCountryUrl;
+        this.vertx = vertx;
+    }
+
+    @Override
+    public void initialize(Promise<Void> initializePromise) {
+
+        vertx.executeBlocking(() -> {
+            try {
+                final URL url = new URL(geoLiteCountryUrl);
+                final Path databasePath = Files.createTempFile("GeoLite2-Country", ".mmdb");
+
+                try (InputStream inputStream = url.openStream();
+                     FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) {
+                    inputStream.transferTo(outputStream);
+                }
+
+                databaseReaderRef.set(new DatabaseReader.Builder(databasePath.toFile()).build());
+            } catch (IOException e) {
+                throw new PreBidException("Failed to initialize DatabaseReader from URL", e);
+            }
+            return null;
+        }).<Void>mapEmpty()
+        .onComplete(initializePromise);
+    }
+
+    public DatabaseReader getDatabaseReader() {
+        return databaseReaderRef.get();
+    }
+}

extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java

@@ -0,0 +1,134 @@
+package org.prebid.server.hooks.modules.greenbids.real.time.data.config;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import io.vertx.core.Vertx;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.model.filter.ThrottlingThresholds;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.ThrottlingThresholdsFactory;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.GreenbidsInferenceDataService;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.FilterService;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.ModelCache;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.OnnxModelRunner;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.OnnxModelRunnerFactory;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.OnnxModelRunnerWithThresholds;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.ThresholdCache;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.core.GreenbidsInvocationService;
+import org.prebid.server.hooks.modules.greenbids.real.time.data.v1.GreenbidsRealTimeDataProcessedAuctionRequestHook;
+import org.prebid.server.json.ObjectMapperProvider;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+@ConditionalOnProperty(prefix = "hooks." + GreenbidsRealTimeDataModule.CODE, name = "enabled", havingValue = "true")
+@Configuration
+@EnableConfigurationProperties(GreenbidsRealTimeDataProperties.class)
+public class GreenbidsRealTimeDataConfiguration {
+
+    @Bean
+    DatabaseReaderFactory databaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx) {
+        return new DatabaseReaderFactory(properties.getGeoLiteCountryPath(), vertx);
+    }
+
+    @Bean
+    GreenbidsInferenceDataService greenbidsInferenceDataService(DatabaseReaderFactory databaseReaderFactory) {
+        return new GreenbidsInferenceDataService(
+                databaseReaderFactory, ObjectMapperProvider.mapper());
+    }
+
+    @Bean
+    GreenbidsRealTimeDataModule greenbidsRealTimeDataModule(
+            FilterService filterService,
+            OnnxModelRunnerWithThresholds onnxModelRunnerWithThresholds,
+            GreenbidsInferenceDataService greenbidsInferenceDataService,
+            GreenbidsInvocationService greenbidsInvocationService) {
+
+        return new GreenbidsRealTimeDataModule(List.of(
+                new GreenbidsRealTimeDataProcessedAuctionRequestHook(
+                        ObjectMapperProvider.mapper(),
+                        filterService,
+                        onnxModelRunnerWithThresholds,
+                        greenbidsInferenceDataService,
+                        greenbidsInvocationService)));
+    }
+
+    @Bean
+    FilterService filterService() {
+        return new FilterService();
+    }
+
+    @Bean
+    Storage storage(GreenbidsRealTimeDataProperties properties) {
+        return StorageOptions.newBuilder()
+                .setProjectId(properties.getGoogleCloudGreenbidsProject()).build().getService();
+    }
+
+    @Bean
+    OnnxModelRunnerFactory onnxModelRunnerFactory() {
+        return new OnnxModelRunnerFactory();
+    }
+
+    @Bean
+    ThrottlingThresholdsFactory throttlingThresholdsFactory() {
+        return new ThrottlingThresholdsFactory();
+    }
+
+    @Bean
+    ModelCache modelCache(
+            GreenbidsRealTimeDataProperties properties,
+            Vertx vertx,
+            Storage storage,
+            OnnxModelRunnerFactory onnxModelRunnerFactory) {
+
+        final Cache<String, OnnxModelRunner> modelCacheWithExpiration = Caffeine.newBuilder()
+                .expireAfterWrite(properties.getCacheExpirationMinutes(), TimeUnit.MINUTES)
+                .build();
+
+        return new ModelCache(
+                storage,
+                properties.getGcsBucketName(),
+                modelCacheWithExpiration,
+                properties.getOnnxModelCacheKeyPrefix(),
+                vertx,
+                onnxModelRunnerFactory);
+    }
+
+    @Bean
+    ThresholdCache thresholdCache(
+            GreenbidsRealTimeDataProperties properties,
+            Vertx vertx,
+            Storage storage,
+            ThrottlingThresholdsFactory throttlingThresholdsFactory) {
+
+        final Cache<String, ThrottlingThresholds> thresholdsCacheWithExpiration = Caffeine.newBuilder()
+                .expireAfterWrite(properties.getCacheExpirationMinutes(), TimeUnit.MINUTES)
+                .build();
+
+        return new ThresholdCache(
+                storage,
+                properties.getGcsBucketName(),
+                ObjectMapperProvider.mapper(),
+                thresholdsCacheWithExpiration,
+                properties.getThresholdsCacheKeyPrefix(),
+                vertx,
+                throttlingThresholdsFactory);
+    }
+
+    @Bean
+    OnnxModelRunnerWithThresholds onnxModelRunnerWithThresholds(
+            ModelCache modelCache,
+            ThresholdCache thresholdCache) {
+
+        return new OnnxModelRunnerWithThresholds(modelCache, thresholdCache);
+    }
+
+    @Bean
+    GreenbidsInvocationService greenbidsInvocationService() {
+        return new GreenbidsInvocationService();
+    }
+}

extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataModule.java

@@ -0,0 +1,29 @@
+package org.prebid.server.hooks.modules.greenbids.real.time.data.config;
+
+import org.prebid.server.hooks.v1.Hook;
+import org.prebid.server.hooks.v1.InvocationContext;
+import org.prebid.server.hooks.v1.Module;
+
+import java.util.Collection;
+import java.util.List;
+
+public class GreenbidsRealTimeDataModule implements Module {
+
+    public static final String CODE = "greenbids-real-time-data";
+
+    private final List<? extends Hook<?, ? extends InvocationContext>> hooks;
+
+    public GreenbidsRealTimeDataModule(List<? extends Hook<?, ? extends InvocationContext>> hooks) {
+        this.hooks = hooks;
+    }
+
+    @Override
+    public String code() {
+        return CODE;
+    }
+
+    @Override
+    public Collection<? extends Hook<?, ? extends InvocationContext>> hooks() {
+        return hooks;
+    }
+}

extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java

@@ -0,0 +1,21 @@
+package org.prebid.server.hooks.modules.greenbids.real.time.data.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@ConfigurationProperties(prefix = "hooks.modules." + GreenbidsRealTimeDataModule.CODE)
+@Data
+public class GreenbidsRealTimeDataProperties {
+
+    String googleCloudGreenbidsProject;
+
+    String geoLiteCountryPath;
+
+    String gcsBucketName;
+
+    Integer cacheExpirationMinutes;
+
+    String onnxModelCacheKeyPrefix;
+
+    String thresholdsCacheKeyPrefix;
+}