Aggregation service tools

tools/aggregatable_report_converter/README.md

@@ -0,0 +1,61 @@
+# Aggregatable Report Converter
+
+## Purpose
+To provide Aggregation Service testers tools to create debug aggregatable reports that can be used for Local Testing and AWS Aggregation Service testing.
+
+## Guide
+1. Clone the privacy-sandbox-demos repository.
+2. Go to \<repository\>/tools/aggregatable_report_converter
+3. Options for usage:
+   1. Use the jar file available in \<repository\>/tools/aggregatable_report_converter/out/artifacts/aggregatable_report_converter_jar.
+   2. Build jar file for the project `aggregatable_report_converter` using Eclipse or Intellij.
+
+### Build your jar file in Intellij
+1. Open Project in Intellij
+2. Go to `Build` > `Build Project`.
+3. Go to `File` > `Project Structure`. Select `Artifacts` under `Project Settings`.
+4. Click `+` > `JAR` > `From modules with dependencies...`.
+5. Ensure `module` selected is `aggregatable_report_converter`.
+6. In Main Class, select `Main`. Click `OK` > `Apply`.
+
+
+## How to use
+
+Once the jar file is created, you can get options available for this tool using the command `java -jar aggregatable_report_converter.jar --help`.
+
+To convert json reports to debug aggregatable reports, you can use the below command:
+```angular2html
+java -jar aggregatable_report_converter.jar \
+--request_type convertToAvro \
+--input_file [filename] \
+--debug
+```
+
+To create output domain avro file, you can use the below command:
+```angular2html
+java -jar aggregatable_report_converter.jar \
+--request_type createDomainAvro \
+--bucket_key [bucket key]
+```
+
+## Options
+
+--request_type [request type] \
+Type of request. Options available:
+- convertToAvro: converts a json report to an avro aggregatable report
+- convertToJson: converts the avro summary report to a json report
+- createDomainAvro: creates an output_domain.avro file from provided bucket key
+
+--input_file [file name] \
+This will be the file that will be encoded. 
+
+--debug \
+This will signify that the request is for debug. \ 
+If the type of request is convertToAvro, this will create a debug avro report for local testing. 
+
+--bucket_key [bucket key] \
+Provide your bucket key here.
+
+--output_file [file name] \
+Output filename the report will be written to.
+

tools/aggregatable_report_converter/pom.xml

@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>groupId</groupId>
+    <artifactId>aggregatable_report_converter</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <junit.version>5.8.2</junit.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+            <version>1.11.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.json/json -->
+        <dependency>
+            <groupId>org.json</groupId>
+            <artifactId>json</artifactId>
+            <version>20220924</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.iot.cbor/cbor -->
+        <dependency>
+            <groupId>co.nstant.in</groupId>
+            <artifactId>cbor</artifactId>
+            <version>0.9</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>2.0.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>2.0.7</version>
+<!--            <scope>runtime</scope>-->
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-war-plugin</artifactId>
+                <version>3.3.2</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.avro</groupId>
+                <artifactId>avro-maven-plugin</artifactId>
+                <version>1.11.1</version>
+                <executions>
+                    <execution>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>schema</goal>
+                        </goals>
+                        <configuration>
+                            <sourceDirectory>${project.basedir}/src/main/java/</sourceDirectory>
+                            <outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

tools/aggregatable_report_converter/src/main/java/AggregatableReport.java

@@ -0,0 +1,34 @@
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import org.apache.avro.data.Json;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+public class AggregatableReport {
+  private String sharedInfo;
+  private ArrayList<Payload> aggregationServicePayloads;
+
+  public AggregatableReport(String reportString){
+    JSONObject aggregatableJson = new JSONObject(reportString);
+    sharedInfo = aggregatableJson.getString("shared_info");
+    aggregationServicePayloads = getPayloads((JSONArray) aggregatableJson.get("aggregation_service_payloads"));
+  }
+
+  private static ArrayList<Payload> getPayloads(JSONArray payloads){
+    ArrayList<Payload> aggregatePayloads = new ArrayList<>();
+    for (Object aggregationReport : payloads){
+      JSONObject report = (JSONObject) aggregationReport;
+      Payload payload = new Payload(report);
+      aggregatePayloads.add(payload);
+    }
+    return aggregatePayloads;
+  }
+
+  public String getSharedInfo() {
+    return sharedInfo;
+  }
+
+  public ArrayList<Payload> getAggregationServicePayloads() {
+    return aggregationServicePayloads;
+  }
+}

tools/aggregatable_report_converter/src/main/java/AvroEncoder.java

@@ -0,0 +1,79 @@
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Scanner;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericData.Record;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumWriter;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+public class AvroEncoder {
+
+  static String REPORT_SCHEMA = "{\n"
+      + "  \"type\": \"record\",\n"
+      + "  \"name\": \"AggregatableReport\",\n"
+      + "  \"fields\": [\n"
+      + "    {\n"
+      + "      \"name\": \"payload\",\n"
+      + "      \"type\": \"bytes\"\n"
+      + "    },\n"
+      + "    {\n"
+      + "      \"name\": \"key_id\",\n"
+      + "      \"type\": \"string\"\n"
+      + "    },\n"
+      + "    {\n"
+      + "      \"name\": \"shared_info\",\n"
+      + "      \"type\": \"string\"\n"
+      + "    }\n"
+      + "  ]\n"
+      + "}";
+
+  private static String readFileAsString(String file) throws IOException {
+    return new String(Files.readAllBytes(Paths.get(file)));
+  }
+
+  public static void convertToAvroReport(HashMap<String, String> requestParameters) throws IOException {
+    String fileName = Tools.getFileName(requestParameters);
+    String file = requestParameters.get("inputFile");
+    File outputAvroReport = new File (fileName);
+    // check if the report is debug mode. If yes, the payload class will use the debug cleartext payload.
+    boolean isDebug = Boolean.parseBoolean(requestParameters.get("debugMode"));
+    Schema schema = new Schema.Parser().parse(REPORT_SCHEMA);
+    DatumWriter<GenericRecord> avroWriter = new GenericDatumWriter<GenericRecord>(schema);
+    DataFileWriter<GenericRecord> avroFileWriter = new DataFileWriter<GenericRecord>(avroWriter);
+    avroFileWriter.create(schema, outputAvroReport);
+    String reportJsonString = readFileAsString(file);
+    try {
+      AggregatableReport report = new AggregatableReport(reportJsonString);
+      for (Payload payload : report.getAggregationServicePayloads()) {
+        GenericRecord avroReport = new GenericData.Record(schema);
+        avroReport.put("key_id", payload.getKeyId());
+        avroReport.put("shared_info", report.getSharedInfo());
+        // the getPayload will check if the request is debug or not.
+        avroReport.put("payload", payload.getPayload(isDebug));
+        avroFileWriter.append(avroReport);
+      }
+      avroFileWriter.close();
+      System.out.println("Avro Report created: " + outputAvroReport);
+    } catch (Exception e){
+      System.out.println(e);
+    }
+  }
+
+}

tools/aggregatable_report_converter/src/main/java/AvroFileReader.java

@@ -0,0 +1,46 @@
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.sql.SQLOutput;
+import java.util.Arrays;
+import java.util.HashMap;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.reflect.ReflectData;
+import java.util.Scanner;
+import org.json.JSONObject;
+
+public class AvroFileReader {
+
+  public static void avroReader(HashMap<String, String> requestParameters) throws IOException {
+    String fileName = Tools.getFileName(requestParameters);
+    String file = requestParameters.get("inputFile");
+
+    DataFileReader<GenericRecord> dataFileReader = new DataFileReader<GenericRecord>(
+        new File(file), new GenericDatumReader<>());
+
+    GenericRecord report = null;
+    FileWriter writer = new FileWriter(fileName);
+
+    while (dataFileReader.hasNext()){
+      report = dataFileReader.next(report);
+      ByteBuffer byteBuffer = (ByteBuffer) report.get("bucket");
+      StringBuilder hexString = new StringBuilder();
+      for (int i=0; i<byteBuffer.capacity(); i++){
+        hexString.append(String.format("%02x", byteBuffer.get(i)));
+      }
+      BigInteger bucket = new BigInteger(String.valueOf(hexString), 16);
+      report.put("bucket", bucket);
+      String reportString = report.toString();
+      System.out.println(reportString);
+      writer.write(reportString);
+    }
+    writer.close();
+  }
+
+}

tools/aggregatable_report_converter/src/main/java/HelpOption.java

@@ -0,0 +1,21 @@
+public class HelpOption {
+  public static void getHelp(){
+    String helpOptions = "Usage: Aggregation Service Tools [options] \n"
+        + "  Options: \n"
+        + "    --request_type [request type]\n"
+        + "      Type of request. Options available: \n"
+        + "        convertToAvro: converts a json report to an avro aggregatable report \n"
+        + "        convertToJson: converts the avro summary report to a json report \n"
+        + "        createDomainAvro: creates an output_domain.avro file from provided bucket key\n\n"
+        + "    --input_file [file name]\n"
+        + "      This will be the file that will be encoded. \n\n"
+        + "    --debug \n"
+        + "      This will signify that the request is for debug. \n"
+        + "      If the type of request is convertToAvro, this will create a debug avro report for local testing. \n\n"
+        + "    --bucket_key [bucket key]\n"
+        + "      Provide your bucket key here.\n\n"
+        + "    --output_file [file name]\n"
+        + "      Output filename the report will be written to.\n";
+    System.out.println(helpOptions);
+  }
+}

tools/aggregatable_report_converter/src/main/java/Main.java

@@ -0,0 +1,22 @@
+import java.io.IOException;
+import java.sql.SQLOutput;
+import java.util.HashMap;
+import java.util.Scanner;
+
+public class Main {
+
+  public static void main(String[] args) throws Exception {
+    int lengthOfInput = args.length;
+    if (lengthOfInput == 0) {
+      System.out.println("Please provide arguments. If you need help, use \"--help\".");
+      return;
+    }
+    if ((lengthOfInput == 1) && (args[0].equals("--help"))){
+      HelpOption.getHelp();
+      return;
+    }
+    HashMap<String, String> requestParams = Tools.getRequestParams(args);
+    Tools.processRequest(requestParams);
+  }
+
+}