Improve google results (#205)

pom.xml

@@ -151,6 +151,11 @@
       <artifactId>guava</artifactId>
       <version>19.0</version>
     </dependency>
+    <dependency>
+      <groupId>com.jakewharton.byteunits</groupId>
+      <artifactId>byteunits</artifactId>
+      <version>0.9.1</version>
+    </dependency>
     <dependency>
       <groupId>us.physion</groupId>
       <artifactId>osx-keychain</artifactId>

src/main/java/com/baulsupp/oksocial/apidocs/ServiceApiDocPresenter.java

@@ -3,6 +3,7 @@
 import com.baulsupp.oksocial.authenticator.ServiceInterceptor;
 import com.baulsupp.oksocial.credentials.CredentialsStore;
 import com.baulsupp.oksocial.output.OutputHandler;
+import com.google.common.base.Throwables;
 import java.io.IOException;
 import java.util.Optional;
 import okhttp3.OkHttpClient;
@@ -22,7 +23,13 @@ public ServiceApiDocPresenter(ServiceInterceptor services, OkHttpClient client,
   public void explainApi(String url, OutputHandler outputHandler, OkHttpClient client)
       throws IOException {
     Optional<ApiDocPresenter> presenter =
-        services.getByUrl(url).map(s -> s.apiDocPresenter(url));
+        services.getByUrl(url).map(s -> {
+          try {
+            return s.apiDocPresenter(url);
+          } catch (IOException e) {
+            throw Throwables.propagate(e);
+          }
+        });
 
     if (presenter.isPresent()) {
       presenter.get().explainApi(url, outputHandler, client);

src/main/java/com/baulsupp/oksocial/authenticator/AuthInterceptor.java

@@ -80,7 +80,7 @@ default Optional<T> defaultCredentials() {
     return empty();
   }
 
-  default ApiDocPresenter apiDocPresenter(String url) {
+  default ApiDocPresenter apiDocPresenter(String url) throws IOException {
     return new ApiDocPresenter() {
       @Override public void explainApi(String url, OutputHandler outputHandler, OkHttpClient client)
           throws IOException {

src/main/java/com/baulsupp/oksocial/services/google/DiscoveryApiDocPresenter.java

@@ -2,9 +2,8 @@
 
 import com.baulsupp.oksocial.apidocs.ApiDocPresenter;
 import com.baulsupp.oksocial.output.OutputHandler;
-import com.baulsupp.oksocial.util.Util;
-import com.google.common.base.Strings;
 import java.io.IOException;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
@@ -16,48 +15,83 @@
 import static java.util.stream.Collectors.toList;
 
 public class DiscoveryApiDocPresenter implements ApiDocPresenter {
+  private DiscoveryIndex discoveryIndex;
+
+  public DiscoveryApiDocPresenter(DiscoveryIndex discoveryIndex) {
+    this.discoveryIndex = discoveryIndex;
+  }
+
   @Override public void explainApi(String url, OutputHandler outputHandler, OkHttpClient client)
       throws IOException {
-    DiscoveryIndex discoveryIndex = DiscoveryIndex.loadStatic();
     List<String> discoveryPaths = discoveryIndex.getDiscoveryUrlForPrefix(url);
 
-    DiscoveryRegistry registry = DiscoveryRegistry.loadStatic();
+    DiscoveryRegistry registry = DiscoveryRegistry.instance(client);
 
     CompletableFuture<List<DiscoveryDocument>> docs =
-        join(discoveryPaths.stream().map(p -> registry.load(client, p)).collect(toList()));
+        join(discoveryPaths.stream().map(p -> registry.load(p)).collect(toList()));
 
     Optional<DiscoveryDocument> bestDoc =
         ioSafeGet(docs.thenApply(d -> {
           Optional<DiscoveryDocument> exactMatch =
-              d.stream().filter(x -> x.getUrls().contains(url)).findFirst();
+              d.stream().filter(x -> matches(url, x)).findFirst();
+
+          if (exactMatch.isPresent()) {
+            return exactMatch;
+          }
 
-          // TODO find the longest matching prefix
-          // exact url match or just first doc
-          return Util.or(exactMatch, () -> d.stream().findFirst());
+          // requested url may be a substring of longest baseUrl
+          // assume that this means that single unique service owns this base url
+          Optional<DiscoveryDocument> best = d.stream()
+              .filter(service -> url.startsWith(service.getBaseUrl()))
+              .max(Comparator.comparing(dd -> dd.getBaseUrl().length()));
+
+          if (best.isPresent()) {
+            return best;
+          }
+
+          // multiple services sharing baseurl - return first
+          outputHandler.info("Multiple services for path " + url);
+          return Optional.empty();
         }));
 
-      if (bestDoc.isPresent()) {
-        DiscoveryDocument s = bestDoc.get();
-        outputHandler.info("name: " + s.getApiName());
-        outputHandler.info("docs: " + s.getDocLink());
-
-        Optional<DiscoveryEndpoint> e = s.findEndpoint(url);
-
-        e.ifPresent(de -> {
-          outputHandler.info("endpoint id: " + de.id());
-          outputHandler.info("url: " + de.url());
-          outputHandler.info("scopes: " + de.scopeNames().stream().collect(joining(", ")));
-          outputHandler.info("");
-          outputHandler.info(de.description());
-          outputHandler.info("");
-          de.parameters().forEach(p -> {
-            outputHandler.info("parameter: " + p.name() + " (" + p.type() + ")");
-            outputHandler.info(p.description());
-          });
+    if (bestDoc.isPresent()) {
+      DiscoveryDocument s = bestDoc.get();
+      outputHandler.info("name: " + s.getApiName());
+      outputHandler.info("docs: " + s.getDocLink());
+
+      Optional<DiscoveryEndpoint> e = s.findEndpoint(url);
+
+      e.ifPresent(de -> {
+        outputHandler.info("endpoint id: " + de.id());
+        outputHandler.info("url: " + de.url());
+        outputHandler.info("scopes: " + de.scopeNames().stream().collect(joining(", ")));
+        outputHandler.info("");
+        outputHandler.info(de.description());
+        outputHandler.info("");
+        de.parameters().forEach(p -> {
+          outputHandler.info("parameter: " + p.name() + " (" + p.type() + ")");
+          outputHandler.info(p.description());
         });
-      } else {
-        outputHandler.info("No specific API found");
-        outputHandler.info("https://developers.google.com/apis-explorer/#p/");
+      });
+
+      if (!e.isPresent()) {
+        outputHandler.info("base: " + s.getBaseUrl());
       }
+    } else {
+      outputHandler.info("No specific API found");
+      outputHandler.info("https://developers.google.com/apis-explorer/#p/");
+    }
+  }
+
+  private boolean matches(String url, DiscoveryDocument x) {
+    List<DiscoveryEndpoint> eps = x.getEndpoints();
+
+    for (DiscoveryEndpoint ep : eps) {
+      if (ep.matches(url)) {
+        return true;
+      }
+    }
+
+    return false;
   }
 }

src/main/java/com/baulsupp/oksocial/services/google/DiscoveryDocument.java

@@ -72,6 +72,10 @@ public String getDocLink() {
   }
 
   public Optional<DiscoveryEndpoint> findEndpoint(String url) {
-    return getEndpoints().stream().filter(e -> e.url().equals(url)).findAny();
+    return getEndpoints().stream().filter(e -> matches(url, e)).findAny();
+  }
+
+  private boolean matches(String url, DiscoveryEndpoint e) {
+    return e.url().equals(url) || e.matches(url);
   }
 }

src/main/java/com/baulsupp/oksocial/services/google/DiscoveryEndpoint.java

@@ -3,6 +3,7 @@
 import com.google.common.collect.Lists;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import static java.util.stream.Collectors.toList;
 
@@ -61,4 +62,39 @@ public List<DiscoveryParameter> parameters() {
         .map(p -> new DiscoveryParameter(p.getKey(), p.getValue()))
         .collect(toList());
   }
+
+  public boolean matches(String requestUrl) {
+    if (!requestUrl.startsWith(baseUrl)) {
+      return false;
+    }
+
+    String requestUrlPath = requestUrl.substring(baseUrl.length());
+
+    return buildDocPathRegex().matcher(requestUrlPath).matches();
+  }
+
+  private Pattern buildDocPathRegex() {
+    List<DiscoveryParameter> parameters = parameters();
+
+    boolean hasQueryParams = false;
+
+    String pathPattern = this.path();
+
+    for (DiscoveryParameter p : parameters) {
+      if (p.location().equals("path")) {
+        String pPattern = p.pattern();
+        if (pPattern == null) {
+          pPattern = ".*";
+        } else if (pPattern.matches("\\^.*\\$")) {
+          pPattern = pPattern.substring(1, pPattern.length() - 1);
+        }
+        String x = "\\{\\+?" + p.name() + "\\}";
+        pathPattern = pathPattern.replaceAll(x, pPattern);
+      } else if (p.location().equals("query")) {
+        hasQueryParams = true;
+      }
+    }
+
+    return Pattern.compile(pathPattern + (hasQueryParams ? "(\\?.*)?" : ""));
+  }
 }

src/main/java/com/baulsupp/oksocial/services/google/DiscoveryParameter.java

@@ -23,4 +23,12 @@ public String type() {
   public String description() {
     return (String) map.get("description");
   }
+
+  public String location() {
+    return (String) map.get("location");
+  }
+
+  public String pattern() {
+    return (String) map.get("pattern");
+  }
 }

src/main/java/com/baulsupp/oksocial/services/google/DiscoveryRegistry.java

@@ -2,41 +2,60 @@
 
 import com.baulsupp.oksocial.authenticator.AuthUtil;
 import com.baulsupp.oksocial.util.JsonUtil;
-import com.google.common.io.Resources;
+import java.io.File;
 import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import okhttp3.Cache;
+import okhttp3.CacheControl;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
+import okhttp3.Response;
+
+import static com.jakewharton.byteunits.BinaryByteUnit.MEBIBYTES;
 
 public class DiscoveryRegistry {
+  private static final Cache cache =
+      new Cache(new File(System.getProperty("user.home"), ".oksocial/google-cache"),
+          MEBIBYTES.toBytes(20));
+
+  private static final CacheControl cacheControl =
+      new CacheControl.Builder().maxStale(1, TimeUnit.DAYS).build();
+
+  private final OkHttpClient client;
   private final Map<String, Object> map;
 
-  public DiscoveryRegistry(Map<String, Object> map) {
+  public DiscoveryRegistry(OkHttpClient client, Map<String, Object> map) {
+    this.client = client;
     this.map = map;
   }
 
-  public static DiscoveryRegistry loadStatic() throws IOException {
-    URL url = DiscoveryRegistry.class.getResource("discovery.json");
+  // TODO make non synchronous
+  public static synchronized DiscoveryRegistry instance(OkHttpClient client) throws IOException {
+    client = client.newBuilder().cache(cache).build();
 
-    String definition = Resources.toString(url, StandardCharsets.UTF_8);
-
-    return DiscoveryRegistry.parse(definition);
-  }
+    String url = "https://www.googleapis.com/discovery/v1/apis";
+    Request request = new Request.Builder().cacheControl(cacheControl).url(url).build();
+    Response response = client.newCall(request).execute();
 
-  public static DiscoveryRegistry parse(String definition) throws IOException {
-    return new DiscoveryRegistry(JsonUtil.map(definition));
+    try {
+      return new DiscoveryRegistry(client, JsonUtil.map(response.body().string()));
+    } finally {
+      if (response != null) {
+        response.close();
+      }
+    }
   }
 
   private Map<String, Map<String, Object>> getItems() {
     //noinspection unchecked
     return (Map<String, Map<String, Object>>) map.get("items");
   }
 
-  public CompletableFuture<DiscoveryDocument> load(OkHttpClient client, String discoveryDocPath) {
-    Request request = new Request.Builder().url(discoveryDocPath).build();
+  public CompletableFuture<DiscoveryDocument> load(String discoveryDocPath) {
+    Request request =
+        new Request.Builder().url(discoveryDocPath).cacheControl(cacheControl).build();
     CompletableFuture<Map<String, Object>> mapFuture =
         AuthUtil.enqueueJsonMapRequest(client, request);
 

src/main/java/com/baulsupp/oksocial/services/google/GoogleAuthInterceptor.java

@@ -37,6 +37,7 @@
  */
 public class GoogleAuthInterceptor implements AuthInterceptor<Oauth2Token> {
   private Set<String> hosts = null;
+  private DiscoveryIndex discoveryIndex;
 
   @Override public Oauth2ServiceDefinition serviceDefinition() {
     return new Oauth2ServiceDefinition("www.googleapis.com", "Google API", "google",
@@ -130,7 +131,7 @@ private <R> String extractHost(String s) {
     if (isPastHost(prefix)) {
       List<String> discoveryPaths = DiscoveryIndex.loadStatic().getDiscoveryUrlForPrefix(prefix);
 
-      return GoogleDiscoveryCompleter.forApis(client, DiscoveryRegistry.loadStatic(),
+      return GoogleDiscoveryCompleter.forApis(DiscoveryRegistry.instance(client),
           discoveryPaths);
     } else {
       UrlList urlList = UrlList.fromResource(name()).get();
@@ -143,7 +144,15 @@ private boolean isPastHost(String prefix) {
     return prefix.matches("https://.*/.*");
   }
 
-  @Override public ApiDocPresenter apiDocPresenter(String url) {
-    return new DiscoveryApiDocPresenter();
+  @Override public ApiDocPresenter apiDocPresenter(String url) throws IOException {
+    return new DiscoveryApiDocPresenter(discoveryIndex());
+  }
+
+  private synchronized DiscoveryIndex discoveryIndex() throws IOException {
+    if (discoveryIndex == null) {
+      discoveryIndex = DiscoveryIndex.loadStatic();
+    }
+
+    return discoveryIndex;
   }
 }

src/main/java/com/baulsupp/oksocial/services/google/GoogleDiscoveryCompleter.java

@@ -7,19 +7,16 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Future;
 import okhttp3.HttpUrl;
-import okhttp3.OkHttpClient;
 
 import static com.baulsupp.oksocial.util.FutureUtil.join;
 import static java.util.stream.Collectors.toList;
 
 public class GoogleDiscoveryCompleter implements ApiCompleter {
-  private OkHttpClient client;
   private DiscoveryRegistry discoveryRegistry;
   private final List<String> discoveryDocPaths;
 
-  public GoogleDiscoveryCompleter(OkHttpClient client, DiscoveryRegistry discoveryRegistry,
+  public GoogleDiscoveryCompleter(DiscoveryRegistry discoveryRegistry,
       List<String> discoveryDocPaths) {
-    this.client = client;
     this.discoveryRegistry = discoveryRegistry;
     this.discoveryDocPaths = discoveryDocPaths;
   }
@@ -41,12 +38,11 @@ private UrlList flattenList(List<List<String>> l) {
   }
 
   private CompletableFuture<List<String>> singleFuture(String discoveryDocPath) {
-    return discoveryRegistry.load(client, discoveryDocPath).thenApply(s -> s.getUrls());
+    return discoveryRegistry.load(discoveryDocPath).thenApply(s -> s.getUrls());
   }
 
-  public static GoogleDiscoveryCompleter forApis(OkHttpClient client,
-      DiscoveryRegistry discoveryRegistry,
+  public static GoogleDiscoveryCompleter forApis(DiscoveryRegistry discoveryRegistry,
       List<String> discoveryDocPaths) {
-    return new GoogleDiscoveryCompleter(client, discoveryRegistry, discoveryDocPaths);
+    return new GoogleDiscoveryCompleter(discoveryRegistry, discoveryDocPaths);
   }
 }