Serve resource files on demand

server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java

@@ -9,10 +9,7 @@
  * ****************************************************************************** */
 package org.eclipse.openvsx.adapter;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
-import io.micrometer.core.instrument.Metrics;
-import io.micrometer.core.instrument.Tag;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.openvsx.entities.Extension;
 import org.eclipse.openvsx.entities.ExtensionVersion;
@@ -23,9 +20,7 @@
 import org.eclipse.openvsx.storage.StorageUtilService;
 import org.eclipse.openvsx.util.*;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.CacheControl;
 import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Component;
 import org.springframework.web.server.ResponseStatusException;
@@ -34,7 +29,6 @@
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.eclipse.openvsx.adapter.ExtensionQueryParam.Criterion.*;
@@ -53,6 +47,7 @@ public class LocalVSCodeService implements IVSCodeService {
     private final SearchUtilService search;
     private final StorageUtilService storageUtil;
     private final ExtensionVersionIntegrityService integrityService;
+    private final WebResourceService webResources;
 
     @Value("${ovsx.webui.url:}")
     String webuiUrl;
@@ -62,13 +57,15 @@ public LocalVSCodeService(
             VersionService versions,
             SearchUtilService search,
             StorageUtilService storageUtil,
-            ExtensionVersionIntegrityService integrityService
+            ExtensionVersionIntegrityService integrityService,
+            WebResourceService webResources
     ) {
         this.repositories = repositories;
         this.versions = versions;
         this.search = search;
         this.storageUtil = storageUtil;
         this.integrityService = integrityService;
+        this.webResources = webResources;
     }
 
     @Override
@@ -307,23 +304,28 @@ public ResponseEntity<StreamingResponseBody> getAsset(
                 FILE_SIGNATURE, DOWNLOAD_SIG
         );
 
-        FileResource resource = null;
         var type = assets.get(assetType);
         if(type != null) {
-            resource = repositories.findFileByType(namespace, extensionName, targetPlatform, version, type);
+            var resource = repositories.findFileByType(namespace, extensionName, targetPlatform, version, type);
+            if (resource == null) {
+                throw new NotFoundException();
+            }
+            if (resource.getType().equals(FileResource.DOWNLOAD)) {
+                storageUtil.increaseDownloadCount(resource);
+            }
+
+            return storageUtil.getFileResponse(resource);
         } else if(asset.startsWith(FILE_WEB_RESOURCES + "/extension/")) {
             var name = asset.substring((FILE_WEB_RESOURCES.length() + 1));
-            resource = repositories.findFileByTypeAndName(namespace, extensionName, targetPlatform, version, FileResource.RESOURCE, name);
-        }
+            var file = webResources.getWebResource(namespace, extensionName, targetPlatform, version, name, false);
+            if(file == null) {
+                throw new NotFoundException();
+            }
 
-        if (resource == null) {
-            throw new NotFoundException();
-        }
-        if (resource.getType().equals(FileResource.DOWNLOAD)) {
-            storageUtil.increaseDownloadCount(resource);
+            return storageUtil.getFileResponse(file);
         }
 
-        return storageUtil.getFileResponse(resource);
+        throw new NotFoundException();
     }
 
     @Override
@@ -376,68 +378,12 @@ public ResponseEntity<StreamingResponseBody> browse(String namespaceName, String
             });
         }
 
-        var extVersion = repositories.findActiveExtensionVersion(version, extensionName, namespaceName);
-        if (extVersion == null) {
+        var file = webResources.getWebResource(namespaceName, extensionName, null, version, path, true);
+        if(file == null) {
             throw new NotFoundException();
         }
 
-        var matches = repositories.findResourceFileResources(extVersion, path);
-        if(matches.isEmpty()) {
-            throw new NotFoundException();
-        }
-
-        var firstMatch = matches.get(0);
-        Metrics.counter("vscode.unpkg", List.of(
-                Tag.of("extension", NamingUtil.toLogFormat(extVersion)),
-                Tag.of("file", String.valueOf(matches.size() == 1 && firstMatch.getName().equals(path))),
-                Tag.of("path", path)
-        )).increment();
-
-        if (matches.size() == 1 && firstMatch.getName().equals(path)) {
-            return storageUtil.getFileResponse(firstMatch);
-        } else {
-            return ResponseEntity.ok()
-                    .contentType(MediaType.APPLICATION_JSON)
-                    .cacheControl(CacheControl.maxAge(7, TimeUnit.DAYS).cachePublic())
-                    .body(outputStream -> {
-                        var urls = browseDirectory(matches, namespaceName, extensionName, version, path);
-                        new ObjectMapper().writeValue(outputStream, urls);
-                    });
-        }
-    }
-
-    private Set<String> browseDirectory(
-            List<FileResource> resources,
-            String namespaceName,
-            String extensionName,
-            String version,
-            String path
-    ) {
-        if(!path.isEmpty() && !path.endsWith("/")) {
-            path += "/";
-        }
-
-        var urls = new HashSet<String>();
-        var baseUrl = UrlUtil.createApiUrl(UrlUtil.getBaseUrl(), "vscode", "unpkg", namespaceName, extensionName, version);
-        for(var resource : resources) {
-            var name = resource.getName();
-            if(name.startsWith(path)) {
-                var index = name.indexOf('/', path.length());
-                var isDirectory = index != -1;
-                if(isDirectory) {
-                    name = name.substring(0, index);
-                }
-
-                var url = UrlUtil.createApiUrl(baseUrl, name.split("/"));
-                if(isDirectory) {
-                    url += '/';
-                }
-
-                urls.add(url);
-            }
-        }
-
-        return urls;
+        return storageUtil.getFileResponse(file);
     }
 
     private ExtensionQueryResult.Extension toQueryExtension(Extension extension, ExtensionVersion latest, List<ExtensionQueryResult.ExtensionVersion> versions, int flags) {

server/src/main/java/org/eclipse/openvsx/adapter/WebResourceService.java

@@ -0,0 +1,100 @@
+/** ******************************************************************************
+ * Copyright (c) 2024 Precies. Software OU and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.adapter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.eclipse.openvsx.entities.FileResource;
+import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.storage.StorageUtilService;
+import org.eclipse.openvsx.util.ErrorResultException;
+import org.eclipse.openvsx.util.NamingUtil;
+import org.eclipse.openvsx.util.UrlUtil;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.stream.Collectors;
+import java.util.zip.ZipFile;
+
+import static org.eclipse.openvsx.cache.CacheService.CACHE_WEB_RESOURCE_FILES;
+import static org.eclipse.openvsx.cache.CacheService.GENERATOR_FILES;
+
+@Component
+public class WebResourceService {
+
+    private final StorageUtilService storageUtil;
+    private final RepositoryService repositories;
+
+    public WebResourceService(StorageUtilService storageUtil, RepositoryService repositories) {
+        this.storageUtil = storageUtil;
+        this.repositories = repositories;
+    }
+
+    @Cacheable(value = CACHE_WEB_RESOURCE_FILES, keyGenerator = GENERATOR_FILES)
+    public Path getWebResource(String namespace, String extension, String targetPlatform, String version, String name, boolean browse) {
+        var download = repositories.findFileByType(namespace, extension, targetPlatform, version, FileResource.DOWNLOAD);
+        if(download == null) {
+            return null;
+        }
+
+        Path path;
+        try {
+            path = storageUtil.getCachedFile(download);
+        } catch(IOException e) {
+            throw new ErrorResultException("Failed to get file for download " + NamingUtil.toLogFormat(download.getExtension()));
+        }
+        if(path == null) {
+            return null;
+        }
+
+        try(var zip = new ZipFile(path.toFile())) {
+            var fileEntry = zip.getEntry(name);
+            if(fileEntry != null) {
+                var fileExtIndex = fileEntry.getName().lastIndexOf('.');
+                var fileExt = fileExtIndex != -1 ? fileEntry.getName().substring(fileExtIndex) : "";
+                var file = Files.createTempFile("webresource_", fileExt);
+                try(var in = zip.getInputStream(fileEntry)) {
+                    Files.copy(in, file, StandardCopyOption.REPLACE_EXISTING);
+                }
+
+                return file;
+            } else if (browse) {
+                var dirName = name.isEmpty() || name.endsWith("/") ? name : name + "/";
+                var dirEntries = zip.stream()
+                        .filter(entry -> entry.getName().startsWith(dirName))
+                        .map(entry -> {
+                            var folderNameEndIndex = entry.getName().indexOf("/", dirName.length());
+                            return folderNameEndIndex == -1 ? entry.getName() : entry.getName().substring(0, folderNameEndIndex + 1);
+                        })
+                        .collect(Collectors.toSet());
+                if(dirEntries.isEmpty()) {
+                    return null;
+                }
+
+                var file = Files.createTempFile("webresource_", ".unpkg.json");
+                var baseUrl = UrlUtil.createApiUrl(UrlUtil.getBaseUrl(), "vscode", "unpkg", namespace, extension, version);
+                var mapper = new ObjectMapper();
+                var node = mapper.createArrayNode();
+                for(var entry : dirEntries) {
+                    node.add(baseUrl + "/" + entry);
+                }
+                mapper.writeValue(file.toFile(), node);
+                return file;
+            } else {
+                return null;
+            }
+        } catch (IOException e) {
+            throw new ErrorResultException("Failed to read extension files for " + NamingUtil.toLogFormat(download.getExtension()));
+        }
+    }
+}

server/src/main/java/org/eclipse/openvsx/cache/CacheService.java

@@ -25,6 +25,8 @@
 public class CacheService {
 
     public static final String CACHE_DATABASE_SEARCH = "database.search";
+    public static final String CACHE_WEB_RESOURCE_FILES = "files.webresource";
+    public static final String CACHE_EXTENSION_FILES = "files.extension";
     public static final String CACHE_EXTENSION_JSON = "extension.json";
     public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version";
     public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json";
@@ -34,6 +36,7 @@ public class CacheService {
 
     public static final String GENERATOR_EXTENSION_JSON = "extensionJsonCacheKeyGenerator";
     public static final String GENERATOR_LATEST_EXTENSION_VERSION = "latestExtensionVersionCacheKeyGenerator";
+    public static final String GENERATOR_FILES = "filesCacheKeyGenerator";
 
     private final CacheManager cacheManager;
     private final RepositoryService repositories;

server/src/main/java/org/eclipse/openvsx/cache/ExpiredFileListener.java

@@ -0,0 +1,38 @@
+/** ******************************************************************************
+ * Copyright (c) 2024 Precies. Software OU and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.cache;
+
+import org.ehcache.event.CacheEvent;
+import org.ehcache.event.CacheEventListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class ExpiredFileListener implements CacheEventListener<String, Path> {
+    protected final Logger logger = LoggerFactory.getLogger(ExpiredFileListener.class);
+    @Override
+    public void onEvent(CacheEvent<? extends String, ? extends Path> cacheEvent) {
+        logger.info("Expired file cache event: {} | key: {}", cacheEvent.getType(), cacheEvent.getKey());
+        var path = cacheEvent.getOldValue();
+        try {
+            var deleted = Files.deleteIfExists(path);
+            if(deleted) {
+                logger.info("Deleted expired file {} successfully", path);
+            } else {
+                logger.warn("Did NOT delete expired file {}", path);
+            }
+        } catch (IOException e) {
+            logger.error("Failed to delete expired file", e);
+        }
+    }
+}

server/src/main/java/org/eclipse/openvsx/cache/FilesCacheKeyGenerator.java

@@ -0,0 +1,47 @@
+/** ******************************************************************************
+ * Copyright (c) 2024 Precies. Software OU and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.cache;
+
+import org.eclipse.openvsx.adapter.WebResourceService;
+import org.eclipse.openvsx.entities.FileResource;
+import org.eclipse.openvsx.storage.IStorageService;
+import org.eclipse.openvsx.util.UrlUtil;
+import org.springframework.cache.interceptor.KeyGenerator;
+import org.springframework.stereotype.Component;
+
+import java.lang.reflect.Method;
+
+@Component
+public class FilesCacheKeyGenerator implements KeyGenerator {
+    @Override
+    public Object generate(Object target, Method method, Object... params) {
+        if(target instanceof WebResourceService) {
+            var namespace = (String) params[0];
+            var extension = (String) params[1];
+            var targetPlatform = (String) params[2];
+            var version = (String) params[3];
+            var name = (String) params[4];
+            return generate(namespace, extension, targetPlatform, version, name);
+        }
+        if(target instanceof IStorageService) {
+            var resource = (FileResource) params[0];
+            var extVersion = resource.getExtension();
+            var extension = extVersion.getExtension();
+            var namespace = extension.getNamespace();
+            return generate(namespace.getName(), extension.getName(), extVersion.getTargetPlatform(), extVersion.getVersion(), resource.getName());
+        }
+
+        throw new UnsupportedOperationException();
+    }
+
+    private String generate(String namespace, String extension, String targetPlatform, String version, String name) {
+        return UrlUtil.createApiFileUrl("", namespace, extension, targetPlatform, version, name);
+    }
+}