From 79cf222da8421e709b59668a418bb11c12a33f2a Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Sat, 20 Apr 2024 10:10:21 +0300 Subject: [PATCH] Get file urls in one query Find latest versions for user --- .../eclipse/openvsx/LocalRegistryService.java | 10 +- .../java/org/eclipse/openvsx/UserAPI.java | 8 +- .../eclipse/openvsx/admin/AdminService.java | 9 +- .../ExtensionVersionJooqRepository.java | 178 +++++++++++++++--- .../repositories/RepositoryService.java | 4 + .../openvsx/storage/StorageUtilService.java | 6 - .../eclipse/openvsx/admin/AdminAPITest.java | 3 +- .../RepositoryServiceSmokeTest.java | 1 + 8 files changed, 170 insertions(+), 49 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index c7e5be351..970733eec 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -149,6 +149,8 @@ public VersionReferencesJson getVersionReferences(String namespace, String exten ? repositories.findActiveVersionsSorted(namespace, extension, pageRequest) : repositories.findActiveVersionsSorted(namespace, extension, targetPlatform, pageRequest); + var fileUrls = storageUtil.getFileUrls(page.getContent(), UrlUtil.getBaseUrl(), withFileTypes(DOWNLOAD)); + var json = new VersionReferencesJson(); json.offset = (int) page.getPageable().getOffset(); json.totalSize = (int) page.getTotalElements(); @@ -159,7 +161,7 @@ public VersionReferencesJson getVersionReferences(String namespace, String exten versionRef.targetPlatform = extVersion.getTargetPlatform(); versionRef.engines = extVersion.getEnginesMap(); versionRef.url = UrlUtil.createApiVersionUrl(UrlUtil.getBaseUrl(), extVersion); - versionRef.files = storageUtil.getFileUrls(extVersion, UrlUtil.getBaseUrl(), withFileTypes(DOWNLOAD)); + versionRef.files = fileUrls.get(extVersion.getId()); if(versionRef.files.containsKey(DOWNLOAD_SIG)) { versionRef.files.put(PUBLIC_KEY, UrlUtil.getPublicKeyUrl(extVersion)); } @@ -396,11 +398,13 @@ public NamespaceDetailsJson getNamespaceDetails(String namespaceName) { : null; var serverUrl = UrlUtil.getBaseUrl(); - json.extensions = repositories.findLatestVersions(namespace).stream() + var extVersions = repositories.findLatestVersions(namespace); + var fileUrls = storageUtil.getFileUrls(extVersions, serverUrl, withFileTypes(DOWNLOAD, ICON)); + json.extensions = extVersions.stream() .map(extVersion -> { var entry = extVersion.toSearchEntryJson(); entry.url = createApiUrl(serverUrl, "api", entry.namespace, entry.name); - entry.files = storageUtil.getFileUrls(extVersion, serverUrl, withFileTypes(DOWNLOAD, ICON)); + entry.files = fileUrls.get(extVersion.getId()); if(entry.files.containsKey(DOWNLOAD_SIG)) { entry.files.put(PUBLIC_KEY, UrlUtil.getPublicKeyUrl(extVersion)); } diff --git a/server/src/main/java/org/eclipse/openvsx/UserAPI.java b/server/src/main/java/org/eclipse/openvsx/UserAPI.java index 55b1ed562..6fa7fe2e0 100644 --- a/server/src/main/java/org/eclipse/openvsx/UserAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/UserAPI.java @@ -187,15 +187,15 @@ public List getOwnExtensions() { throw new ResponseStatusException(HttpStatus.FORBIDDEN); } + var extVersions = repositories.findLatestVersions(user); var types = new String[]{ DOWNLOAD, MANIFEST, ICON, README, LICENSE, CHANGELOG, VSIXMANIFEST }; - return repositories.findExtensions(user) - .map(e -> repositories.findLatestVersion(e, null, false, false)) + var fileUrls = storageUtil.getFileUrls(extVersions, UrlUtil.getBaseUrl(), types); + return extVersions.stream() .map(latest -> { var json = latest.toExtensionJson(); json.preview = latest.isPreview(); json.active = latest.getExtension().isActive(); - json.files = storageUtil.getFileUrls(latest, UrlUtil.getBaseUrl(), types); - + json.files = fileUrls.get(latest.getId()); return json; }) .toList(); diff --git a/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java b/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java index a4902c97f..06ac5dd9d 100644 --- a/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java +++ b/server/src/main/java/org/eclipse/openvsx/admin/AdminService.java @@ -263,14 +263,15 @@ public UserPublishInfoJson getUserPublishInfo(String provider, String loginName) userPublishInfo.user = user.toUserJson(); eclipse.enrichUserJson(userPublishInfo.user, user); userPublishInfo.activeAccessTokenNum = (int) repositories.countActiveAccessTokens(user); - userPublishInfo.extensions = repositories.findExtensions(user).stream() - .map(e -> repositories.findLatestVersion(e, null, false, false)) + var extVersions = repositories.findLatestVersions(user); + var types = new String[]{DOWNLOAD, MANIFEST, ICON, README, LICENSE, CHANGELOG, VSIXMANIFEST}; + var fileUrls = storageUtil.getFileUrls(extVersions, UrlUtil.getBaseUrl(), types); + userPublishInfo.extensions = extVersions.stream() .map(latest -> { var json = latest.toExtensionJson(); json.preview = latest.isPreview(); json.active = latest.getExtension().isActive(); - json.files = storageUtil.getFileUrls(latest, UrlUtil.getBaseUrl(), - DOWNLOAD, MANIFEST, ICON, README, LICENSE, CHANGELOG, VSIXMANIFEST); + json.files = fileUrls.get(latest.getId()); return json; }) diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java index 36e2b7a9a..820cd19a8 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java @@ -449,16 +449,24 @@ private List fetch(SelectQuery query) { } private ExtensionVersion toExtensionVersionFull(Record record) { - return toExtensionVersionFull(record, null); + return toExtensionVersionFull(record, null, null); } - private ExtensionVersion toExtensionVersionFull(Record record, Extension extension) { - var extVersion = toExtensionVersionCommon(record, extension); - extVersion.setLicense(record.get(EXTENSION_VERSION.LICENSE)); - extVersion.setHomepage(record.get(EXTENSION_VERSION.HOMEPAGE)); - extVersion.setBugs(record.get(EXTENSION_VERSION.BUGS)); - extVersion.setMarkdown(record.get(EXTENSION_VERSION.MARKDOWN)); - extVersion.setQna(record.get(EXTENSION_VERSION.QNA)); + private ExtensionVersion toExtensionVersionFull( + Record record, + Extension extension, + FieldMapper extensionVersionMapper + ) { + if(extensionVersionMapper == null) { + extensionVersionMapper = new DefaultFieldMapper(); + } + + var extVersion = toExtensionVersionCommon(record, extension, extensionVersionMapper); + extVersion.setLicense(record.get(extensionVersionMapper.map(EXTENSION_VERSION.LICENSE))); + extVersion.setHomepage(record.get(extensionVersionMapper.map(EXTENSION_VERSION.HOMEPAGE))); + extVersion.setBugs(record.get(extensionVersionMapper.map(EXTENSION_VERSION.BUGS))); + extVersion.setMarkdown(record.get(extensionVersionMapper.map(EXTENSION_VERSION.MARKDOWN))); + extVersion.setQna(record.get(extensionVersionMapper.map(EXTENSION_VERSION.QNA))); if(extension == null) { var newExtension = extVersion.getExtension(); @@ -490,34 +498,38 @@ private ExtensionVersion toExtensionVersionFull(Record record, Extension extensi } private ExtensionVersion toExtensionVersion(Record record) { - var extVersion = toExtensionVersionCommon(record, null); + var extVersion = toExtensionVersionCommon(record, null, new DefaultFieldMapper()); extVersion.setType(ExtensionVersion.Type.MINIMAL); return extVersion; } - private ExtensionVersion toExtensionVersionCommon(Record record, Extension extension) { + private ExtensionVersion toExtensionVersionCommon( + Record record, + Extension extension, + FieldMapper extensionVersionMapper + ) { var converter = new ListOfStringConverter(); var extVersion = new ExtensionVersion(); - extVersion.setId(record.get(EXTENSION_VERSION.ID)); - extVersion.setVersion(record.get(EXTENSION_VERSION.VERSION)); - extVersion.setTargetPlatform(record.get(EXTENSION_VERSION.TARGET_PLATFORM)); - extVersion.setPreview(record.get(EXTENSION_VERSION.PREVIEW)); - extVersion.setPreRelease(record.get(EXTENSION_VERSION.PRE_RELEASE)); - extVersion.setTimestamp(record.get(EXTENSION_VERSION.TIMESTAMP)); - extVersion.setDisplayName(record.get(EXTENSION_VERSION.DISPLAY_NAME)); - extVersion.setDescription(record.get(EXTENSION_VERSION.DESCRIPTION)); - extVersion.setEngines(toList(record.get(EXTENSION_VERSION.ENGINES), converter)); - extVersion.setCategories(toList(record.get(EXTENSION_VERSION.CATEGORIES), converter)); - extVersion.setTags(toList(record.get(EXTENSION_VERSION.TAGS), converter)); - extVersion.setExtensionKind(toList(record.get(EXTENSION_VERSION.EXTENSION_KIND), converter)); - extVersion.setRepository(record.get(EXTENSION_VERSION.REPOSITORY)); - extVersion.setGalleryColor(record.get(EXTENSION_VERSION.GALLERY_COLOR)); - extVersion.setGalleryTheme(record.get(EXTENSION_VERSION.GALLERY_THEME)); - extVersion.setLocalizedLanguages(toList(record.get(EXTENSION_VERSION.LOCALIZED_LANGUAGES), converter)); - extVersion.setDependencies(toList(record.get(EXTENSION_VERSION.DEPENDENCIES), converter)); - extVersion.setBundledExtensions(toList(record.get(EXTENSION_VERSION.BUNDLED_EXTENSIONS), converter)); - extVersion.setSponsorLink(record.get(EXTENSION_VERSION.SPONSOR_LINK)); + extVersion.setId(record.get(extensionVersionMapper.map(EXTENSION_VERSION.ID))); + extVersion.setVersion(record.get(extensionVersionMapper.map(EXTENSION_VERSION.VERSION))); + extVersion.setTargetPlatform(record.get(extensionVersionMapper.map(EXTENSION_VERSION.TARGET_PLATFORM))); + extVersion.setPreview(record.get(extensionVersionMapper.map(EXTENSION_VERSION.PREVIEW))); + extVersion.setPreRelease(record.get(extensionVersionMapper.map(EXTENSION_VERSION.PRE_RELEASE))); + extVersion.setTimestamp(record.get(extensionVersionMapper.map(EXTENSION_VERSION.TIMESTAMP))); + extVersion.setDisplayName(record.get(extensionVersionMapper.map(EXTENSION_VERSION.DISPLAY_NAME))); + extVersion.setDescription(record.get(extensionVersionMapper.map(EXTENSION_VERSION.DESCRIPTION))); + extVersion.setEngines(toList(record.get(extensionVersionMapper.map(EXTENSION_VERSION.ENGINES)), converter)); + extVersion.setCategories(toList(record.get(extensionVersionMapper.map(EXTENSION_VERSION.CATEGORIES)), converter)); + extVersion.setTags(toList(record.get(extensionVersionMapper.map(EXTENSION_VERSION.TAGS)), converter)); + extVersion.setExtensionKind(toList(record.get(extensionVersionMapper.map(EXTENSION_VERSION.EXTENSION_KIND)), converter)); + extVersion.setRepository(record.get(extensionVersionMapper.map(EXTENSION_VERSION.REPOSITORY))); + extVersion.setGalleryColor(record.get(extensionVersionMapper.map(EXTENSION_VERSION.GALLERY_COLOR))); + extVersion.setGalleryTheme(record.get(extensionVersionMapper.map(EXTENSION_VERSION.GALLERY_THEME))); + extVersion.setLocalizedLanguages(toList(record.get(extensionVersionMapper.map(EXTENSION_VERSION.LOCALIZED_LANGUAGES)), converter)); + extVersion.setDependencies(toList(record.get(extensionVersionMapper.map(EXTENSION_VERSION.DEPENDENCIES)), converter)); + extVersion.setBundledExtensions(toList(record.get(extensionVersionMapper.map(EXTENSION_VERSION.BUNDLED_EXTENSIONS)), converter)); + extVersion.setSponsorLink(record.get(extensionVersionMapper.map(EXTENSION_VERSION.SPONSOR_LINK))); if(extension == null) { var namespace = new Namespace(); @@ -653,7 +665,7 @@ public ExtensionVersion findLatest( query.addJoin(USER_DATA, USER_DATA.ID.eq(PERSONAL_ACCESS_TOKEN.USER_DATA)); query.addJoin(SIGNATURE_KEY_PAIR, JoinType.LEFT_OUTER_JOIN, SIGNATURE_KEY_PAIR.ID.eq(EXTENSION_VERSION.SIGNATURE_KEY_PAIR_ID)); query.addConditions(EXTENSION_VERSION.EXTENSION_ID.eq(extension.getId())); - return query.fetchOne((record) -> toExtensionVersionFull(record, extension)); + return query.fetchOne((record) -> toExtensionVersionFull(record, extension, null)); } public List findLatest(Namespace namespace) { @@ -716,6 +728,96 @@ public List findLatest(Namespace namespace) { }); } + public List findLatest(UserData user) { + var latestQuery = findLatestQuery(null, false, false); + latestQuery.addSelect( + EXTENSION_VERSION.ID, + EXTENSION_VERSION.VERSION, + EXTENSION_VERSION.TARGET_PLATFORM, + EXTENSION_VERSION.PREVIEW, + EXTENSION_VERSION.PRE_RELEASE, + EXTENSION_VERSION.TIMESTAMP, + EXTENSION_VERSION.DISPLAY_NAME, + EXTENSION_VERSION.DESCRIPTION, + EXTENSION_VERSION.ENGINES, + EXTENSION_VERSION.CATEGORIES, + EXTENSION_VERSION.TAGS, + EXTENSION_VERSION.EXTENSION_KIND, + EXTENSION_VERSION.LICENSE, + EXTENSION_VERSION.HOMEPAGE, + EXTENSION_VERSION.REPOSITORY, + EXTENSION_VERSION.SPONSOR_LINK, + EXTENSION_VERSION.BUGS, + EXTENSION_VERSION.MARKDOWN, + EXTENSION_VERSION.GALLERY_COLOR, + EXTENSION_VERSION.GALLERY_THEME, + EXTENSION_VERSION.LOCALIZED_LANGUAGES, + EXTENSION_VERSION.QNA, + EXTENSION_VERSION.DEPENDENCIES, + EXTENSION_VERSION.BUNDLED_EXTENSIONS, + EXTENSION_VERSION.SIGNATURE_KEY_PAIR_ID, + EXTENSION_VERSION.PUBLISHED_WITH_ID + ); + latestQuery.addConditions(EXTENSION_VERSION.EXTENSION_ID.eq(EXTENSION.ID)); + var latest = latestQuery.asTable(); + + var query = dsl.selectQuery(); + query.addSelect( + NAMESPACE.ID, + NAMESPACE.NAME, + NAMESPACE.DISPLAY_NAME, + NAMESPACE.PUBLIC_ID, + EXTENSION.ID, + EXTENSION.NAME, + EXTENSION.PUBLIC_ID, + EXTENSION.AVERAGE_RATING, + EXTENSION.REVIEW_COUNT, + EXTENSION.DOWNLOAD_COUNT, + EXTENSION.PUBLISHED_DATE, + EXTENSION.LAST_UPDATED_DATE, + latest.field(EXTENSION_VERSION.ID), + latest.field(EXTENSION_VERSION.VERSION), + latest.field(EXTENSION_VERSION.TARGET_PLATFORM), + latest.field(EXTENSION_VERSION.PREVIEW), + latest.field(EXTENSION_VERSION.PRE_RELEASE), + latest.field(EXTENSION_VERSION.TIMESTAMP), + latest.field(EXTENSION_VERSION.DISPLAY_NAME), + latest.field(EXTENSION_VERSION.DESCRIPTION), + latest.field(EXTENSION_VERSION.ENGINES), + latest.field(EXTENSION_VERSION.CATEGORIES), + latest.field(EXTENSION_VERSION.TAGS), + latest.field(EXTENSION_VERSION.EXTENSION_KIND), + latest.field(EXTENSION_VERSION.LICENSE), + latest.field(EXTENSION_VERSION.HOMEPAGE), + latest.field(EXTENSION_VERSION.REPOSITORY), + latest.field(EXTENSION_VERSION.SPONSOR_LINK), + latest.field(EXTENSION_VERSION.BUGS), + latest.field(EXTENSION_VERSION.MARKDOWN), + latest.field(EXTENSION_VERSION.GALLERY_COLOR), + latest.field(EXTENSION_VERSION.GALLERY_THEME), + latest.field(EXTENSION_VERSION.LOCALIZED_LANGUAGES), + latest.field(EXTENSION_VERSION.QNA), + latest.field(EXTENSION_VERSION.DEPENDENCIES), + latest.field(EXTENSION_VERSION.BUNDLED_EXTENSIONS), + SIGNATURE_KEY_PAIR.PUBLIC_ID, + USER_DATA.ID, + USER_DATA.ROLE, + USER_DATA.LOGIN_NAME, + USER_DATA.FULL_NAME, + USER_DATA.AVATAR_URL, + USER_DATA.PROVIDER_URL, + USER_DATA.PROVIDER + ); + query.addFrom(NAMESPACE); + query.addJoin(EXTENSION, EXTENSION.NAMESPACE_ID.eq(NAMESPACE.ID)); + query.addJoin(latest, JoinType.CROSS_APPLY, DSL.condition(true)); + query.addJoin(SIGNATURE_KEY_PAIR, JoinType.LEFT_OUTER_JOIN, SIGNATURE_KEY_PAIR.ID.eq(latest.field(EXTENSION_VERSION.SIGNATURE_KEY_PAIR_ID))); + query.addJoin(PERSONAL_ACCESS_TOKEN, JoinType.LEFT_OUTER_JOIN, PERSONAL_ACCESS_TOKEN.ID.eq(latest.field(EXTENSION_VERSION.PUBLISHED_WITH_ID))); + query.addJoin(USER_DATA, USER_DATA.ID.eq(PERSONAL_ACCESS_TOKEN.USER_DATA)); + query.addConditions(PERSONAL_ACCESS_TOKEN.USER_DATA.eq(user.getId())); + return query.fetch(record -> toExtensionVersionFull(record, null, new TableFieldMapper(latest))); + } + public ExtensionVersion findLatestForAllUrls( Extension extension, String targetPlatform, @@ -849,4 +951,20 @@ public List findDistinctTargetPlatforms(Extension extension) { .and(EXTENSION_VERSION.ACTIVE.eq(true)) .fetch(EXTENSION_VERSION.TARGET_PLATFORM); } + + private interface FieldMapper { + Field map(Field field); + } + + private record TableFieldMapper(Table table) implements FieldMapper { + public Field map(Field field) { + return table.field(field); + } + } + + private static class DefaultFieldMapper implements FieldMapper { + public Field map(Field field) { + return field; + } + } } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index 5ebd1655f..be6785fb5 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -552,6 +552,10 @@ public List findLatestVersions(Namespace namespace) { return extensionVersionJooqRepo.findLatest(namespace); } + public List findLatestVersions(UserData user) { + return extensionVersionJooqRepo.findLatest(user); + } + public List findExtensionTargetPlatforms(Extension extension) { return extensionVersionJooqRepo.findDistinctTargetPlatforms(extension); } diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index 26dcaf6ff..8431609e3 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -251,12 +251,6 @@ private String getFileUrl(String name, ExtensionVersion extVersion, String serve return UrlUtil.createApiFileUrl(serverUrl, extVersion, name); } - @Observed - public Map getFileUrls(ExtensionVersion extVersion, String serverUrl, String... types) { - var fileUrls = getFileUrls(List.of(extVersion), serverUrl, types); - return fileUrls.get(extVersion.getId()); - } - /** * Returns URLs for the given file types as a map of ExtensionVersion.id by a map of type by file URL, to be used in JSON response data. */ diff --git a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java index 131eb7410..038772d8d 100644 --- a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java @@ -483,8 +483,7 @@ public void testGetUserPublishInfo() throws Exception { Mockito.when(repositories.findUserByLoginName("github", "test")).thenReturn(user); Mockito.when(repositories.countActiveAccessTokens(user)).thenReturn(1L); - Mockito.when(repositories.findExtensions(user)) - .thenReturn(Streamable.of(versions.get(0).getExtension())); + Mockito.when(repositories.findLatestVersions(user)).thenReturn(versions); mockMvc.perform(get("/admin/publisher/{provider}/{loginName}", "github", "test") .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index c0ef6df3e..e1df1d77a 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -201,6 +201,7 @@ void testExecuteQueries() { () -> repositories.findLatestVersionForAllUrls(extension, "targetPlatform", false, false), () -> repositories.findLatestVersion(extension, "targetPlatform", false, false), () -> repositories.findLatestVersions(namespace), + () -> repositories.findLatestVersions(userData), () -> repositories.findExtensionTargetPlatforms(extension) );