diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f7d099..54cb96e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Changed - Used virtual threads when available +- [#26](https://github.com/devatherock/artifactory-badge/issues/26): Fetched child folder information in parallel ## [3.0.0] - 2024-10-16 ### Added diff --git a/src/gatling/resources/wiremock/responses/versions-response.json b/src/gatling/resources/wiremock/responses/versions-response.json index cc60d6b..7eb5ff5 100644 --- a/src/gatling/resources/wiremock/responses/versions-response.json +++ b/src/gatling/resources/wiremock/responses/versions-response.json @@ -8,10 +8,30 @@ "uri": "/1.1.2-rc.1", "folder": true }, + { + "uri": "/1.2.0", + "folder": true + }, + { + "uri": "/1.3.0", + "folder": true + }, + { + "uri": "/1.4.0", + "folder": true + }, + { + "uri": "/1.5.0", + "folder": true + }, { "uri": "/latest", "folder": true }, + { + "uri": "/2.0.0", + "folder": true + }, { "uri": "/_uploads", "folder": true diff --git a/src/main/java/io/github/devatherock/artifactory/config/AppConfig.java b/src/main/java/io/github/devatherock/artifactory/config/AppConfig.java index 6560bb8..e00fde6 100644 --- a/src/main/java/io/github/devatherock/artifactory/config/AppConfig.java +++ b/src/main/java/io/github/devatherock/artifactory/config/AppConfig.java @@ -1,9 +1,15 @@ package io.github.devatherock.artifactory.config; +import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; + +import io.github.devatherock.artifactory.util.ParallelProcessor; + import io.micronaut.context.annotation.Factory; import io.micronaut.http.client.BlockingHttpClient; import io.micronaut.http.client.HttpClient; import io.micronaut.http.client.annotation.Client; +import jakarta.inject.Named; import jakarta.inject.Singleton; /** @@ -22,4 +28,13 @@ public class AppConfig { public BlockingHttpClient httpClient(@Client HttpClient httpClient) { return httpClient.toBlocking(); } + + /** + * @param appProperties miscellaneous application properties + * @return a parallel processor bean + */ + @Singleton + public ParallelProcessor parallelProcessor(@Named("blocking") Executor executor, AppProperties appProperties) { + return new ParallelProcessor(executor, new Semaphore(appProperties.getParallelism())); + } } diff --git a/src/main/java/io/github/devatherock/artifactory/config/AppProperties.java b/src/main/java/io/github/devatherock/artifactory/config/AppProperties.java new file mode 100644 index 0000000..23feb4f --- /dev/null +++ b/src/main/java/io/github/devatherock/artifactory/config/AppProperties.java @@ -0,0 +1,16 @@ +package io.github.devatherock.artifactory.config; + +import io.micronaut.context.annotation.ConfigurationProperties; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@ConfigurationProperties("artifactory.badge") +public class AppProperties { + /** + * Amount of parallelization to use when fetching details about versions of an + * image + */ + private int parallelism = 5; +} diff --git a/src/main/java/io/github/devatherock/artifactory/service/DockerBadgeService.java b/src/main/java/io/github/devatherock/artifactory/service/DockerBadgeService.java index 7251c84..22254fb 100644 --- a/src/main/java/io/github/devatherock/artifactory/service/DockerBadgeService.java +++ b/src/main/java/io/github/devatherock/artifactory/service/DockerBadgeService.java @@ -1,6 +1,8 @@ package io.github.devatherock.artifactory.service; import java.time.Instant; +import java.util.List; +import java.util.function.Supplier; import java.util.regex.Pattern; import io.github.devatherock.artifactory.config.ArtifactoryProperties; @@ -10,6 +12,7 @@ import io.github.devatherock.artifactory.entities.DockerLayer; import io.github.devatherock.artifactory.entities.DockerManifest; import io.github.devatherock.artifactory.util.BadgeGenerator; +import io.github.devatherock.artifactory.util.ParallelProcessor; import io.micronaut.cache.annotation.Cacheable; import io.micronaut.core.annotation.Blocking; @@ -53,6 +56,7 @@ public class DockerBadgeService { private final BlockingHttpClient artifactoryClient; private final BadgeGenerator badgeGenerator; + private final ParallelProcessor parallelProcessor; private final ArtifactoryProperties artifactoryConfig; @Cacheable(cacheNames = "size-cache") @@ -93,15 +97,19 @@ public String getPullsCountBadge(String packageName, String badgeLabel) { if (null != folderInfo && CollectionUtils.isNotEmpty(folderInfo.getChildren())) { long downloadCount = 0; - for (ArtifactoryFolderElement child : folderInfo.getChildren()) { - if (isTag(child)) { - ArtifactoryFileStats fileStats = getManifestStats(packageName, child.getUri()); + List> statsSuppliers = folderInfo.getChildren() + .stream() + .filter(this::isTag) + .map(child -> (Supplier) () -> getManifestStats(packageName, child.getUri())) + .toList(); + List statsList = parallelProcessor.parallelProcess(statsSuppliers); - if (null != fileStats) { - downloadCount += fileStats.getDownloadCount(); - } + for (ArtifactoryFileStats fileStats : statsList) { + if (null != fileStats) { + downloadCount += fileStats.getDownloadCount(); } } + LOGGER.info("Download count of {}: {}", packageName, downloadCount); return badgeGenerator.generateBadge(badgeLabel, DockerBadgeServiceHelper.formatDownloadCount(downloadCount)); @@ -126,9 +134,9 @@ public String getLatestVersionBadge(String packageName, String badgeLabel, Strin if (null != folderInfo && CollectionUtils.isNotEmpty(folderInfo.getChildren())) { ArtifactoryFolderInfo latestVersion = null; - for (ArtifactoryFolderElement child : folderInfo.getChildren()) { - if (isTag(child)) { - if (SORT_TYPE_SEMVER.equals(sortType)) { + if (SORT_TYPE_SEMVER.equals(sortType)) { + for (ArtifactoryFolderElement child : folderInfo.getChildren()) { + if (isTag(child)) { // Substring to remove the leading slash String currentVersion = child.getUri().substring(1); @@ -145,20 +153,28 @@ public String getLatestVersionBadge(String packageName, String badgeLabel, Strin } } } - } else { - // Find the modified time of each subfolder - each subfolder corresponds to a - // tag - ArtifactoryFolderInfo currentVersion = getArtifactoryFolderInfo(packageName + child.getUri()); - - if (null == latestVersion || (null != currentVersion - && Instant - .from(artifactoryConfig.getDateParser().parse(currentVersion.getLastModified())) - .compareTo( - Instant.from( - artifactoryConfig.getDateParser() - .parse(latestVersion.getLastModified()))) > 0)) { - latestVersion = currentVersion; - } + } + } + } else { + // Find the modified time of each subfolder - each subfolder corresponds to a + // tag + List> versionSuppliers = folderInfo.getChildren() + .stream() + .filter(this::isTag) + .map(child -> (Supplier) () -> getArtifactoryFolderInfo( + packageName + child.getUri())) + .toList(); + List versions = parallelProcessor.parallelProcess(versionSuppliers); + + for (ArtifactoryFolderInfo currentVersion : versions) { + if (null == latestVersion || (null != currentVersion + && Instant + .from(artifactoryConfig.getDateParser().parse(currentVersion.getLastModified())) + .compareTo( + Instant.from( + artifactoryConfig.getDateParser() + .parse(latestVersion.getLastModified()))) > 0)) { + latestVersion = currentVersion; } } } @@ -255,7 +271,7 @@ private String generateNotFoundBadge(String badgeLabel) { /** * Checks if the supplied artifactory folder content corresponds to a tag - * + * * @param child * @return a flag */ diff --git a/src/main/java/io/github/devatherock/artifactory/util/ParallelProcessor.java b/src/main/java/io/github/devatherock/artifactory/util/ParallelProcessor.java new file mode 100644 index 0000000..4970c82 --- /dev/null +++ b/src/main/java/io/github/devatherock/artifactory/util/ParallelProcessor.java @@ -0,0 +1,54 @@ +package io.github.devatherock.artifactory.util; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class ParallelProcessor { + private final Executor executor; + private final Semaphore semaphore; // To limit parallelism in case of outbound API calls + + public List parallelProcess(List> suppliers) { + Map result = new ConcurrentHashMap<>(); + CountDownLatch countDownLatch = new CountDownLatch(suppliers.size()); + + try { + for (int index = 0; index < suppliers.size(); index++) { + int currentIndex = index; + + semaphore.acquire(); + executor.execute(() -> { + try { + LOGGER.trace("In parallelProcess. Index: {}", currentIndex); + result.put(currentIndex, suppliers.get(currentIndex).get()); + } finally { + countDownLatch.countDown(); + } + }); + semaphore.release(); + } + + countDownLatch.await(); + } catch (InterruptedException exception) { + LOGGER.warn("Exception during parallel processing", exception); + } + + return result.entrySet() + .stream() + .sorted(Entry.comparingByKey(Comparator.naturalOrder())) + .map(Entry::getValue) + .collect(Collectors.toList()); + } +} diff --git a/src/test/groovy/io/github/devatherock/artifactory/service/DockerBadgeServiceSpec.groovy b/src/test/groovy/io/github/devatherock/artifactory/service/DockerBadgeServiceSpec.groovy index 0a1a750..2ac95c1 100644 --- a/src/test/groovy/io/github/devatherock/artifactory/service/DockerBadgeServiceSpec.groovy +++ b/src/test/groovy/io/github/devatherock/artifactory/service/DockerBadgeServiceSpec.groovy @@ -3,8 +3,12 @@ package io.github.devatherock.artifactory.service import static com.github.tomakehurst.wiremock.client.WireMock.equalTo import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo +import java.util.concurrent.Executors +import java.util.concurrent.Semaphore + import io.github.devatherock.artifactory.config.ArtifactoryProperties import io.github.devatherock.artifactory.util.BadgeGenerator +import io.github.devatherock.artifactory.util.ParallelProcessor import io.github.devatherock.test.TestUtil import com.github.tomakehurst.wiremock.WireMockServer @@ -41,10 +45,12 @@ class DockerBadgeServiceSpec extends Specification { BlockingHttpClient httpClient = HttpClient.create(new URL('http://localhost:8081')).toBlocking() BadgeGenerator badgeGenerator = Mock() ArtifactoryProperties config = new ArtifactoryProperties(url: 'http://localhost:8081', apiKey: 'dummyKey') + ParallelProcessor parallelProcessor = + new ParallelProcessor(Executors.newSingleThreadExecutor(), new Semaphore(1)) void setup() { config.init() - dockerBadgeService = new DockerBadgeService(httpClient, badgeGenerator, config) + dockerBadgeService = new DockerBadgeService(httpClient, badgeGenerator, parallelProcessor, config) } void 'test get image size badge - manifest not found'() {