Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MRESOLVER-587] Memory usage improvements #536

Merged
merged 3 commits into from
Aug 2, 2024
Merged

[MRESOLVER-587] Memory usage improvements #536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3348e42
[MRESOLVER-587] Memory usage improvements
cstamas Jul 13, 2024
779a630
Make newly added interning configurable (default: false to both, as b…
cstamas Aug 2, 2024
b4ebda0
Make default false/true
cstamas Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 89 additions & 9 deletions maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,23 @@ public final class DataPool {

private static final String CONFIG_PROP_COLLECTOR_POOL_DESCRIPTOR = "aether.dependencyCollector.pool.descriptor";

private static final String CONFIG_PROP_COLLECTOR_POOL_DEPENDENCY_LISTS =
"aether.dependencyCollector.pool.dependencyLists";

private static final String CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_DEPENDENCIES =
"aether.dependencyCollector.pool.internArtifactDescriptorDependencies";

private static final String CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES =
"aether.dependencyCollector.pool.internArtifactDescriptorManagedDependencies";

private static final String ARTIFACT_POOL = DataPool.class.getName() + "$Artifact";

private static final String DEPENDENCY_POOL = DataPool.class.getName() + "$Dependency";

private static final String DESCRIPTORS = DataPool.class.getName() + "$Descriptors";

private static final String DEPENDENCY_LISTS_POOL = DataPool.class.getName() + "$DependencyLists";

public static final ArtifactDescriptorResult NO_DESCRIPTOR =
new ArtifactDescriptorResult(new ArtifactDescriptorRequest());

Expand All @@ -80,7 +91,12 @@ public final class DataPool {
/**
* Descriptor interning pool, lives across session (if session carries non-null {@link RepositoryCache}).
*/
private final InternPool<Object, Descriptor> descriptors;
private final InternPool<DescriptorKey, Descriptor> descriptors;

/**
* {@link Dependency} list interning pool, lives across session (if session carries non-null {@link RepositoryCache}).
*/
private final InternPool<List<Dependency>, List<Dependency>> dependencyLists;

/**
* Constraint cache, lives during single collection invocation (same as this DataPool instance).
Expand All @@ -92,17 +108,29 @@ public final class DataPool {
*/
private final ConcurrentHashMap<Object, List<DependencyNode>> nodes;

private final boolean internArtifactDescriptorDependencies;

private final boolean internArtifactDescriptorManagedDependencies;

@SuppressWarnings("unchecked")
public DataPool(RepositorySystemSession session) {
final RepositoryCache cache = session.getCache();

internArtifactDescriptorDependencies = ConfigUtils.getBoolean(
session, false, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_DEPENDENCIES);
internArtifactDescriptorManagedDependencies = ConfigUtils.getBoolean(
session, true, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES);

InternPool<Artifact, Artifact> artifactsPool = null;
InternPool<Dependency, Dependency> dependenciesPool = null;
InternPool<Object, Descriptor> descriptorsPool = null;
InternPool<DescriptorKey, Descriptor> descriptorsPool = null;
InternPool<List<Dependency>, List<Dependency>> dependencyListsPool = null;
if (cache != null) {
artifactsPool = (InternPool<Artifact, Artifact>) cache.get(session, ARTIFACT_POOL);
dependenciesPool = (InternPool<Dependency, Dependency>) cache.get(session, DEPENDENCY_POOL);
descriptorsPool = (InternPool<Object, Descriptor>) cache.get(session, DESCRIPTORS);
descriptorsPool = (InternPool<DescriptorKey, Descriptor>) cache.get(session, DESCRIPTORS);
dependencyListsPool =
(InternPool<List<Dependency>, List<Dependency>>) cache.get(session, DEPENDENCY_LISTS_POOL);
}

if (artifactsPool == null) {
Expand Down Expand Up @@ -132,9 +160,20 @@ public DataPool(RepositorySystemSession session) {
}
}

if (dependencyListsPool == null) {
String dependencyListsPoolType =
ConfigUtils.getString(session, HARD, CONFIG_PROP_COLLECTOR_POOL_DEPENDENCY_LISTS);

dependencyListsPool = createPool(dependencyListsPoolType);
if (cache != null) {
cache.put(session, DEPENDENCY_LISTS_POOL, dependencyListsPool);
}
}

this.artifacts = artifactsPool;
this.dependencies = dependenciesPool;
this.descriptors = descriptorsPool;
this.dependencyLists = dependencyListsPool;

this.constraints = new ConcurrentHashMap<>(256);
this.nodes = new ConcurrentHashMap<>(256);
Expand All @@ -148,26 +187,36 @@ public Dependency intern(Dependency dependency) {
return dependencies.intern(dependency, dependency);
}

public Object toKey(ArtifactDescriptorRequest request) {
return request.getArtifact();
public DescriptorKey toKey(ArtifactDescriptorRequest request) {
return new DescriptorKey(request.getArtifact());
}

public ArtifactDescriptorResult getDescriptor(Object key, ArtifactDescriptorRequest request) {
public ArtifactDescriptorResult getDescriptor(DescriptorKey key, ArtifactDescriptorRequest request) {
Descriptor descriptor = descriptors.get(key);
if (descriptor != null) {
return descriptor.toResult(request);
}
return null;
}

public void putDescriptor(Object key, ArtifactDescriptorResult result) {
public void putDescriptor(DescriptorKey key, ArtifactDescriptorResult result) {
if (internArtifactDescriptorDependencies) {
result.setDependencies(intern(result.getDependencies()));
}
if (internArtifactDescriptorManagedDependencies) {
result.setManagedDependencies(intern(result.getManagedDependencies()));
}
descriptors.intern(key, new GoodDescriptor(result));
}

public void putDescriptor(Object key, ArtifactDescriptorException e) {
public void putDescriptor(DescriptorKey key, ArtifactDescriptorException e) {
descriptors.intern(key, BadDescriptor.INSTANCE);
}

private List<Dependency> intern(List<Dependency> dependencies) {
return dependencyLists.intern(dependencies, dependencies);
}

public Object toKey(VersionRangeRequest request) {
return new ConstraintKey(request);
}
Expand Down Expand Up @@ -202,8 +251,39 @@ public void putChildren(Object key, List<DependencyNode> children) {
nodes.put(key, children);
}

abstract static class Descriptor {
public static final class DescriptorKey {
private final Artifact artifact;
private final int hashCode;

private DescriptorKey(Artifact artifact) {
this.artifact = artifact;
this.hashCode = Objects.hashCode(artifact);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah sorry, I was used to more complex hashCode construction and kept this pattern (we avoid using Objects.hash as it allocates an array and can be problematic in some cases so we end up with traditional boilerplate hashCodes).

}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
DescriptorKey that = (DescriptorKey) o;
return Objects.equals(artifact, that.artifact);
}

@Override
public int hashCode() {
return hashCode;
}

@Override
public String toString() {
return getClass().getSimpleName() + "{" + "artifact='" + artifact + '\'' + '}';
}
}

abstract static class Descriptor {
public abstract ArtifactDescriptorResult toResult(ArtifactDescriptorRequest request);
}

Expand Down
2 changes: 1 addition & 1 deletion ...impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ private ArtifactDescriptorResult resolveCachedArtifactDescriptor(
RepositorySystemSession session,
DependencyProcessingContext context,
Results results) {
Object key = pool.toKey(descriptorRequest);
DataPool.DescriptorKey key = pool.toKey(descriptorRequest);
ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest);
if (descriptorResult == null) {
try {
Expand Down
2 changes: 1 addition & 1 deletion ...impl/src/main/java/org/eclipse/aether/internal/impl/collect/df/DfDependencyCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ private ArtifactDescriptorResult resolveCachedArtifactDescriptor(
Dependency d,
Results results,
Args args) {
Object key = pool.toKey(descriptorRequest);
DataPool.DescriptorKey key = pool.toKey(descriptorRequest);
ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest);
if (descriptorResult == null) {
try {
Expand Down
2 changes: 1 addition & 1 deletion maven-resolver-impl/src/test/java/org/eclipse/aether/internal/impl/collect/DataPoolTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public void testArtifactDescriptorCaching() {
result.addAlias(new DefaultArtifact("gid:alias:4"));

DataPool pool = newDataPool();
Object key = pool.toKey(request);
DataPool.DescriptorKey key = pool.toKey(request);
pool.putDescriptor(key, result);
ArtifactDescriptorResult cached = pool.getDescriptor(key, request);
assertNotNull(cached);
Expand Down