From db286f79f8f2ab0b0ae002a03f52293ef25116cb Mon Sep 17 00:00:00 2001 From: Geod24 Date: Thu, 2 Mar 2023 17:14:17 +0100 Subject: [PATCH] PackageManager: Store packages hierarchically, by version As explained in the changelog, this hierarchy makes much more sense than the current one, and will make it possible to read the version from the directory name. --- changelog/hierarchy.dd | 16 +++++ source/dub/packagemanager.d | 61 +++++++++++-------- test/dpath-variable.sh | 4 +- test/interactive-remove.sh | 18 +++--- .../{foo-1.0.0 => foo/1.0.0}/foo/dub.sdl | 0 test/version-spec.sh | 6 +- 6 files changed, 65 insertions(+), 40 deletions(-) create mode 100644 changelog/hierarchy.dd rename test/issue838-custom-cache-paths/cache/{foo-1.0.0 => foo/1.0.0}/foo/dub.sdl (100%) diff --git a/changelog/hierarchy.dd b/changelog/hierarchy.dd new file mode 100644 index 000000000..c087297a7 --- /dev/null +++ b/changelog/hierarchy.dd @@ -0,0 +1,16 @@ +The way packages are stored internally has changed + +Previous versions of dub stored packages in the following format: +`$CACHE_PATH/$PACKAGE_NAME-$PACKAGE_VERSION/$PACKAGE_NAME/` +Starting from this version, the format will be: +`$CACHE_PATH/$PACKAGE_NAME/$PACKAGE_VERSION/$PACKAGE_NAME`. + +Introducing a new level will help users quickly list what packages +they actually have installed, and reduce visibility of packages that +might update frequently. It will render various commands (e.g. `du`) +more useful, pave the way for a package GC function, and make manual +browsing easier. + +More importantly, it will allow future version of dub to infer the +version from the path to the package, removing the need to read (or edit) +the recipe file on every dub invocation. diff --git a/source/dub/packagemanager.d b/source/dub/packagemanager.d index 04c82e3e5..0300ba16b 100644 --- a/source/dub/packagemanager.d +++ b/source/dub/packagemanager.d @@ -1298,29 +1298,8 @@ private struct Location { if (!path.existsDirectory()) return; - logDebug("iterating dir %s", path.toNativeString()); - try foreach (pdir; iterateDirectory(path)) { - logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name); - if (!pdir.isDirectory) continue; - - // Old / flat directory structure, used in non-standard path - // Packages are stored in $ROOT/$SOMETHING/` - auto pack_path = path ~ (pdir.name ~ "/"); - auto packageFile = Package.findPackageFile(pack_path); - - // New (since 2015) managed structure: - // $ROOT/$NAME-$VERSION/$NAME - // This is the most common code path - if (mgr.isManagedPath(path) && packageFile.empty) { - foreach (subdir; iterateDirectory(path ~ (pdir.name ~ "/"))) - if (subdir.isDirectory && pdir.name.startsWith(subdir.name)) { - pack_path ~= subdir.name ~ "/"; - packageFile = Package.findPackageFile(pack_path); - break; - } - } - - if (packageFile.empty) continue; + void loadInternal (NativePath pack_path, NativePath packageFile) + { Package p; try { foreach (pp; existing_packages) @@ -1338,6 +1317,38 @@ private struct Location { logDiagnostic("Full error: %s", e.toString().sanitize()); } } + + logDebug("iterating dir %s", path.toNativeString()); + try foreach (pdir; iterateDirectory(path)) { + logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name); + if (!pdir.isDirectory) continue; + + // Old / flat directory structure, used in non-standard path + // Packages are stored in $ROOT/$SOMETHING/` + const pack_path = path ~ (pdir.name ~ "/"); + auto packageFile = Package.findPackageFile(pack_path); + if (!packageFile.empty) { + // Deprecated unmanaged directory structure + logWarn("Package at path '%s' should be under '%s'", + pack_path.toNativeString().color(Mode.bold), + (pack_path ~ "$VERSION" ~ pdir.name).toNativeString().color(Mode.bold)); + logWarn("The package will no longer be detected starting from v1.42.0"); + loadInternal(pack_path, packageFile); + } + + // Managed structure: $ROOT/$NAME/$VERSION/$NAME + // This is the most common code path + else if (mgr.isManagedPath(path)) { + // Iterate over versions of a package + foreach (versdir; iterateDirectory(pack_path)) { + if (!versdir.isDirectory) continue; + auto vers_path = pack_path ~ versdir.name ~ (pdir.name ~ "/"); + if (!vers_path.existsDirectory()) continue; + packageFile = Package.findPackageFile(vers_path); + loadInternal(vers_path, packageFile); + } + } + } catch (Exception e) logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString()); } @@ -1417,9 +1428,7 @@ private struct Location { */ private NativePath getPackagePath (string name, string vers) { - // + has special meaning for Optlink - string clean_vers = vers.chompPrefix("~").replace("+", "_"); - NativePath result = this.packagePath ~ (name ~ "-" ~ clean_vers); + NativePath result = this.packagePath ~ name ~ vers; result.endsWithSlash = true; return result; } diff --git a/test/dpath-variable.sh b/test/dpath-variable.sh index 7e68020d2..db4ac1dc6 100755 --- a/test/dpath-variable.sh +++ b/test/dpath-variable.sh @@ -6,7 +6,7 @@ rm -rf "$DPATH" cd "${CURR_DIR}/dpath-variable" "${DUB}" upgrade -if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then +if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then die $LINENO 'Did not get dependencies installed into $DPATH.' fi @@ -24,6 +24,6 @@ trap cleanup EXIT "${DUB}" upgrade -if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then +if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then die $LINENO 'Did not get dependencies installed into dubHome (set from config).' fi diff --git a/test/interactive-remove.sh b/test/interactive-remove.sh index ee5f5a351..8919fefcc 100755 --- a/test/interactive-remove.sh +++ b/test/interactive-remove.sh @@ -9,32 +9,32 @@ # we need to nuke every `dub` version in the user cache... $DUB remove dub -n || true -$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ] -$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ] +$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ] +$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ] echo 1 | $DUB remove dub | tr -d '\n' | grep --ignore-case 'select.*1\.9\.0.*1\.10\.0.*' -if [ -d $HOME/.dub/packages/dub-1.9.0/dub ]; then +if [ -d $HOME/.dub/packages/dub/1.9.0/dub ]; then die $LINENO 'Failed to remove dub-1.9.0' fi -$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ] +$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ] # EOF aborts remove echo -xn '' | $DUB remove dub -if [ ! -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub-1.10.0/dub ]; then +if [ ! -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub/1.10.0/dub ]; then die $LINENO 'Aborted dub still removed a package' fi # validates input echo -e 'abc\n4\n-1\n3' | $DUB remove dub -if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then +if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then die $LINENO 'Failed to remove all version of dub' fi -$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ] -$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ] +$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ] +$DUB fetch dub@1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ] # is non-interactive with a $DUB remove dub@1.9.0 $DUB remove dub@1.10.0 -if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then +if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then die $LINENO 'Failed to non-interactively remove specified versions' fi diff --git a/test/issue838-custom-cache-paths/cache/foo-1.0.0/foo/dub.sdl b/test/issue838-custom-cache-paths/cache/foo/1.0.0/foo/dub.sdl similarity index 100% rename from test/issue838-custom-cache-paths/cache/foo-1.0.0/foo/dub.sdl rename to test/issue838-custom-cache-paths/cache/foo/1.0.0/foo/dub.sdl diff --git a/test/version-spec.sh b/test/version-spec.sh index 498d59f7b..053d17788 100755 --- a/test/version-spec.sh +++ b/test/version-spec.sh @@ -43,10 +43,10 @@ $DUB add-local "$CURR_DIR/version-spec/oldfoo" $DUB remove-local "$CURR_DIR/version-spec/newfoo" $DUB remove-local "$CURR_DIR/version-spec/oldfoo" -$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub-1.9.0/dub ] -$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ] +$DUB fetch dub@1.9.0 && [ -d $HOME/.dub/packages/dub/1.9.0/dub ] +$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ] $DUB remove dub@1.9.0 $DUB remove dub=1.10.0 -if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then +if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then die $LINENO 'Failed to remove specified versions' fi