feat(manager/pep621): implement manager (#22082)

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

lib/modules/datasource/pypi/index.ts

@@ -21,9 +21,9 @@ export class PypiDatasource extends Datasource {
 
   override readonly customRegistrySupport = true;
 
-  override readonly defaultRegistryUrls = [
-    process.env.PIP_INDEX_URL ?? 'https://pypi.org/pypi/',
-  ];
+  static readonly defaultURL =
+    process.env.PIP_INDEX_URL ?? 'https://pypi.org/pypi/';
+  override readonly defaultRegistryUrls = [PypiDatasource.defaultURL];
 
   override readonly defaultVersioning = pep440.id;
 

lib/modules/manager/api.ts

@@ -58,6 +58,7 @@ import * as npm from './npm';
 import * as nuget from './nuget';
 import * as nvm from './nvm';
 import * as osgi from './osgi';
+import * as pep621 from './pep621';
 import * as pipCompile from './pip-compile';
 import * as pip_requirements from './pip_requirements';
 import * as pip_setup from './pip_setup';
@@ -146,6 +147,7 @@ api.set('npm', npm);
 api.set('nuget', nuget);
 api.set('nvm', nvm);
 api.set('osgi', osgi);
+api.set('pep621', pep621);
 api.set('pip-compile', pipCompile);
 api.set('pip_requirements', pip_requirements);
 api.set('pip_setup', pip_setup);

lib/modules/manager/pep621/__fixtures__/pyproject_pdm_sources.toml

@@ -0,0 +1,33 @@
+[project]
+name = "pdm"
+dynamic = ["version"]
+requires-python = ">=3.7"
+license = {text = "MIT"}
+dependencies = [
+  "blinker",
+  "packaging>=20.9,!=22.0",
+]
+readme = "README.md"
+
+[project.optional-dependencies]
+pytest = [
+  "pytest>12",
+]
+
+[tool.pdm.dev-dependencies]
+test = [
+  "pytest-rerunfailures>=10.2",
+]
+tox = [
+  "tox-pdm>=0.5",
+]
+
+[[tool.pdm.source]]
+url = "https://private-site.org/pypi/simple"
+verify_ssl = true
+name = "internal"
+
+[[tool.pdm.source]]
+url = "https://private.pypi.org/simple"
+verify_ssl = true
+name = "pypi"

lib/modules/manager/pep621/__fixtures__/pyproject_with_pdm.toml

@@ -0,0 +1,37 @@
+[project]
+name = "pdm"
+dynamic = ["version"]
+requires-python = ">=3.7"
+license = {text = "MIT"}
+dependencies = [
+  "blinker",
+  "packaging>=20.9,!=22.0",
+  "rich>=12.3.0",
+  "virtualenv==20.0.0",
+  "pyproject-hooks",
+  "unearth>=0.9.0",
+  "tomlkit>=0.11.1,<1",
+  "installer<0.8,>=0.7",
+  "cachecontrol[filecache]>=0.12.11",
+  "tomli>=1.1.0; python_version < \"3.11\"",
+  "typing-extensions; python_version < \"3.8\"",
+  "importlib-metadata>=3.6; python_version < \"3.10\"",
+]
+readme = "README.md"
+
+[project.optional-dependencies]
+pytest = [
+  "pytest>12",
+  "pytest-mock",
+]
+
+[tool.pdm.dev-dependencies]
+test = [
+  "pdm[pytest]",
+  "pytest-rerunfailures>=10.2",
+]
+tox = [
+  "tox",
+  "tox-pdm>=0.5",
+  "", # fail to parse
+]

lib/modules/manager/pep621/extract.spec.ts

@@ -0,0 +1,267 @@
+import { codeBlock } from 'common-tags';
+import { Fixtures } from '../../../../test/fixtures';
+import { extractPackageFile } from './extract';
+
+const pdmPyProject = Fixtures.get('pyproject_with_pdm.toml');
+const pdmSourcesPyProject = Fixtures.get('pyproject_pdm_sources.toml');
+
+describe('modules/manager/pep621/extract', () => {
+  describe('extractPackageFile()', () => {
+    it('should return null for empty content', function () {
+      const result = extractPackageFile('', 'pyproject.toml');
+      expect(result).toBeNull();
+    });
+
+    it('should return null for invalid toml', function () {
+      const result = extractPackageFile(
+        codeBlock`
+        [project]
+        name =
+      `,
+        'pyproject.toml'
+      );
+      expect(result).toBeNull();
+    });
+
+    it('should return dependencies for valid content', function () {
+      const result = extractPackageFile(pdmPyProject, 'pyproject.toml');
+
+      const dependencies = result?.deps.filter(
+        (dep) => dep.depType === 'project.dependencies'
+      );
+      expect(dependencies).toEqual([
+        {
+          packageName: 'blinker',
+          depName: 'blinker',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          skipReason: 'any-version',
+        },
+        {
+          packageName: 'packaging',
+          depName: 'packaging',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=20.9,!=22.0',
+        },
+        {
+          packageName: 'rich',
+          depName: 'rich',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=12.3.0',
+        },
+        {
+          packageName: 'virtualenv',
+          depName: 'virtualenv',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '==20.0.0',
+        },
+        {
+          packageName: 'pyproject-hooks',
+          depName: 'pyproject-hooks',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          skipReason: 'any-version',
+        },
+        {
+          packageName: 'unearth',
+          depName: 'unearth',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=0.9.0',
+        },
+        {
+          packageName: 'tomlkit',
+          depName: 'tomlkit',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=0.11.1,<1',
+        },
+        {
+          packageName: 'installer',
+          depName: 'installer',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '<0.8,>=0.7',
+        },
+        {
+          packageName: 'cachecontrol',
+          depName: 'cachecontrol',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=0.12.11',
+        },
+        {
+          packageName: 'tomli',
+          depName: 'tomli',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=1.1.0',
+        },
+        {
+          packageName: 'typing-extensions',
+          depName: 'typing-extensions',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          skipReason: 'any-version',
+        },
+        {
+          packageName: 'importlib-metadata',
+          depName: 'importlib-metadata',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=3.6',
+        },
+      ]);
+
+      const optionalDependencies = result?.deps.filter(
+        (dep) => dep.depType === 'project.optional-dependencies'
+      );
+      expect(optionalDependencies).toEqual([
+        {
+          packageName: 'pytest',
+          datasource: 'pypi',
+          depType: 'project.optional-dependencies',
+          currentValue: '>12',
+          depName: 'pytest/pytest',
+        },
+        {
+          packageName: 'pytest-mock',
+          datasource: 'pypi',
+          depType: 'project.optional-dependencies',
+          skipReason: 'any-version',
+          depName: 'pytest/pytest-mock',
+        },
+      ]);
+
+      const pdmDevDependencies = result?.deps.filter(
+        (dep) => dep.depType === 'tool.pdm.dev-dependencies'
+      );
+      expect(pdmDevDependencies).toEqual([
+        {
+          packageName: 'pdm',
+          datasource: 'pypi',
+          depType: 'tool.pdm.dev-dependencies',
+          skipReason: 'any-version',
+          depName: 'test/pdm',
+        },
+        {
+          packageName: 'pytest-rerunfailures',
+          datasource: 'pypi',
+          depType: 'tool.pdm.dev-dependencies',
+          currentValue: '>=10.2',
+          depName: 'test/pytest-rerunfailures',
+        },
+        {
+          packageName: 'tox',
+          datasource: 'pypi',
+          depType: 'tool.pdm.dev-dependencies',
+          skipReason: 'any-version',
+          depName: 'tox/tox',
+        },
+        {
+          packageName: 'tox-pdm',
+          datasource: 'pypi',
+          depType: 'tool.pdm.dev-dependencies',
+          currentValue: '>=0.5',
+          depName: 'tox/tox-pdm',
+        },
+      ]);
+    });
+
+    it('should return dependencies with overwritten pypi registryUrl', function () {
+      const result = extractPackageFile(pdmSourcesPyProject, 'pyproject.toml');
+
+      expect(result?.deps).toEqual([
+        {
+          packageName: 'blinker',
+          depName: 'blinker',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          skipReason: 'any-version',
+          registryUrls: [
+            'https://private-site.org/pypi/simple',
+            'https://private.pypi.org/simple',
+          ],
+        },
+        {
+          packageName: 'packaging',
+          depName: 'packaging',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=20.9,!=22.0',
+          registryUrls: [
+            'https://private-site.org/pypi/simple',
+            'https://private.pypi.org/simple',
+          ],
+        },
+        {
+          packageName: 'pytest',
+          datasource: 'pypi',
+          depType: 'project.optional-dependencies',
+          currentValue: '>12',
+          depName: 'pytest/pytest',
+          registryUrls: [
+            'https://private-site.org/pypi/simple',
+            'https://private.pypi.org/simple',
+          ],
+        },
+        {
+          packageName: 'pytest-rerunfailures',
+          datasource: 'pypi',
+          depType: 'tool.pdm.dev-dependencies',
+          currentValue: '>=10.2',
+          depName: 'test/pytest-rerunfailures',
+          registryUrls: [
+            'https://private-site.org/pypi/simple',
+            'https://private.pypi.org/simple',
+          ],
+        },
+        {
+          packageName: 'tox-pdm',
+          datasource: 'pypi',
+          depType: 'tool.pdm.dev-dependencies',
+          currentValue: '>=0.5',
+          depName: 'tox/tox-pdm',
+          registryUrls: [
+            'https://private-site.org/pypi/simple',
+            'https://private.pypi.org/simple',
+          ],
+        },
+      ]);
+    });
+
+    it('should return dependencies with original pypi registryUrl', function () {
+      const result = extractPackageFile(
+        codeBlock`
+      [project]
+      dependencies = [
+        "packaging>=20.9,!=22.0",
+      ]
+
+      [[tool.pdm.source]]
+      url = "https://private-site.org/pypi/simple"
+      verify_ssl = true
+      name = "internal"
+      `,
+        'pyproject.toml'
+      );
+
+      expect(result?.deps).toEqual([
+        {
+          packageName: 'packaging',
+          depName: 'packaging',
+          datasource: 'pypi',
+          depType: 'project.dependencies',
+          currentValue: '>=20.9,!=22.0',
+          registryUrls: [
+            'https://pypi.org/pypi/',
+            'https://private-site.org/pypi/simple',
+          ],
+        },
+      ]);
+    });
+  });
+});