fix(rules): add Python partial support

CONTRIBUTING.md

@@ -70,6 +70,7 @@ Provide as much dependencies as possible when creating the rule.
 - GO: <https://pkg.go.dev/>
 - JS: <https://yarnpkg.com/search?q=>
 - PHP: <https://packagist.org/>
+- Python <https://pypi.org/>
 - Ruby: <https://rubygems.org/>
 - Rust: <https://crates.io/>
 - Terraform: <https://registry.terraform.io/>

README.md

@@ -7,8 +7,8 @@ This library provide a simple way to extract dependencies, and metadata from any
 
 Detect more than **+500 technologies** in your code base.
 
-- **Full Support:** NodeJS, PHP, Docker, Github Actions
-- Partial Support: Deno, Golang, Ruby, Rust, Terraform, Zig
+- **Full Support:** Docker, NodeJS, Typescript, PHP, Github Actions
+- Partial Support: Deno, Golang, Python, Ruby, Rust, Terraform, Zig
 
 Check all [rules](https://github.com/specfy/stack-analyser/tree/main/src/rules).
 

src/loader.ts

@@ -29,6 +29,7 @@ export const dependencies: Record<
   golang: [],
   npm: [],
   php: [],
+  python: [],
   ruby: [],
   rust: [],
   terraform: [],

src/rules/language/index.ts

@@ -6,5 +6,4 @@ import './css.js';
 import './dart.js';
 import './elixir.js';
 import './java.js';
-import './python.js';
 import './scss.js';

src/rules/monitoring/datadog.ts

@@ -22,5 +22,6 @@ register({
     { type: 'php', name: 'datadog/dd-trace' },
     { type: 'githubAction', name: 'DataDog/synthetics-ci-github-action' },
     { type: 'githubAction', name: 'datadog/agent-github-action' },
+    { type: 'python', name: 'datadog' },
   ],
 });

src/rules/saas/algolia.ts

@@ -25,5 +25,7 @@ register({
       type: 'githubAction',
       name: 'algolia/algoliasearch-crawler-github-actions',
     },
+    { type: 'python', name: 'algoliasearch' },
+    { type: 'python', name: 'algoliasearch-django' },
   ],
 });

src/rules/spec/docker/__snapshots__/component.test.ts.snap

@@ -83,6 +83,7 @@ exports[`docker > should match everything 1`] = `
   "postgresql",
   "prometheus",
   "puppeteer",
+  "python",
   "qovery",
   "questdb",
   "rabbitmq",
@@ -315,6 +316,11 @@ exports[`docker > should match everything 2`] = `
     "bitnami/prometheus-operator",
     "0.0.0",
   ],
+  [
+    "docker",
+    "bitnami/python",
+    "0.0.0",
+  ],
   [
     "docker",
     "bitnami/solr",
@@ -400,6 +406,11 @@ exports[`docker > should match everything 2`] = `
     "cimg/postgres",
     "0.0.0",
   ],
+  [
+    "docker",
+    "cimg/python",
+    "0.0.0",
+  ],
   [
     "docker",
     "cimg/rust",
@@ -430,6 +441,11 @@ exports[`docker > should match everything 2`] = `
     "circleci/postgres",
     "0.0.0",
   ],
+  [
+    "docker",
+    "circleci/python",
+    "0.0.0",
+  ],
   [
     "docker",
     "circleci/rust",
@@ -885,6 +901,11 @@ exports[`docker > should match everything 2`] = `
     "public.ecr.aws/r3m4q3r9/qovery-cli",
     "0.0.0",
   ],
+  [
+    "docker",
+    "python",
+    "0.0.0",
+  ],
   [
     "docker",
     "questdb/questdb",

src/rules/spec/githubActions/__snapshots__/component.test.ts.snap

@@ -140,6 +140,11 @@ exports[`docker > should match all dependencies 2`] = `
     "actions/deploy-pages",
     "1",
   ],
+  [
+    "githubAction",
+    "actions/setup-python",
+    "1",
+  ],
   [
     "githubAction",
     "actions/upload-pages-artifact",

src/rules/spec/index.ts

@@ -4,6 +4,7 @@ import './githubActions/index.js';
 import './golang/index.js';
 import './nodejs/index.js';
 import './php/index.js';
+import './python/index.js';
 import './ruby/index.js';
 import './rust/index.js';
 import './terraform/index.js';

src/rules/spec/python/__snapshots__/lockfile.test.ts.snap

@@ -0,0 +1,62 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`python (lockfile) > should match everything 1`] = `
+[
+  "algolia",
+  "datadog",
+  "python",
+]
+`;
+
+exports[`python (lockfile) > should match everything 2`] = `
+[
+  [
+    "python",
+    "algoliasearch",
+    "1.0.0",
+  ],
+  [
+    "python",
+    "algoliasearch-django",
+    "1.0.0",
+  ],
+  [
+    "python",
+    "datadog",
+    "1.0.0",
+  ],
+  [
+    "python",
+    "tensorflow",
+    "1.0.0",
+  ],
+]
+`;
+
+exports[`python (lockfile) > should match weird syntax 1`] = `
+[
+  "algolia",
+  "datadog",
+  "python",
+]
+`;
+
+exports[`python (lockfile) > should match weird syntax 2`] = `
+[
+  [
+    "python",
+    "algoliasearch",
+    "1.9.1",
+  ],
+  [
+    "python",
+    "datadog",
+    "1.4.3",
+  ],
+  [
+    "python",
+    "tensorflow",
+    "latest",
+  ],
+]
+`;

src/rules/spec/python/index.ts

@@ -0,0 +1,29 @@
+import { register } from '../../../register.js';
+
+import { detectPythonLockfile } from './lockfile.js';
+
+register({
+  tech: 'python',
+  name: 'Python',
+  type: 'language',
+  files: ['requirements.txt'],
+  extensions: [
+    '.py',
+    '.gyp',
+    '.py3',
+    '.pyde',
+    '.pyi',
+    '.pyp',
+    '.pyt',
+    '.pyw',
+    '.rpy',
+  ],
+  dependencies: [
+    { type: 'githubAction', name: 'actions/setup-python' },
+    { type: 'docker', name: 'python' },
+    { type: 'docker', name: 'circleci/python' },
+    { type: 'docker', name: 'cimg/python' },
+    { type: 'docker', name: 'bitnami/python' },
+  ],
+  detect: [detectPythonLockfile],
+});

src/rules/spec/python/lockfile.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from 'vitest';
+
+import { analyser } from '../../../analyser/index.js';
+import { rawList } from '../../../loader.js';
+import { flatten } from '../../../payload/helpers.js';
+import { FakeProvider } from '../../../provider/fake.js';
+
+import '../../../autoload.js';
+
+describe('python (lockfile)', () => {
+  it('should match everything', async () => {
+    const deps = [];
+    for (const item of rawList) {
+      if (item.type !== 'dependency' || item.ref.type !== 'python') {
+        continue;
+      }
+
+      const example = 'example' in item.ref ? item.ref.example : item.ref.name;
+      deps.push(`${example}==1.0.0`);
+    }
+
+    const lockfile: string = deps.join('\n');
+
+    const res = await analyser({
+      provider: new FakeProvider({
+        paths: {
+          '/': ['requirements.txt'],
+        },
+        files: {
+          '/requirements.txt': lockfile,
+        },
+      }),
+    });
+
+    const merged = flatten(res, { merge: true });
+    expect(Array.from(merged.techs).sort()).toMatchSnapshot();
+    expect(Array.from(merged.dependencies).sort()).toMatchSnapshot();
+  });
+
+  it('should match weird syntax', async () => {
+    const res = await analyser({
+      provider: new FakeProvider({
+        paths: {
+          '/': ['requirements.txt'],
+        },
+        files: {
+          '/requirements.txt': [
+            // No version
+            'tensorflow',
+            // gte
+            'algoliasearch>=1.9.1',
+            // gte-lt
+            'datadog>=1.4.3,<7.0.0',
+          ].join('\n'),
+        },
+      }),
+    });
+
+    const merged = flatten(res, { merge: true });
+    expect(Array.from(merged.techs).sort()).toMatchSnapshot();
+    expect(Array.from(merged.dependencies).sort()).toMatchSnapshot();
+  });
+});

src/rules/spec/python/lockfile.ts

@@ -0,0 +1,64 @@
+import path from 'node:path';
+
+import { matchDependencies } from '../../../matchDependencies.js';
+import { Payload } from '../../../payload/index.js';
+import type { Dependency } from '../../../types/index.js';
+import type { ComponentMatcher } from '../../../types/rule.js';
+
+const LOCKFILE = 'requirements.txt';
+const lineReg = /(^([a-zA-Z0-9._-]+)$|^([a-zA-Z0-9._-]+)(([>=]+)([0-9.]+)))/;
+
+export const detectPythonLockfile: ComponentMatcher = async (
+  files,
+  provider
+) => {
+  for (const file of files) {
+    if (file.name !== LOCKFILE) {
+      continue;
+    }
+
+    const content = await provider.open(file.fp);
+    if (!content) {
+      continue;
+    }
+
+    const pl = new Payload({
+      name: 'virtual',
+      folderPath: path.dirname(file.fp),
+    });
+
+    const dependencies: Dependency[] = [];
+    // We only register docker service with image and that we know
+    for (const line of content.split(/\r?\n/)) {
+      if (line.startsWith('#')) {
+        continue;
+      }
+
+      const match = line.match(lineReg);
+      if (!match) {
+        continue;
+      }
+
+      const name = match[2] || match[3];
+      const version = match[6] || 'latest';
+
+      if (!name) {
+        continue;
+      }
+
+      dependencies.push(['python', name, version || 'latest']);
+      const matched = matchDependencies([name], 'python');
+      if (matched.size <= 0) {
+        continue;
+      }
+
+      pl.addTechs(matched);
+    }
+
+    pl.dependencies = [...dependencies];
+
+    return pl;
+  }
+
+  return false;
+};

src/rules/tool/tensorflow.ts

@@ -10,5 +10,6 @@ register({
     { type: 'npm', name: '@tensorflow/tfjs' },
     { type: 'npm', name: 'face-api.js' },
     { type: 'golang', name: 'github.com/wamuir/graft/tensorflow' },
+    { type: 'python', name: 'tensorflow' },
   ],
 });

src/types/rule.ts

@@ -10,6 +10,7 @@ export type SupportedDeps =
   | 'golang'
   | 'npm'
   | 'php'
+  | 'python'
   | 'ruby'
   | 'rust'
   | 'terraform.resource'