Implement executable plugin

Makefile

@@ -1,3 +1,5 @@
+.DEFAULT_GOAL := up
+
 .PHONY: up
 up:
 	docker compose --compatibility up -d --build

README.md

@@ -365,6 +365,12 @@ You can automate data processing by directly calling API via `curl` or something
 Visit [/api-doc](http://localhost:8080/api-doc) from top-right of the UI, for API documentation in OpenAPI format.
 Equivalent curl commands for each API call are also obtainable from the swagger UI.
 
+### (Advanced Usage) Using Plugins
+
+While RefSearch natively support RefactoringMiner and RefDiff, you can bring your own program to detect more refactoring instances.
+
+See [./docs/development.md](./docs/development.md) for more.
+
 ## Refactoring Types
 
 RefSearch uses the following tools under the hood to automatically detect refactorings inside repositories' commits.

backend.Dockerfile

@@ -42,6 +42,8 @@ COPY --from=builder /work/backend/out /work
 COPY package.json /work
 COPY backend/package.json /work/backend
 
+RUN sh -c 'chmod +x ./backend/src/cmd/*.js'
+
 # NOTE: "node pid 1 problem"
 ENTRYPOINT ["/sbin/tini", "--"]
 CMD ["node", "backend/src/cmd/jobRunner.js"]

backend/src/api/serve/refactorings.ts

@@ -7,6 +7,7 @@ interface PostRequest extends Request {
   body: {
     repository: string
     commit: string
+    toolName: string
 
     refactorings: PureRefactoringMeta[]
   }
@@ -33,7 +34,7 @@ export const postRefactoringsHandler = async (req: PostRequest, res: Response) =
   }
 
   // Insert
-  const insertRes = await transformAndInsertRefactorings(body.repository, body.commit, body.refactorings)
+  const insertRes = await transformAndInsertRefactorings(body.repository, body.commit, body.toolName, body.refactorings)
 
   return res.status(200).json({
     message: `Inserted ${insertRes.insertedCount} document(s)`,

backend/src/api/tools/refdiff.ts

@@ -4,12 +4,13 @@ import { URLSearchParams } from 'url'
 import { humanishName } from '../../utils.js'
 import { RefDiffRefactoring } from '../../../../common/refdiff.js'
 import { HTTPStatusError } from '../error.js'
+import { memo } from '../../../../common/utils.js'
 
-const baseUrl = `http://${config.tool.refDiff.host}:${config.tool.refDiff.port}/detect`
+const baseUrl = memo(() => `http://${config().tool.refDiff.host}:${config().tool.refDiff.port}/detect`)
 
 export const detectRefDiffRefactorings = async (repoUrl: string, commit: string, timeoutSeconds: number): Promise<RefDiffRefactoring[]> => {
-  const json = await fetch(baseUrl + '?' + new URLSearchParams({
-    dir: config.tool.refDiff.baseRepoPath + '/' + humanishName(repoUrl) + '/.git',
+  const json = await fetch(baseUrl() + '?' + new URLSearchParams({
+    dir: config().tool.refDiff.baseRepoPath + '/' + humanishName(repoUrl) + '/.git',
     commit: commit,
     timeout: '' + timeoutSeconds,
   }).toString())

backend/src/api/tools/rminer.ts

@@ -4,12 +4,13 @@ import { URLSearchParams } from 'url'
 import { humanishName } from '../../utils.js'
 import { RMRefactoring } from '../../../../common/rminer.js'
 import { HTTPStatusError } from '../error.js'
+import { memo } from '../../../../common/utils.js'
 
-const baseUrl = `http://${config.tool.rminer.host}:${config.tool.rminer.port}/detect`
+const baseUrl = memo(() => `http://${config().tool.rminer.host}:${config().tool.rminer.port}/detect`)
 
 export const detectRMinerRefactorings = async (repoUrl: string, commit: string, timeoutSeconds: number): Promise<RMRefactoring[]> => {
-  const json = await fetch(baseUrl + '?' + new URLSearchParams({
-    dir: config.tool.rminer.baseRepoPath + '/' + humanishName(repoUrl),
+  const json = await fetch(baseUrl() + '?' + new URLSearchParams({
+    dir: config().tool.rminer.baseRepoPath + '/' + humanishName(repoUrl),
     commit: commit,
     timeout: '' + timeoutSeconds,
   }).toString())

backend/src/cmd/index.ts

@@ -10,7 +10,7 @@ const main = async () => {
   app.use(express.json())
   registerRoutes(app)
 
-  app.listen(config.port, () => console.log(`API server started on port ${config.port}`))
+  app.listen(config().port, () => console.log(`API server started on port ${config().port}`))
 }
 
 main()

backend/src/cmd/jobRunner.ts

@@ -46,7 +46,7 @@ const reservedNextJob = async (): Promise<JobWithId | undefined> => {
   // Find already reserved / running jobs (in case this job runner has restarted)
   const order: [keyof Job, 'asc' | 'desc'][] = [['queuedAt', 'asc']]
   const reserved = await readAllFromCursor(
-    jobWithData.find({ status: { $in: [JobStatus.Ready, JobStatus.Running] }, 'data.runnerId': config.runnerId }, { sort: order }),
+    jobWithData.find({ status: { $in: [JobStatus.Ready, JobStatus.Running] }, 'data.runnerId': config().runnerId }, { sort: order }),
   )
   const running = reserved.find((j) => j.status === JobStatus.Running)
   if (running) return running
@@ -64,7 +64,7 @@ const findNextJob = async (): Promise<JobWithId | undefined> => {
   // Atomically find and reserve next pipeline
   const next = await jobDataCol.findOneAndUpdate({
     runnerId: { $exists: false }
-  }, { $set: { runnerId: config.runnerId } })
+  }, { $set: { runnerId: config().runnerId } })
   if (next) {
     return reservedNextJob()
   }

backend/src/cmd/plugin-refdiff.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+
+import { pluginRefDiffMain } from '../plugins/refdiff.js'
+
+if (process.argv.length < 4) {
+  throw new Error(`Expected at least 4 argv.length, got: ${JSON.stringify(process.argv)}`)
+}
+
+pluginRefDiffMain(process.argv[2], process.argv[3])
+  .then(res => console.log(JSON.stringify(res)))
+  .then(() => process.exit(0))

backend/src/cmd/plugin-rminer.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+
+import { pluginRMinerMain } from '../plugins/rminer.js'
+
+if (process.argv.length < 4) {
+  throw new Error(`Expected at least 4 argv.length, got: ${JSON.stringify(process.argv)}`)
+}
+
+pluginRMinerMain(process.argv[2], process.argv[3])
+  .then(res => console.log(JSON.stringify(res)))
+  .then(() => process.exit(0))

backend/src/config.ts

@@ -1,33 +1,135 @@
-export const config = {
-  port: Number.parseInt(process.env.PORT ?? '') || 3000,
-  db: {
-    user: process.env.MONGODB_USER || 'root',
-    password: process.env.MONGODB_PASSWORD || 'password',
-    host: process.env.MONGODB_HOST || 'localhost',
-    port: process.env.MONGODB_PORT || '27017',
-  },
-  tool: {
-    rminer: {
-      host: process.env.RMINER_HOST || 'rminer',
-      port: process.env.RMINER_PORT || '3000',
-      baseRepoPath: process.env.RMINER_BASE_PATH || '/data/repos',
+import { memo, unique } from '../../common/utils.js'
+import path from 'path'
+import { PureRefactoringMeta } from '../../common/common.js'
+import { spawnSync } from 'node:child_process'
+import { repoDirName } from './jobs/info.js'
+import { pluginRMinerMain } from './plugins/rminer.js'
+import { pluginRefDiffMain } from './plugins/refdiff.js'
+
+type ProcessPluginFunc = (repoUrl: string, commit: string) => Promise<PureRefactoringMeta[]>
+
+export class ProcessPlugin {
+  private readonly executable: string
+
+  private override: ProcessPluginFunc | undefined
+
+  constructor(executable: string) {
+    this.executable = executable
+  }
+
+  public setOverride(f: ProcessPluginFunc): this {
+    this.override = f
+    return this
+  }
+
+  public async run(repoUrl: string, commit: string): Promise<PureRefactoringMeta[]> {
+    if (this.override) return this.override(repoUrl, commit)
+
+    const res = spawnSync(this.executable, [repoUrl, commit], {
+      cwd: repoDirName(repoUrl),
+      stdio: ['pipe', 'pipe', process.stderr],
+    })
+
+    // Check return code
+    if (res.status !== 0) {
+      if (res.error) console.trace(res.error)
+      return Promise.reject(`executable plugin process exited with code ${res.status}`)
+    }
+
+    // Validate output
+    const out = JSON.parse(res.stdout.toString())
+    if (!Array.isArray(out)) {
+      return Promise.reject(`plugin output an invalid json (not an array)`)
+    }
+    for (const refactoring of out) {
+      if (typeof refactoring.type !== 'string') {
+        return Promise.reject(`plugin output an invalid json ("type" string field is required)`)
+      }
+      if (typeof refactoring.description !== 'string') {
+        return Promise.reject(`plugin output an invalid json ("description" string field is required)`)
+      }
+    }
+
+    return out
+  }
+}
+
+export const config = memo(() => {
+  const c = {
+    port: Number.parseInt(process.env.PORT ?? '') || 3000,
+    db: {
+      user: process.env.MONGODB_USER || 'root',
+      password: process.env.MONGODB_PASSWORD || 'password',
+      host: process.env.MONGODB_HOST || 'localhost',
+      port: process.env.MONGODB_PORT || '27017',
     },
-    refDiff: {
-      host: process.env.REFDIFF_HOST || 'refdiff',
-      port: process.env.REFDIFF_PORT || '3000',
-      baseRepoPath: process.env.REFDIFF_BASE_PATH || '/data/repos',
+    tool: {
+      plugins: {} as Record<string, ProcessPlugin>,
+      rminer: {
+        host: process.env.RMINER_HOST || 'rminer',
+        port: process.env.RMINER_PORT || '3000',
+        baseRepoPath: process.env.RMINER_BASE_PATH || '/data/repos',
+      },
+      refDiff: {
+        host: process.env.REFDIFF_HOST || 'refdiff',
+        port: process.env.REFDIFF_PORT || '3000',
+        baseRepoPath: process.env.REFDIFF_BASE_PATH || '/data/repos',
+      },
     },
-  },
-  runnerId: process.env.RUNNER_ID || '',
-  dataDir: process.env.DATA_DIR || '',
-} as const
+    runnerId: process.env.RUNNER_ID || '',
+    dataDir: process.env.DATA_DIR || '',
+  } as const
+
+  const readProcessPlugins = () => {
+    const processPluginEnvPrefix = 'PROCESS_PLUGIN_'
+    const processPlugins = unique(
+      Object.entries(process.env)
+        .filter(([key]) => key.startsWith(processPluginEnvPrefix))
+        .map(([key]) => {
+          const envSuffix = key.substring(processPluginEnvPrefix.length)
+          return envSuffix.split('_')[0]
+        }),
+    )
+
+    // Built-in plugins
+    c.tool.plugins['RefactoringMiner'] = new ProcessPlugin(
+      path.join(import.meta.dirname, './cmd/plugin-rminer.js'),
+    )
+      .setOverride(pluginRMinerMain) // Bypass process spawning to avoid mongo connection overheads
+    c.tool.plugins['RefDiff'] = new ProcessPlugin(
+      path.join(import.meta.dirname, './cmd/plugin-refdiff.js'),
+    )
+      .setOverride(pluginRefDiffMain)
+
+    for (const pluginPrefix of processPlugins) {
+      const prefix = processPluginEnvPrefix + pluginPrefix + '_'
 
-export const validateRunnerConfig = () => {
+      const name = process.env[prefix + 'NAME'] || ''
+      const executable = process.env[prefix + 'EXECUTABLE'] || ''
+
+      if (!name || !executable) {
+        console.warn(`Not all required environment variables are present for ${prefix} group, skipping plugin addition`)
+        continue
+      }
+      if (c.tool.plugins[name]) {
+        console.warn(`${prefix} group has conflicted name ${name}`)
+        continue
+      }
+
+      c.tool.plugins[name] = new ProcessPlugin(executable)
+    }
+  }
+  readProcessPlugins()
+
+  return c
+})
+
+export const validateRunnerConfig = memo(() => {
   const rules: [v: string, name: string, msg: string][] = [
-    [config.runnerId, 'RUNNER_ID', 'Please set it to a unique value for each job runner.'],
-    [config.dataDir, 'DATA_DIR', 'Please set it to the path to data directory inside container.'],
+    [config().runnerId, 'RUNNER_ID', 'Please set it to a unique value for each job runner.'],
+    [config().dataDir, 'DATA_DIR', 'Please set it to the path to data directory inside container.'],
   ]
   for (const [v, name, msg] of rules) {
     if (!v) throw new Error(`Environment variable ${name} not set. ${msg}`)
   }
-}
+})

backend/src/jobs/info.ts

@@ -3,8 +3,8 @@ import fs from 'fs'
 import path from 'path'
 import { config } from '../config.js'
 
-export const repositoriesDir = (baseDir: string = config.dataDir) => path.resolve(baseDir, './repos')
-export const repoDirName = (repoUrl: string, baseDir: string = config.dataDir): string => `${repositoriesDir(baseDir)}/${humanishName(repoUrl)}`
+export const repositoriesDir = (baseDir: string = config().dataDir) => path.resolve(baseDir, './repos')
+export const repoDirName = (repoUrl: string, baseDir: string = config().dataDir): string => `${repositoriesDir(baseDir)}/${humanishName(repoUrl)}`
 
 const makeDirIfNotExists = (dir: string) => {
   if (!fs.existsSync(dir)) {