Improve isolation, execution limits and execution metrics by using Is…

…olate (#683) * Initial: use Isolate for isolation * Continue: use Isolate for isolation * Bug fixes * timeout is wall-time for backward compatibility * Documentation, signal names, reported time in ms * Report memory usage in bytes * Add privileged flags where needed * Remove tmpfs * Remove tmpfs * Fix package installation * Fix path, fix Zig: CRLF -> LF
engineer-man · Sep 8, 2024 · bd42fe3 · bd42fe3
1 parent 59338ee
commit bd42fe3
Show file tree

Hide file tree

Showing 22 changed files with 509 additions and 455 deletions.
diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml
@@ -92,7 +92,7 @@ jobs:
                   docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build
                   docker pull docker.pkg.github.com/engineer-man/piston/api
                   docker build -t piston-api api
-                  docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api piston-api
+                  docker run --privileged --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api piston-api
                   echo Waiting for API to start..
                   docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v2/runtimes
 

diff --git a/api/Dockerfile b/api/Dockerfile
@@ -1,20 +1,29 @@
-FROM node:15.10.0-buster-slim
+FROM buildpack-deps:bookworm AS isolate
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git libcap-dev && \
+    rm -rf /var/lib/apt/lists/* && \
+    git clone https://github.com/envicutor/isolate.git /tmp/isolate/ && \
+    cd /tmp/isolate && \
+    git checkout af6db68042c3aa0ded80787fbb78bc0846ea2114 && \
+    make -j$(nproc) install && \
+    rm -rf /tmp/*
+
+FROM node:20-bookworm-slim
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN dpkg-reconfigure -p critical dash
-RUN for i in $(seq 1001 1500); do \
-        groupadd -g $i runner$i && \
-        useradd -M runner$i -g $i -u $i ; \
-    done
 RUN apt-get update && \
     apt-get install -y libxml2 gnupg tar coreutils util-linux libc6-dev \
     binutils build-essential locales libpcre3-dev libevent-dev libgmp3-dev \
     libncurses6 libncurses5 libedit-dev libseccomp-dev rename procps python3 \
     libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev \
     libfftw3-dev libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev \
-    libsundials-dev libpcre2-dev && \
+    libsundials-dev libpcre2-dev libcap-dev && \
     rm -rf /var/lib/apt/lists/*
+RUN useradd -M piston
+COPY --from=isolate /usr/local/bin/isolate /usr/local/bin
+COPY --from=isolate /usr/local/etc/isolate /usr/local/etc/isolate
 
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
 
@@ -23,7 +32,5 @@ COPY ["package.json", "package-lock.json", "./"]
 RUN npm install
 COPY ./src ./src
 
-RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install
-
-CMD [ "node", "src"]
+CMD ["/piston_api/src/docker-entrypoint.sh"]
 EXPOSE 2000/tcp
diff --git a/api/src/api/v2.js b/api/src/api/v2.js
@@ -6,50 +6,9 @@ const events = require('events');
 const runtime = require('../runtime');
 const { Job } = require('../job');
 const package = require('../package');
+const globals = require('../globals');
 const logger = require('logplease').create('api/v2');
 
-const SIGNALS = [
-    'SIGABRT',
-    'SIGALRM',
-    'SIGBUS',
-    'SIGCHLD',
-    'SIGCLD',
-    'SIGCONT',
-    'SIGEMT',
-    'SIGFPE',
-    'SIGHUP',
-    'SIGILL',
-    'SIGINFO',
-    'SIGINT',
-    'SIGIO',
-    'SIGIOT',
-    'SIGKILL',
-    'SIGLOST',
-    'SIGPIPE',
-    'SIGPOLL',
-    'SIGPROF',
-    'SIGPWR',
-    'SIGQUIT',
-    'SIGSEGV',
-    'SIGSTKFLT',
-    'SIGSTOP',
-    'SIGTSTP',
-    'SIGSYS',
-    'SIGTERM',
-    'SIGTRAP',
-    'SIGTTIN',
-    'SIGTTOU',
-    'SIGUNUSED',
-    'SIGURG',
-    'SIGUSR1',
-    'SIGUSR2',
-    'SIGVTALRM',
-    'SIGXCPU',
-    'SIGXFSZ',
-    'SIGWINCH',
-];
-// ref: https://man7.org/linux/man-pages/man7/signal.7.html
-
 function get_job(body) {
     let {
         language,
@@ -61,6 +20,8 @@ function get_job(body) {
         run_memory_limit,
         run_timeout,
         compile_timeout,
+        run_cpu_time,
+        compile_cpu_time,
     } = body;
 
     return new Promise((resolve, reject) => {
@@ -106,7 +67,7 @@ function get_job(body) {
             });
         }
 
-        for (const constraint of ['memory_limit', 'timeout']) {
+        for (const constraint of ['memory_limit', 'timeout', 'cpu_time']) {
             for (const type of ['compile', 'run']) {
                 const constraint_name = `${type}_${constraint}`;
                 const constraint_value = body[constraint_name];
@@ -135,23 +96,23 @@ function get_job(body) {
             }
         }
 
-        compile_timeout = compile_timeout || rt.timeouts.compile;
-        run_timeout = run_timeout || rt.timeouts.run;
-        compile_memory_limit = compile_memory_limit || rt.memory_limits.compile;
-        run_memory_limit = run_memory_limit || rt.memory_limits.run;
         resolve(
             new Job({
                 runtime: rt,
-                args: args || [],
-                stdin: stdin || '',
+                args: args ?? [],
+                stdin: stdin ?? '',
                 files,
                 timeouts: {
-                    run: run_timeout,
-                    compile: compile_timeout,
+                    run: run_timeout ?? rt.timeouts.run,
+                    compile: compile_timeout ?? rt.timeouts.compile,
+                },
+                cpu_times: {
+                    run: run_cpu_time ?? rt.cpu_times.run,
+                    compile: compile_cpu_time ?? rt.cpu_times.compile,
                 },
                 memory_limits: {
-                    run: run_memory_limit,
-                    compile: compile_memory_limit,
+                    run: run_memory_limit ?? rt.memory_limits.run,
+                    compile: compile_memory_limit ?? rt.memory_limits.compile,
                 },
             })
         );
@@ -211,7 +172,7 @@ router.ws('/connect', async (ws, req) => {
                         job = await get_job(msg);
 
                         try {
-                            await job.prime();
+                            const box = await job.prime();
 
                             ws.send(
                                 JSON.stringify({
@@ -221,7 +182,7 @@ router.ws('/connect', async (ws, req) => {
                                 })
                             );
 
-                            await job.execute(event_bus);
+                            await job.execute(box, event_bus);
                         } catch (error) {
                             logger.error(
                                 `Error cleaning up job: ${job.uuid}:\n${error}`
@@ -248,7 +209,9 @@ router.ws('/connect', async (ws, req) => {
                     break;
                 case 'signal':
                     if (job !== null) {
-                        if (SIGNALS.includes(msg.signal)) {
+                        if (
+                            Object.values(globals.SIGNALS).includes(msg.signal)
+                        ) {
                             event_bus.emit('signal', msg.signal);
                         } else {
                             ws.close(4005, 'Invalid signal');
@@ -279,9 +242,9 @@ router.post('/execute', async (req, res) => {
         return res.status(400).json(error);
     }
     try {
-        await job.prime();
+        const box = await job.prime();
 
-        let result = await job.execute();
+        let result = await job.execute(box);
         // Backward compatibility when the run stage is not started
         if (result.run === undefined) {
             result.run = result.compile;

diff --git a/api/src/config.js b/api/src/config.js
@@ -90,6 +90,18 @@ const options = {
         parser: parse_int,
         validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
     },
+    compile_cpu_time: {
+        desc: 'Max CPU time allowed for compile stage in milliseconds',
+        default: 10000, // 10 seconds
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    run_cpu_time: {
+        desc: 'Max CPU time allowed for run stage in milliseconds',
+        default: 3000, // 3 seconds
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
     compile_memory_limit: {
         desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)',
         default: -1, // no limit
@@ -117,7 +129,7 @@ const options = {
     limit_overrides: {
         desc: 'Per-language exceptions in JSON format for each of:\
         max_process_count, max_open_files, max_file_size, compile_memory_limit,\
-        run_memory_limit, compile_timeout, run_timeout, output_max_size',
+        run_memory_limit, compile_timeout, run_timeout, compile_cpu_time, run_cpu_time, output_max_size',
         default: {},
         parser: parse_overrides,
         validators: [
@@ -165,6 +177,8 @@ function parse_overrides(overrides_string) {
                     'run_memory_limit',
                     'compile_timeout',
                     'run_timeout',
+                    'compile_cpu_time',
+                    'run_cpu_time',
                     'output_max_size',
                 ].includes(key)
             ) {

diff --git a/api/src/docker-entrypoint.sh b/api/src/docker-entrypoint.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cd /sys/fs/cgroup && \
+mkdir isolate/ && \
+echo 1 > isolate/cgroup.procs && \
+echo '+cpuset +cpu +io +memory +pids' > cgroup.subtree_control && \
+cd isolate && \
+mkdir init && \
+echo 1 > init/cgroup.procs && \
+echo '+cpuset +memory' > cgroup.subtree_control && \
+echo "Initialized cgroup" && \
+chown -R piston:piston /piston && \
+exec su -- piston -c 'ulimit -n 65536 && node /piston_api/src'
diff --git a/api/src/globals.js b/api/src/globals.js
@@ -7,14 +7,78 @@ const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs
     .split('\n')
     .find(x => x.startsWith('ID'))
     .replace('ID=', '')}`;
+const SIGNALS = {
+    1: 'SIGHUP',
+    2: 'SIGINT',
+    3: 'SIGQUIT',
+    4: 'SIGILL',
+    5: 'SIGTRAP',
+    6: 'SIGABRT',
+    7: 'SIGBUS',
+    8: 'SIGFPE',
+    9: 'SIGKILL',
+    10: 'SIGUSR1',
+    11: 'SIGSEGV',
+    12: 'SIGUSR2',
+    13: 'SIGPIPE',
+    14: 'SIGALRM',
+    15: 'SIGTERM',
+    16: 'SIGSTKFLT',
+    17: 'SIGCHLD',
+    18: 'SIGCONT',
+    19: 'SIGSTOP',
+    20: 'SIGTSTP',
+    21: 'SIGTTIN',
+    22: 'SIGTTOU',
+    23: 'SIGURG',
+    24: 'SIGXCPU',
+    25: 'SIGXFSZ',
+    26: 'SIGVTALRM',
+    27: 'SIGPROF',
+    28: 'SIGWINCH',
+    29: 'SIGIO',
+    30: 'SIGPWR',
+    31: 'SIGSYS',
+    34: 'SIGRTMIN',
+    35: 'SIGRTMIN+1',
+    36: 'SIGRTMIN+2',
+    37: 'SIGRTMIN+3',
+    38: 'SIGRTMIN+4',
+    39: 'SIGRTMIN+5',
+    40: 'SIGRTMIN+6',
+    41: 'SIGRTMIN+7',
+    42: 'SIGRTMIN+8',
+    43: 'SIGRTMIN+9',
+    44: 'SIGRTMIN+10',
+    45: 'SIGRTMIN+11',
+    46: 'SIGRTMIN+12',
+    47: 'SIGRTMIN+13',
+    48: 'SIGRTMIN+14',
+    49: 'SIGRTMIN+15',
+    50: 'SIGRTMAX-14',
+    51: 'SIGRTMAX-13',
+    52: 'SIGRTMAX-12',
+    53: 'SIGRTMAX-11',
+    54: 'SIGRTMAX-10',
+    55: 'SIGRTMAX-9',
+    56: 'SIGRTMAX-8',
+    57: 'SIGRTMAX-7',
+    58: 'SIGRTMAX-6',
+    59: 'SIGRTMAX-5',
+    60: 'SIGRTMAX-4',
+    61: 'SIGRTMAX-3',
+    62: 'SIGRTMAX-2',
+    63: 'SIGRTMAX-1',
+    64: 'SIGRTMAX',
+};
 
 module.exports = {
     data_directories: {
         packages: 'packages',
-        jobs: 'jobs',
     },
     version: require('../package.json').version,
     platform,
     pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed
     clean_directories: ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'],
+    SIGNALS,
 };
diff --git a/api/src/index.js b/api/src/index.js
@@ -35,10 +35,6 @@ expressWs(app);
             }
         }
     });
-    fss.chmodSync(
-        path.join(config.data_directory, globals.data_directories.jobs),
-        0o711
-    );
 
     logger.info('Loading packages');
     const pkgdir = path.join(