Add job start and end time

.github/workflows/test.yml

@@ -0,0 +1,154 @@
+name: Scapyd-k8s CI
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test-unit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-test.txt
+
+      - name: Run tests
+        run: pytest -vv --color=yes scrapyd_k8s/tests/unit/
+
+  test-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-test.txt
+
+      - name: Pull example spider
+        run: docker pull ghcr.io/q-m/scrapyd-k8s-spider-example
+
+      - name: Run scrapyd-k8s
+        run: |
+          cp scrapyd_k8s.sample-docker.conf scrapyd_k8s.conf
+          python -m scrapyd_k8s &
+          while ! nc -q 1 localhost 6800 </dev/null; do sleep 1; done
+          curl http://localhost:6800/daemonstatus.json
+
+      - name: Run tests
+        run: pytest -vv --color=yes scrapyd_k8s/tests/integration/
+
+  test-manifest:
+    container:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-test.txt
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build container
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: false
+          load: true
+          tags: test:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Start minikube
+        uses: medyagh/setup-minikube@master
+
+      - name: Deploy to minikube
+        run: |
+          minikube image load test:latest
+          # already pull image so we don't have to wait for it later
+          minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
+          # load manifest
+          sed -i 's/\(imagePullPolicy:\s*\)\w\+/\1Never/' kubernetes.yaml
+          sed -i 's/\(image:\s*\)ghcr\.io\/q-m\/scrapyd-k8s:/\1test:/' kubernetes.yaml
+          sed -i 's/\(type:\s*\)ClusterIP/\1NodePort/' kubernetes.yaml
+          kubectl create -f kubernetes.yaml
+          # and wait for scrapyd-k8s to become ready
+          kubectl wait --for=condition=Available deploy/scrapyd-k8s --timeout=60s
+          curl --retry 10 --retry-delay 2 --retry-all-errors `minikube service scrapyd-k8s --url`/daemonstatus.json
+
+      - name: Run tests
+        run: |
+          TEST_WITH_K8S=1 \
+          TEST_BASE_URL=`minikube service scrapyd-k8s --url` \
+          TEST_MAX_WAIT=60 \
+          TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
+          pytest -vv --color=yes scrapyd_k8s/tests/integration/
+  test-k8s:
+    container:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-test.txt
+
+      - name: Start minikube
+        uses: medyagh/setup-minikube@master
+
+      - name: Prepare Kubernetes environment
+        run: |
+          kubectl create secret generic example-env-secret --from-literal=FOO_1=bar
+          kubectl create configmap example-env-configmap --from-literal=FOO_2=baz
+          # already pull image so we don't have to wait for it later
+          minikube image pull ghcr.io/q-m/scrapyd-k8s-spider-example:latest
+
+      - name: Run scrapyd-k8s
+        run: |
+          cp scrapyd_k8s.sample-k8s.conf scrapyd_k8s.conf
+          python -m scrapyd_k8s &
+          while ! nc -q 1 localhost 6800 </dev/null; do sleep 1; done
+          curl http://localhost:6800/daemonstatus.json
+
+      - name: Run tests
+        run: |
+          TEST_WITH_K8S=1 \
+          TEST_MAX_WAIT=60 \
+          TEST_AVAILABLE_VERSIONS=latest,`skopeo list-tags docker://ghcr.io/q-m/scrapyd-k8s-spider-example | jq -r '.Tags | map(select(. != "latest" and (startswith("sha-") | not))) | join(",")'` \
+          pytest -vv --color=yes scrapyd_k8s/tests/integration/

README.md

@@ -151,7 +151,7 @@ curl http://localhost:6800/listjobs.json
 {
   "finished":[],
   "pending":[],
-  "running":[{"id":"e9b81fccbec211eeb3b109f30f136c01","project":"example","spider":"quotes","state":"running"}],
+  "running":[{"id":"e9b81fccbec211eeb3b109f30f136c01","project":"example","spider":"quotes","state":"running", "start_time":"2012-09-12 10:14:03.594664", "end_time":null}],
   "status":"ok"
 }
 ```
@@ -228,6 +228,7 @@ Lists spiders from the spider image's `org.scrapy.spiders` label.
 ### `listjobs.json` ([➽](https://scrapyd.readthedocs.io/en/latest/api.html#listjobs-json))
 
 Lists current jobs by looking at Docker containers or Kubernetes jobs.
+Note that `end_time` is not yet supported for Docker.
 
 ### ~~`delversion.json`~~ ([➽](https://scrapyd.readthedocs.io/en/latest/api.html#delversion-json))
 

scrapyd_k8s/launcher/docker.py

@@ -3,7 +3,7 @@
 import logging
 
 import docker
-from ..utils import native_stringify_dict
+from ..utils import format_iso_date_string, native_stringify_dict
 
 logger = logging.getLogger(__name__)
 
@@ -72,11 +72,14 @@ def enable_joblogs(self, config):
         logger.warning("Job logs are not supported when using the Docker launcher.")
 
     def _parse_job(self, c):
+        state = self._docker_to_scrapyd_status(c.status)
         return {
             'id': c.labels.get(self.LABEL_JOB_ID),
-            'state': self._docker_to_scrapyd_status(c.status),
+            'state': state,
             'project': c.labels.get(self.LABEL_PROJECT),
-            'spider': c.labels.get(self.LABEL_SPIDER)
+            'spider': c.labels.get(self.LABEL_SPIDER),
+            'start_time': format_iso_date_string(c.attrs['State']['StartedAt']) if state in ['running', 'finished'] else None,
+            'end_time': None,  # Not available using Docker's API. Add to the job representation to keep it the same as K8s jobs listing.
         }
 
     def _get_container(self, project_id, job_id):

scrapyd_k8s/launcher/k8s.py

@@ -4,7 +4,7 @@
 import kubernetes.stream
 from signal import Signals
 
-from ..utils import native_stringify_dict
+from ..utils import format_datetime_object, native_stringify_dict
 from scrapyd_k8s.joblogs import joblogs_init
 
 class K8s:
@@ -125,11 +125,14 @@ def enable_joblogs(self, config):
         joblogs_init(config)
 
     def _parse_job(self, job):
+        state = self._k8s_job_to_scrapyd_status(job)
         return {
             'id': job.metadata.labels.get(self.LABEL_JOB_ID),
-            'state': self._k8s_job_to_scrapyd_status(job),
+            'state': state,
             'project': job.metadata.labels.get(self.LABEL_PROJECT),
-            'spider': job.metadata.labels.get(self.LABEL_SPIDER)
+            'spider': job.metadata.labels.get(self.LABEL_SPIDER),
+            'start_time': format_datetime_object(job.status.start_time) if state in ['running', 'finished'] else None,
+            'end_time': format_datetime_object(job.status.completion_time) if job.status.completion_time and state == 'finished' else None,
         }
 
     def _get_job(self, project, job_id):