Merge branch 'integration' of github.com:cisagov/XFD into JD-add-requ…

…est-field-to-response

.github/workflows/backend.yml

@@ -117,28 +117,8 @@ jobs:
           DB_USERNAME: crossfeed
           DB_PASSWORD: password
 
-  build_worker:
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '18'
-      - name: Restore npm cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
-          restore-keys: ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: npm ci
-      - name: Build worker container
-        run: npm run build-worker
-        working-directory: ./backend
-
   deploy_staging:
-    needs: [build_worker, lint, test, test_worker]
+    needs: [lint, test, test_worker]
     runs-on: ubuntu-latest
     environment: staging
     concurrency: '1'
@@ -228,19 +208,12 @@ jobs:
           SLS_DEBUG: '*'
 
       - name: Deploy backend
-        run: npx sls deploy --stage=integration
+        run: npx sls deploy --stage=integration --force
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
-      # - name: Deploy worker
-      #   run: npm run deploy-worker-integration
-      #   working-directory: backend
-      #   env:
-      #     AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      #     AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
       - name: Run syncdb
         run: |
           aws lambda invoke --function-name crossfeed-integration-syncdb \

backend/Dockerfile.worker

@@ -1,84 +1,34 @@
-FROM node:18-alpine3.17 as build
-USER root
-
-RUN apk update && apk upgrade
-
-WORKDIR /app
-
-COPY ./package* ./
-
-RUN npm install -g npm@9
-
-RUN npm ci
-
-COPY tsconfig.json ./tsconfig.json
-COPY webpack.worker.config.js ./webpack.worker.config.js
-COPY mock.js ./mock.js
-COPY src ./src
-
-RUN npx webpack --config webpack.worker.config.js
-
-FROM golang:1.19-alpine as deps
-
-RUN apk update && apk upgrade
+# Base image for Python environment
+FROM python:3.10-bullseye
 
 WORKDIR /app
 
-RUN apk add --no-cache curl unzip musl-dev
-
-RUN curl -4LO http://github.com/Findomain/Findomain/releases/latest/download/findomain-linux.zip
-RUN unzip findomain-linux.zip && chmod +x findomain && cp findomain /usr/bin/findomain
-
-RUN go mod init crossfeed-worker
-
-RUN go install github.com/facebookincubator/nvdtools/...@latest
-RUN go install -v github.com/owasp-amass/amass/v3/...@master
-
-FROM ruby:alpine as rubyBuild
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    wget \
+    unzip \
+    build-essential \
+    netcat \
+    --no-install-recommends && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
-RUN apk add --update --no-cache build-base git ruby ruby-dev openssl-dev
+# Install Python dependencies from worker and application
+COPY worker/requirements.txt /app/worker/requirements.txt
+COPY requirements.txt /app/requirements.txt
 
-RUN gem install bundler:2.3.21
-RUN export RUBY_VERSION=$(ruby -e "print RUBY_VERSION") && git clone https://github.com/intrigueio/intrigue-ident.git && cd intrigue-ident && git checkout ee119abeac20564e728a92ab786400126e7a97f0 && sed -i "s/2.7.2/$RUBY_VERSION/g" Gemfile && sed -i "s/2.7.2p114/$RUBY_VERSION/g" Gemfile.lock && bundle install --jobs=4
-RUN echo 'cd /app/intrigue-ident && bundle exec ruby ./util/ident.rb $@' > /usr/bin/intrigue-ident && chmod +x /usr/bin/intrigue-ident
-
-FROM node:18-bullseye
-
-#RUN apt update && apt upgrade -y && apt install zip -y
-RUN apt update && apt install wget build-essential libreadline-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev zip git -y
-WORKDIR /app
-
-RUN npm install -g pm2@5 wait-port@1
-RUN wget -c https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tar.xz && tar -Jxvf Python-3.10.11.tar.xz
-
-RUN cd Python-3.10.11 && ./configure && make -j4 && make altinstall
-RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.10 1
-RUN update-alternatives --install /usr/bin/pip pip /usr/local/bin/pip3.10 1
-RUN pip3.10 install --upgrade pip
-
-RUN apt remove dav1d && apt autoclean && apt autoremove
-
-# Python dependencies
-
-COPY worker/requirements.txt worker/requirements.txt
-
-RUN pip install -r worker/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/worker/requirements.txt
 
+# Copy worker scripts and Django project
 COPY worker worker
+COPY src/xfd_django .
 
-RUN wget https://publicsuffix.org/list/public_suffix_list.dat --no-use-server-timestamps
-
-COPY --from=build /app/dist/worker.bundle.js worker.bundle.js
-
-COPY --from=deps /usr/bin/findomain /usr/bin/
-COPY --from=deps /go/bin/amass /usr/bin/
-COPY --from=deps /go/bin/csv2cpe /go/bin/nvdsync /go/bin/cpe2cve /usr/bin/
-
-COPY --from=deps /etc/ssl/certs /etc/ssl/certs
-
-COPY --from=rubyBuild /usr/bin/intrigue-ident  /usr/bin/
+# Set environment variables
+ENV DJANGO_SETTINGS_MODULE=xfd_django.settings
+ENV PYTHONPATH="/app:$PYTHONPATH"
 
-ENV GLOBAL_AGENT_HTTP_PROXY=http://localhost:8080
-ENV GLOBAL_AGENT_NO_PROXY=censys.io
+# Make the entry script executable
+RUN chmod +x worker/worker-entry.sh
 
-CMD ["./worker/worker-entry.sh"]
+# Default entrypoint
+CMD ["worker/worker-entry.sh"]

backend/Makefile

@@ -6,4 +6,4 @@ syncdb:
 
 pytest:
 	docker compose exec backend python manage.py makemigrations
-	docker compose exec backend pytest --no-migrations --reuse-db $(FILE)
+	docker compose exec backend pytest --no-migrations --reuse-db $(FILE)