docker-compose.yml

name: trusted-crawler
services:
  app:
    container_name: trusted-crawler-main
    build:
      context: ./
      dockerfile: dockerFiles/app_docker
    depends_on:
      - mongo
      - i2p-instance
      - redis_server
      - tor-extend-1
      - tor-extend-2
      - tor-extend-3
      - tor-extend-4
      - tor-extend-5
      - tor-extend-6
      - tor-extend-7
      - tor-extend-8
      - tor-extend-9
      - tor-extend-10
    restart: always
    volumes:
      - /mnt/c/Workspace/Genesis-Crawler/app:/app
    ulimits:
      nproc: 65535
      nofile:
        soft: 26677
        hard: 46677
    env_file:
      - .env
    networks:
      - backend
      - shared_bridge
    entrypoint: [ "bash", "-c", "./start_app.sh start_app" ]
    healthcheck:
      test: ["CMD-SHELL", "exit 0"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  api:
    container_name: trusted-crawler-api
    build:
      context: ./
      dockerfile: dockerFiles/api_docker
    restart: always
    networks:
      - backend
      - shared_bridge
    env_file:
      - .env
    volumes:
      - ./app/api:/app/api:cached
    deploy:
      resources:
        limits:
          memory: 4G
    healthcheck:
      test: ["CMD-SHELL", "timeout 5 bash -c '</dev/tcp/localhost/8000' || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3

  redis_server:
    container_name: trusted-crawler-redis
    image: redis:7.4.0
    logging:
      driver: none
    command: ["redis-server", "--requirepass", "${REDIS_PASSWORD}"]
    restart: always
    volumes:
      - redis:/data
    networks:
      - backend
    env_file:
      - .env
    deploy:
      resources:
        limits:
          memory: 2G
    healthcheck:
      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
      interval: 10s
      timeout: 5s
      retries: 3

  mongo:
    image: mongo:latest
    container_name: trustly-crawler-mongodb
    restart: always
    environment:
      MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USERNAME}
      MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD}
    ports:
      - "27019:27017"
    volumes:
      - redis:/data
    command: >
      bash -c "
      mongod --bind_ip_all --fork --logpath /var/log/mongodb.log &&
      sleep 10 &&
      mongosh --eval '
        db = db.getSiblingDB(\"admin\");
        if (db.getUser(\"${MONGO_ROOT_USERNAME}\") === null) {
          db.createUser({
            user: \"${MONGO_ROOT_USERNAME}\",
            pwd: \"${MONGO_ROOT_PASSWORD}\",
            roles: [{ role: \"userAdminAnyDatabase\", db: \"admin\" }, { role: \"readWriteAnyDatabase\", db: \"admin\" }]
          });
        }
      ' &&
      mongod --shutdown &&
      sleep 5 &&
      mongod --auth --bind_ip_all --fork --logpath /var/log/mongodb.log &&
      tail -f /var/log/mongodb.log
      "
    deploy:
      resources:
        limits:
          memory: 5G
    networks:
      - backend
    healthcheck:
      test: ["CMD-SHELL", "timeout 5 bash -c '</dev/tcp/localhost/27017' || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3

  celery_worker:
    container_name: trusted-crawler-celery
    build:
      context: .
      dockerfile: dockerFiles/app_docker
    command: celery -A crawler.crawler_services.celery_manager worker --loglevel=DEBUG --concurrency=${CELERY_WORKER_COUNT} --prefetch-multiplier=1 --max-tasks-per-child=3 -Q crawler_queue,unique_crawler_queue
    env_file:
      - .env
    depends_on:
      - redis_server
    restart: always
    volumes:
      - ./app:/app
    networks:
      - backend
      - shared_bridge
    deploy:
      resources:
        limits:
          memory: 8G
    healthcheck:
      test: ["CMD-SHELL", "celery -A crawler.crawler_services.celery_manager inspect ping || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3

  flower:
    container_name: trusted-crawler-flower
    build:
      context: .
      dockerfile: dockerFiles/app_docker
    command: celery -A crawler.crawler_services.celery_manager flower --port=5555 --broker=redis://:${REDIS_PASSWORD}@redis_server:6379/0 --basic_auth=${FLOWER_USERNAME}:${FLOWER_PASSWORD}
    env_file:
      - .env
    depends_on:
      - redis_server
    ports:
      - "5555:5555"
    restart: always
    networks:
      - backend
    environment:
      - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis_server:6379/0
    healthcheck:
      test: ["CMD-SHELL", "timeout 5 bash -c '</dev/tcp/localhost/5555' || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3

  tor-extend-1:
    container_name: trusted-crawler_tor_instace_1
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_1.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9152:9152"
      - "0.0.0.0:9153:9153"
    environment:
      PUID: "20002"
      PGID: "20003"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9152"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9153"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.10
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9152 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-2:
    container_name: trusted-crawler_tor_instace_2
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_2.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9154:9154"
      - "0.0.0.0:9155:9155"
    environment:
      PUID: "20004"
      PGID: "20005"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9154"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9155"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.11
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9154 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-3:
    container_name: trusted-crawler_tor_instace_3
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_3.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9156:9156"
      - "0.0.0.0:9157:9157"
    environment:
      PUID: "20006"
      PGID: "20007"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9156"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9157"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.12
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9156 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-4:
    container_name: trusted-crawler_tor_instace_4
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_4.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9158:9158"
      - "0.0.0.0:9159:9159"
    environment:
      PUID: "20008"
      PGID: "20009"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9158"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9159"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.13
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9158 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-5:
    container_name: trusted-crawler_tor_instace_5
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_5.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9160:9160"
      - "0.0.0.0:9161:9161"
    environment:
      PUID: "20010"
      PGID: "20011"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9160"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9161"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.14
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9160 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-6:
    container_name: trusted-crawler_tor_instace_6
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_6.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9162:9162"
      - "0.0.0.0:9163:9163"
    environment:
      PUID: "20012"
      PGID: "20013"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9162"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9163"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.15
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9162 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-7:
    container_name: trusted-crawler_tor_instace_7
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_7.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9164:9164"
      - "0.0.0.0:9165:9165"
    environment:
      PUID: "20014"
      PGID: "20015"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9164"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9165"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.16
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9164 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-8:
    container_name: trusted-crawler_tor_instace_8
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_8.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9166:9166"
      - "0.0.0.0:9167:9167"
    environment:
      PUID: "20016"
      PGID: "20017"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9166"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9167"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.17
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9166 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-9:
    container_name: trusted-crawler_tor_instace_9
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_9.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9168:9168"
      - "0.0.0.0:9169:9169"
    environment:
      PUID: "20018"
      PGID: "20019"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9168"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9169"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.18
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9168 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  tor-extend-10:
    container_name: trusted-crawler_tor_instace_10
    image: barneybuffet/tor:latest
    env_file:
      - .env
    volumes:
      - ./config/tor/torrc_10.txt:/etc/tor/torrc:rw

    ports:
      - "0.0.0.0:9170:9170"
      - "0.0.0.0:9171:9171"
    environment:
      PUID: "20030"
      PGID: "20031"
      TOR_CONFIG_OVERWRITE: "true"
      TOR_LOG_CONFIG: "true"
      TOR_PROXYL: "true"
      TOR_PROXY_PORT: "0.0.0.0:9170"
      TOR_PROXY_SOCKET: "true"
      TOR_PROXY_ACCEPT: 'accept 172.19.0.0/24,accept 127.0.0.1,accept 10.0.0.0/8,accept 172.16.0.0/12,accept 192.168.0.0/16'
      TOR_CONTROL: "true"
      TOR_CONTROL_PORT: "0.0.0.0:9171"
      TOR_CONTROL_SOCKET: "true"
      TOR_CONTROL_PASSWORD: ${TOR_PASSWORD}
      TOR_CONTROL_COOKIE: "true"
      TOR_RELAY: "false"
    networks:
      backend:
        ipv4_address: 172.19.0.19
    healthcheck:
      test: ["CMD-SHELL", "curl --proxy http://localhost:9170 http://example.com || exit 1"]
      interval: 120s
      timeout: 10s
      retries: 3
      start_period: 5s

  i2p-instance:
    container_name: trusted-crawler-i2p
    image: geti2p/i2p:latest
    restart: unless-stopped
    ports:
      - "7657:7657"
      - "4444:4444"
      - "4445:4445"
      - "6668:6668"
      - "7654:7654"
      - "7656:7656"
      - "12345:12345"
      - "12345:12345/udp"
    volumes:
      - ./config/i2p:/i2p/.i2p
    environment:
      - I2P_LANG=en
      - CONSOLE_AUTH=true
      - CONSOLE_USER=${I2P_USERNAME}
      - CONSOLE_PASSWORD=${I2P_PASSWORD}
    networks:
      backend:
        ipv4_address: 172.19.0.20
    healthcheck:
      test: ["CMD-SHELL", "wget -qO- http://localhost:7657 || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 5s

volumes:
  grafana-data:
  redis:
  mongo:

networks:
  shared_bridge:
    external: true
  backend:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.19.0.0/24