Developing inside Docker with VS Code remote containers

.devcontainer/devcontainer.json

@@ -0,0 +1,56 @@
+{
+
+	"name": "ebayKleinanzeigen Dockerfile",
+
+	// Sets the run context to one level up instead of the .devcontainer folder.
+	"context": "..",
+
+	// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+	"dockerFile": "../Dockerfile",
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"terminal.integrated.shell.linux": null
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance"
+	],
+
+	// settings to run firefox inside of the docker container
+	// https://medium.com/better-programming/running-desktop-apps-in-docker-43a70a5265c4
+	// Ubuntu: do not forget to execute `xhosts +` on host (refer to above link for other OSs)
+	"containerEnv": {
+		"DISPLAY": "${localEnv:DISPLAY}"
+	},
+	// TODO works in Ubuntu. probably different if running under Windows/Mac OS
+	"mounts": ["source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind"],
+
+	// https://stelligent.com/2020/05/29/development-acceleration-through-vs-code-remote-containers-how-we-leverage-vs-code-remote-containers-for-rapid-development-of-cfn_nag/
+	// ...:ro stands for read-only
+	"runArgs": [
+		// SSH
+		// TODO better to use ${remoteEnv:HOME}, does not work yet
+		"-v", "${localEnv:HOME}/.ssh:/home/container-dev/.ssh:ro",
+		// GPG
+		// TODO this does not work yet
+		//"-v", "${localEnv:HOME}/.gnupg/private-keys-v1.d:/home/container-dev/.gnupg/private-keys-v1.d:ro",
+		//"-v", "${localEnv:HOME}/.gnupg/pubring.kbx:/home/container-dev/.gnupg/pubring.kbx:ro",
+		//"-v", "${localEnv:HOME}/.gnupg/trustdb.gpg:/home/container-dev/.gnupg/trustdb.gpg:ro"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line to run commands after the container is created - for example installing curl.
+	// "postCreateCommand": "apt-get update && apt-get install -y curl",
+
+	// Uncomment when using a ptrace-based debugger like C++, Go, and Rust
+	// "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
+
+	// Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker.
+	// "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ],
+	// Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
+	// "remoteUser": "vscode"
+}

.gitignore

@@ -64,4 +64,4 @@ config.json
 # VSC history extension
 .history
 # pictures for the ads
-data
+data

.vscode/launch.json

@@ -0,0 +1,19 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "kleinanzeigen.py",
+            "type": "python",
+            "request": "launch",
+            "program": "kleinanzeigen.py",
+            "console": "integratedTerminal",
+            "args": [
+                "--profile", "../data/config.json"
+            ],
+            "cwd": "${workspaceFolder}/src"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.languageServer": "Pylance",
+    "python.linting.enabled": true,
+    "python.linting.pylintEnabled": true,
+    // not using venv since running inside docker container
+    "python.pythonPath": "/usr/bin/python3", 
+}

Dockerfile

@@ -0,0 +1,32 @@
+FROM ubuntu:18.04
+
+RUN apt-get update
+
+RUN apt-get -y install wget bash zip rsync python3.6 python3-pip \ 
+    build-essential git vim
+RUN python3 -m pip install --upgrade pip==20.3.1
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+COPY src/requirements.txt /tmp/pip-tmp/
+RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+    && rm -rf /tmp/pip-tmp/requirements.txt
+
+RUN apt -y install firefox
+RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz \
+    && tar xzf geckodriver-v0.26.0-linux64.tar.gz && rm geckodriver-v0.26.0-linux64.tar.gz \
+    && mv geckodriver /usr/bin/geckodriver
+
+# adding non-root user as described in 
+# https://stelligent.com/2020/05/29/development-acceleration-through-vs-code-remote-containers-how-we-leverage-vs-code-remote-containers-for-rapid-development-of-cfn_nag/
+ARG USERNAME=container-dev
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+RUN groupadd --gid $USER_GID $USERNAME \
+  && useradd --uid $USER_UID --gid $USER_GID --shell /bin/bash -m $USERNAME \
+  && apt-get install -y sudo \
+  && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+  && chmod 0440 /etc/sudoers.d/$USERNAME
+
+RUN chown -R $USERNAME /usr/bin/geckodriver
+
+USER $USERNAME

README.md

@@ -1,11 +1,5 @@
 # ebayKleinanzeigen
 
-## Prerequisites
-
-* config.json (from config.json.example)
-* geckodriver (to /usr/local/bin): <https://github.com/mozilla/geckodriver/releases>
-* selenium: ```pip install selenium```
-
 ## Features
 
 * **New** Upload all images from a set subdirectory
@@ -22,7 +16,61 @@
 
 * Uploads multiple photos
 
-## Installation Guide (Ubuntu)
+## Project structure
+
+* `ebayKleinanzeigen`
+  * `src`: python code
+  * `data`: place for custom `config.json`, `.log` files are saved here
+
+## Installation
+
+There are two possible ways to run the project:
+
+   1. Install all the requirements locally: python version and relevant libraries, firefox, geckodriver for selenium
+   2. Work with a self-contained dockerized version of the project
+The sections below describe installation steps for both approaches
+
+### Installation Guide: Dockerized project
+
+#### *Writing code INSIDE container*
+
+* Install [Docker](https://docs.docker.com/get-docker/)
+
+The easiest way to work with the project is by using *VS Code* and its *Remote Containers extension*. This approach, unlike the following one, allows us to use the docker container both for development and running the application. All the project dependencies (python version and libraries, firefox and geckodriver installation) as well as VS Code set-up are taken care of by the container set-up. 
+
+* Install [VS Code](https://code.visualstudio.com/download)
+* Install [VS Code Remote Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension
+* Checkout the project with `git clone`
+* Enable access to your host X server
+  * Ubuntu: execute `xhosts +` on the host
+  * Windows / Mac OS: not tested, refer to this [source](https://medium.com/better-programming/running-desktop-apps-in-docker-43a70a5265c4)
+    * further configurations in `devcontainer.json` will likely be needed
+* Open the project in container and start development
+  * ![Opening project in VS Code remote container](https://microsoft.github.io/vscode-remote-release/images/remote-containers-readme.gif)
+  * `launch.json` already defines one debug configuration that can be used right away
+  * Docker image has Firefox and geckdriver installed, they will be spun off from the container and shown on the host
+
+#### *Writing code on the host but running it inside Docker*
+
+ Alternatively (not a well tested option), one can start a docker container with its volumes linked to the project directory on the host. This way, all the code editing happens on the host and the changes are mirrored back in the container. After editing is done:
+
+* `cd` to project folder
+* Build image from the `Dockerfile`
+  * `docker build -t ebaykleinanzeigen:1.0 .`
+* Start up container in interactive mode
+  * `docker run -it --workdir="/app" -e DISPLAY=$DISPLAY --mount=source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind -v $(pwd):/app --name ebaykleinanzeigen ebaykleinanzeigen:1.0`
+* inside the container:
+  * `cd src`
+  * `python3 kleinanzeigen.py --profile=../data/config.json` (`config.json`) should be there
+* Debugging while inside docker: IDK...
+
+### Installation Guide: Prerequisites for **not Dockerized** development environments
+
+* config.json (from config.json.example)
+* geckodriver (to /usr/local/bin): <https://github.com/mozilla/geckodriver/releases>
+* selenium: ```pip install selenium```
+
+### Installation Guide: Ubuntu
 
 1. Install Python 3 and PIP
 
@@ -74,7 +122,7 @@
 
 Now a browser window should start, login and fill out the fields automatically.
 
-## Installation Guide (MacOS, Tested on catalina)
+### Installation Guide: MacOS, Tested on catalina
 
 ```bash
 # create new virtual env, for instance with conda
@@ -98,7 +146,7 @@ cp config.json.example config.json
 
 Now a browser window should start, login and fill the fields automatically.
 
-## Installation Guide (Windows)
+### Installation Guide: Windows
 
 1. Download and install Python 3 for Windows from <https://www.python.org/downloads/>
 
@@ -161,4 +209,4 @@ Now a browser window should start, login and fill the fields automatically.
 
 * @therealsupermario - Description Files, ad-level zip codes, custom update interval, support for additional category fields
 
-* @denisergashbaev - python 3.6 fixes, README.md, running from VS Code
+* @denisergashbaev - Dockerization for VS Code with remote containers extension

config.json.example → src/config.example.json

@@ -33,6 +33,6 @@
           "foto.art_s": "Zubehör",
           "foto.condition_s": "Gebraucht"
       }
-    },
+    }
   ]
 }

kleinanzeigen.py → src/kleinanzeigen.py

@@ -13,6 +13,7 @@
 import json
 import logging
 import os
+from os import path
 import signal
 import sys
 import time
@@ -31,7 +32,7 @@
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
-fh = logging.FileHandler('kleinanzeigen.log')
+fh = logging.FileHandler(path.join('..', 'data', f"{path.basename(__file__).split('.')[0]}.log"))
 fh.setLevel(logging.DEBUG)
 
 ch = logging.StreamHandler()
@@ -428,7 +429,7 @@ def session_create(config):
     if config.get('webdriver_enabled') is False:
         options.set_preference("dom.webdriver.enabled", False)
 
-    driver = webdriver.Firefox(options=options)
+    driver = webdriver.Firefox(options=options, service_log_path=path.join("..", "data","geckodriver.log"))
 
     log.info("New session is: %s %s" % (driver.session_id, driver.command_executor._url))
 
@@ -492,7 +493,7 @@ def signal_handler(sig, frame):
         log.info("Handling '%s'" % ad["title"])
 
         if "date_updated" in ad:
-            # python < 3.7 do not support datetime.datetime_fromisoformat()
+            # python < 3.7 does not support datetime.datetime_fromisoformat()
             # https://stackoverflow.com/a/60852111/256002
             if int(python_version_tuple()[1]) < 7:
                 from backports.datetime_fromisoformat import MonkeyPatch

requirements.txt → src/requirements.txt

@@ -3,7 +3,6 @@ backports-datetime-fromisoformat==1.0.0
 isort==5.6.4
 lazy-object-proxy==1.4.3
 mccabe==0.6.1
-pkg-resources==0.0.0
 pylint==2.6.0
 python-dateutil==2.8.1
 selenium==3.141.0