diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..969d288 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,56 @@ +{ + + "name": "ebayKleinanzeigen Dockerfile", + + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerFile": "../Dockerfile", + + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": null + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + + // settings to run firefox inside of the docker container + // https://medium.com/better-programming/running-desktop-apps-in-docker-43a70a5265c4 + // Ubuntu: do not forget to execute `xhosts +` on host (refer to above link for other OSs) + "containerEnv": { + "DISPLAY": "${localEnv:DISPLAY}" + }, + // TODO works in Ubuntu. probably different if running under Windows/Mac OS + "mounts": ["source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind"], + + // https://stelligent.com/2020/05/29/development-acceleration-through-vs-code-remote-containers-how-we-leverage-vs-code-remote-containers-for-rapid-development-of-cfn_nag/ + // ...:ro stands for read-only + "runArgs": [ + // SSH + // TODO better to use ${remoteEnv:HOME}, does not work yet + "-v", "${localEnv:HOME}/.ssh:/home/container-dev/.ssh:ro", + // GPG + // TODO this does not work yet + //"-v", "${localEnv:HOME}/.gnupg/private-keys-v1.d:/home/container-dev/.gnupg/private-keys-v1.d:ro", + //"-v", "${localEnv:HOME}/.gnupg/pubring.kbx:/home/container-dev/.gnupg/pubring.kbx:ro", + //"-v", "${localEnv:HOME}/.gnupg/trustdb.gpg:/home/container-dev/.gnupg/trustdb.gpg:ro" + ], + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line to run commands after the container is created - for example installing curl. + // "postCreateCommand": "apt-get update && apt-get install -y curl", + + // Uncomment when using a ptrace-based debugger like C++, Go, and Rust + // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], + + // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker. + // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], + // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" +} diff --git a/.gitignore b/.gitignore index 1bfcdbb..239a9ba 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,4 @@ config.json # VSC history extension .history # pictures for the ads -data \ No newline at end of file +data diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..09bcb2a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "kleinanzeigen.py", + "type": "python", + "request": "launch", + "program": "kleinanzeigen.py", + "console": "integratedTerminal", + "args": [ + "--profile", "../data/config.json" + ], + "cwd": "${workspaceFolder}/src" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..8f87b54 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.languageServer": "Pylance", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + // not using venv since running inside docker container + "python.pythonPath": "/usr/bin/python3", +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3cbc91c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:18.04 + +RUN apt-get update + +RUN apt-get -y install wget bash zip rsync python3.6 python3-pip \ + build-essential git vim +RUN python3 -m pip install --upgrade pip==20.3.1 + +# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +COPY src/requirements.txt /tmp/pip-tmp/ +RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ + && rm -rf /tmp/pip-tmp/requirements.txt + +RUN apt -y install firefox +RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz \ + && tar xzf geckodriver-v0.26.0-linux64.tar.gz && rm geckodriver-v0.26.0-linux64.tar.gz \ + && mv geckodriver /usr/bin/geckodriver + +# adding non-root user as described in +# https://stelligent.com/2020/05/29/development-acceleration-through-vs-code-remote-containers-how-we-leverage-vs-code-remote-containers-for-rapid-development-of-cfn_nag/ +ARG USERNAME=container-dev +ARG USER_UID=1000 +ARG USER_GID=$USER_UID +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID --shell /bin/bash -m $USERNAME \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +RUN chown -R $USERNAME /usr/bin/geckodriver + +USER $USERNAME diff --git a/README.md b/README.md index 6601ed6..d849212 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,5 @@ # ebayKleinanzeigen -## Prerequisites - -* config.json (from config.json.example) -* geckodriver (to /usr/local/bin): -* selenium: ```pip install selenium``` - ## Features * **New** Upload all images from a set subdirectory @@ -22,7 +16,61 @@ * Uploads multiple photos -## Installation Guide (Ubuntu) +## Project structure + +* `ebayKleinanzeigen` + * `src`: python code + * `data`: place for custom `config.json`, `.log` files are saved here + +## Installation + +There are two possible ways to run the project: + + 1. Install all the requirements locally: python version and relevant libraries, firefox, geckodriver for selenium + 2. Work with a self-contained dockerized version of the project +The sections below describe installation steps for both approaches + +### Installation Guide: Dockerized project + +#### *Writing code INSIDE container* + +* Install [Docker](https://docs.docker.com/get-docker/) + +The easiest way to work with the project is by using *VS Code* and its *Remote Containers extension*. This approach, unlike the following one, allows us to use the docker container both for development and running the application. All the project dependencies (python version and libraries, firefox and geckodriver installation) as well as VS Code set-up are taken care of by the container set-up. + +* Install [VS Code](https://code.visualstudio.com/download) +* Install [VS Code Remote Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension +* Checkout the project with `git clone` +* Enable access to your host X server + * Ubuntu: execute `xhosts +` on the host + * Windows / Mac OS: not tested, refer to this [source](https://medium.com/better-programming/running-desktop-apps-in-docker-43a70a5265c4) + * further configurations in `devcontainer.json` will likely be needed +* Open the project in container and start development + * ![Opening project in VS Code remote container](https://microsoft.github.io/vscode-remote-release/images/remote-containers-readme.gif) + * `launch.json` already defines one debug configuration that can be used right away + * Docker image has Firefox and geckdriver installed, they will be spun off from the container and shown on the host + +#### *Writing code on the host but running it inside Docker* + + Alternatively (not a well tested option), one can start a docker container with its volumes linked to the project directory on the host. This way, all the code editing happens on the host and the changes are mirrored back in the container. After editing is done: + +* `cd` to project folder +* Build image from the `Dockerfile` + * `docker build -t ebaykleinanzeigen:1.0 .` +* Start up container in interactive mode + * `docker run -it --workdir="/app" -e DISPLAY=$DISPLAY --mount=source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind -v $(pwd):/app --name ebaykleinanzeigen ebaykleinanzeigen:1.0` +* inside the container: + * `cd src` + * `python3 kleinanzeigen.py --profile=../data/config.json` (`config.json`) should be there +* Debugging while inside docker: IDK... + +### Installation Guide: Prerequisites for **not Dockerized** development environments + +* config.json (from config.json.example) +* geckodriver (to /usr/local/bin): +* selenium: ```pip install selenium``` + +### Installation Guide: Ubuntu 1. Install Python 3 and PIP @@ -74,7 +122,7 @@ Now a browser window should start, login and fill out the fields automatically. -## Installation Guide (MacOS, Tested on catalina) +### Installation Guide: MacOS, Tested on catalina ```bash # create new virtual env, for instance with conda @@ -98,7 +146,7 @@ cp config.json.example config.json Now a browser window should start, login and fill the fields automatically. -## Installation Guide (Windows) +### Installation Guide: Windows 1. Download and install Python 3 for Windows from @@ -161,4 +209,4 @@ Now a browser window should start, login and fill the fields automatically. * @therealsupermario - Description Files, ad-level zip codes, custom update interval, support for additional category fields -* @denisergashbaev - python 3.6 fixes, README.md, running from VS Code +* @denisergashbaev - Dockerization for VS Code with remote containers extension \ No newline at end of file diff --git a/categories.ods b/src/categories.ods similarity index 100% rename from categories.ods rename to src/categories.ods diff --git a/config.json.example b/src/config.example.json similarity index 99% rename from config.json.example rename to src/config.example.json index ee82146..023c475 100644 --- a/config.json.example +++ b/src/config.example.json @@ -33,6 +33,6 @@ "foto.art_s": "Zubehör", "foto.condition_s": "Gebraucht" } - }, + } ] } diff --git a/kleinanzeigen.py b/src/kleinanzeigen.py similarity index 98% rename from kleinanzeigen.py rename to src/kleinanzeigen.py index 6729a37..4a8c78c 100755 --- a/kleinanzeigen.py +++ b/src/kleinanzeigen.py @@ -13,6 +13,7 @@ import json import logging import os +from os import path import signal import sys import time @@ -31,7 +32,7 @@ log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) -fh = logging.FileHandler('kleinanzeigen.log') +fh = logging.FileHandler(path.join('..', 'data', f"{path.basename(__file__).split('.')[0]}.log")) fh.setLevel(logging.DEBUG) ch = logging.StreamHandler() @@ -428,7 +429,7 @@ def session_create(config): if config.get('webdriver_enabled') is False: options.set_preference("dom.webdriver.enabled", False) - driver = webdriver.Firefox(options=options) + driver = webdriver.Firefox(options=options, service_log_path=path.join("..", "data","geckodriver.log")) log.info("New session is: %s %s" % (driver.session_id, driver.command_executor._url)) @@ -492,7 +493,7 @@ def signal_handler(sig, frame): log.info("Handling '%s'" % ad["title"]) if "date_updated" in ad: - # python < 3.7 do not support datetime.datetime_fromisoformat() + # python < 3.7 does not support datetime.datetime_fromisoformat() # https://stackoverflow.com/a/60852111/256002 if int(python_version_tuple()[1]) < 7: from backports.datetime_fromisoformat import MonkeyPatch diff --git a/requirements.txt b/src/requirements.txt similarity index 91% rename from requirements.txt rename to src/requirements.txt index 990b37a..975367e 100644 --- a/requirements.txt +++ b/src/requirements.txt @@ -3,7 +3,6 @@ backports-datetime-fromisoformat==1.0.0 isort==5.6.4 lazy-object-proxy==1.4.3 mccabe==0.6.1 -pkg-resources==0.0.0 pylint==2.6.0 python-dateutil==2.8.1 selenium==3.141.0 diff --git a/sandbox.py b/src/sandbox.py similarity index 100% rename from sandbox.py rename to src/sandbox.py