From c833dac726cb980ea53615b8321c87cde8048fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sim=C3=A3o=20Silva?= Date: Thu, 5 Oct 2023 01:31:41 +0100 Subject: [PATCH] Improved script to wait for elements to load --- .github/workflows/docker-build-debian.yml | 8 +- .github/workflows/pr-alpine.yml | 7 +- .github/workflows/pr-debian.yml | 11 +- Dockerfile | 32 ++++-- Dockerfile.debian | 53 ++++++--- renew.py | 128 ++++++++++++---------- renovate.json | 4 +- 7 files changed, 148 insertions(+), 95 deletions(-) diff --git a/.github/workflows/docker-build-debian.yml b/.github/workflows/docker-build-debian.yml index aab2ca88..45c6052b 100644 --- a/.github/workflows/docker-build-debian.yml +++ b/.github/workflows/docker-build-debian.yml @@ -13,7 +13,7 @@ on: env: IMAGE_NAME: "simaofsilva/noip-renewer" PIP_VERSION: "23.2.1" # renovate: datasource=pypi depName=pip versioning=pep440 - GECKO_DRIVER_VERSION: "0.33.0" # renovate: datasource=github-tags depName=mozilla/geckodriver + GECKODRIVER_VERSION: "0.33.0" # renovate: datasource=github-tags depName=mozilla/geckodriver jobs: build_debian: @@ -42,7 +42,7 @@ jobs: file: Dockerfile.debian build-args: | PIP_VERSION=${{ env.PIP_VERSION }} - GECKO_DRIVER_VERSION=${{ env.GECKO_DRIVER_VERSION }} + GECKODRIVER_VERSION=${{ env.GECKODRIVER_VERSION }} - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master @@ -70,7 +70,7 @@ jobs: file: Dockerfile.debian build-args: | PIP_VERSION=${{ env.PIP_VERSION }} - GECKO_DRIVER_VERSION=${{ env.GECKO_DRIVER_VERSION }} + GECKODRIVER_VERSION=${{ env.GECKODRIVER_VERSION }} - name: Build and push all to Docker Hub uses: docker/build-push-action@v5.0.0 @@ -84,7 +84,7 @@ jobs: file: Dockerfile.debian build-args: | PIP_VERSION=${{ env.PIP_VERSION }} - GECKO_DRIVER_VERSION=${{ env.GECKO_DRIVER_VERSION }} + GECKODRIVER_VERSION=${{ env.GECKODRIVER_VERSION }} # delete_old_runs: # runs-on: ubuntu-latest diff --git a/.github/workflows/pr-alpine.yml b/.github/workflows/pr-alpine.yml index b551ffdb..28dc1099 100644 --- a/.github/workflows/pr-alpine.yml +++ b/.github/workflows/pr-alpine.yml @@ -22,9 +22,10 @@ jobs: - name: Checkout uses: actions/checkout@v4.1.0 - - name: Get commit short hash - id: short_digest - run: echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Check isort compliance + uses: isort/isort-action@v1.1.0 + with: + sort-paths: renew.py - name: Build image for tests uses: docker/build-push-action@v5.0.0 diff --git a/.github/workflows/pr-debian.yml b/.github/workflows/pr-debian.yml index b1dd7b9a..ec1e7e53 100644 --- a/.github/workflows/pr-debian.yml +++ b/.github/workflows/pr-debian.yml @@ -8,7 +8,7 @@ on: env: IMAGE_NAME: "simaofsilva/noip-renewer" PIP_VERSION: "23.2.1" # renovate: datasource=pypi depName=pip versioning=pep440 - GECKO_DRIVER_VERSION: "0.33.0" # renovate: datasource=github-tags depName=mozilla/geckodriver + GECKODRIVER_VERSION: "0.33.0" # renovate: datasource=github-tags depName=mozilla/geckodriver jobs: build_debian: @@ -23,9 +23,10 @@ jobs: - name: Checkout uses: actions/checkout@v4.1.0 - - name: Get commit short hash - id: short_digest - run: echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Check isort compliance + uses: isort/isort-action@v1.1.0 + with: + sort-paths: renew.py - name: Build image for tests uses: docker/build-push-action@v5.0.0 @@ -37,7 +38,7 @@ jobs: file: Dockerfile.debian build-args: | PIP_VERSION=${{ env.PIP_VERSION }} - GECKO_DRIVER_VERSION=${{ env.GECKO_DRIVER_VERSION }} + GECKODRIVER_VERSION=${{ env.GECKODRIVER_VERSION }} - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master diff --git a/Dockerfile b/Dockerfile index 0bdb48ad..cf7a07ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,25 @@ -FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac +FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac as builder + +# Prevent Python from writing out pyc files +ENV PYTHONDONTWRITEBYTECODE 1 + +# Keep Python from buffering stdin/stdout +ENV PYTHONUNBUFFERED 1 + +# Enable custom virtual environment +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" ARG PIP_VERSION -COPY requirements.txt /requirements.txt +# Add requirements file +COPY requirements.txt requirements.txt + +# Install requirements +RUN python3 -m venv $VIRTUAL_ENV && \ + pip install --upgrade pip=="${PIP_VERSION}" && \ + pip install --no-cache-dir -r requirements.txt -RUN apk add --no-cache gcc libc-dev libffi-dev && \ - pip install --no-cache-dir pip=="$PIP_VERSION" && \ - pip install --no-cache-dir --user -r /requirements.txt FROM python:3.12.0-alpine@sha256:ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac @@ -16,10 +29,13 @@ RUN apk add --no-cache firefox && \ ln -s /usr/bin/geckodriver /usr/local/bin/geckodriver && \ rm -rf /var/cache/apk/* /tmp/* -COPY --from=0 /root/.local /root/.local +# Enable custom virtual environment +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" -ENV PATH=/root/.local/bin:$PATH +# Copy dependencies from previous stage +COPY --from=builder $VIRTUAL_ENV $VIRTUAL_ENV +# Copy and set the entrypoint bash script COPY renew.py . - ENTRYPOINT ["python3", "renew.py"] diff --git a/Dockerfile.debian b/Dockerfile.debian index 6afe97f5..681d70f3 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,42 +1,64 @@ -FROM python:3.12.0-slim-bullseye@sha256:8c5ca5725a49e3ab3a1e76bd7e93fff1aeec2fdfd91288627f4510eea5a72e09 +FROM python:3.12.0-slim-bullseye@sha256:8c5ca5725a49e3ab3a1e76bd7e93fff1aeec2fdfd91288627f4510eea5a72e09 as builder SHELL ["/bin/bash", "-c"] -ARG PIP_VERSION +# Prevent Python from writing out pyc files +ENV PYTHONDONTWRITEBYTECODE 1 + +# Keep Python from buffering stdin/stdout +ENV PYTHONUNBUFFERED 1 + +# Disable any user interaction ENV DEBIAN_FRONTEND=noninteractive + +# Enable custom virtual environment +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +ARG PIP_VERSION ARG ARMV7_DEPS="gcc libc6-dev libffi-dev rustc cargo libssl-dev" +# Install required packages RUN apt-get update && \ if [ $(getconf LONG_BIT) -eq 32 ]; then apt-get install -y --no-install-recommends ${ARMV7_DEPS}; fi -COPY requirements.txt /requirements.txt +# Add requirements file +COPY requirements.txt . -RUN pip install --no-cache-dir pip=="${PIP_VERSION}" && \ - pip install --no-cache-dir --user -r /requirements.txt +# Install requirements +RUN python3 -m venv $VIRTUAL_ENV && \ + pip install --upgrade pip=="${PIP_VERSION}" && \ + pip install --no-cache-dir -r requirements.txt -FROM python:3.12.0-slim-bullseye@sha256:8c5ca5725a49e3ab3a1e76bd7e93fff1aeec2fdfd91288627f4510eea5a72e09 +FROM python:3.12.0-slim-bullseye@sha256:8c5ca5725a49e3ab3a1e76bd7e93fff1aeec2fdfd91288627f4510eea5a72e09 as geckodriver SHELL ["/bin/bash", "-c"] -ARG GECKO_DRIVER_VERSION +ARG GECKODRIVER_VERSION + +# Disable any user interaction ENV DEBIAN_FRONTEND=noninteractive +# Install required packages RUN apt-get update && \ apt-get install -y --no-install-recommends curl +# Download geckodriver RUN set -x && \ if [ "$(uname --m)" == "x86_64" ]; then ARCH="linux64"; elif [ "$(uname --m)" == "aarch64" ]; then ARCH="linux-aarch64"; else ARCH="linux32"; fi && \ - curl -sSL -O https://github.com/mozilla/geckodriver/releases/download/v${GECKO_DRIVER_VERSION}/geckodriver-v${GECKO_DRIVER_VERSION}-${ARCH}.tar.gz && \ - tar zxf geckodriver-v${GECKO_DRIVER_VERSION}-${ARCH}.tar.gz + curl -sSL -O https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${ARCH}.tar.gz && \ + tar zxf geckodriver-v${GECKODRIVER_VERSION}-${ARCH}.tar.gz FROM python:3.12.0-slim-bullseye@sha256:8c5ca5725a49e3ab3a1e76bd7e93fff1aeec2fdfd91288627f4510eea5a72e09 +# Disable any user interaction ENV DEBIAN_FRONTEND=noninteractive +# Install required packages RUN apt-get update && \ apt-get install -y --no-install-recommends firefox-esr && \ apt-get autoremove -y && \ @@ -50,11 +72,16 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/* /usr/share/doc /usr/share/man -COPY --from=0 /root/.local /root/.local -COPY --from=1 /geckodriver /usr/local/bin/geckodriver +# Enable custom virtual environment +ENV VIRTUAL_ENV=/opt/venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" -ENV PATH=/root/.local/bin:$PATH +# Copy dependencies from previous stage +COPY --from=builder $VIRTUAL_ENV $VIRTUAL_ENV -COPY renew.py renew.py +# Copy geckodriver from previous stage +COPY --from=geckodriver --chmod=755 /geckodriver /usr/local/bin/geckodriver +# Copy and set the entrypoint bash script +COPY renew.py . ENTRYPOINT ["python3", "renew.py"] diff --git a/renew.py b/renew.py index 75b26100..6ddfdd65 100644 --- a/renew.py +++ b/renew.py @@ -6,9 +6,11 @@ import requests from deep_translator import GoogleTranslator from selenium import webdriver -from selenium.common.exceptions import NoSuchElementException +from selenium.common.exceptions import NoSuchElementException, TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.firefox.service import Service +from selenium.webdriver.support import expected_conditions +from selenium.webdriver.support.ui import WebDriverWait def method1(): @@ -54,11 +56,9 @@ def get_user_agent(): # OPEN BROWSER print("Opening browser") browser_options = webdriver.FirefoxOptions() - browser_options.add_argument("--headless") - browser_options.add_argument("user-agent=" + str(get_user_agent())) - + # browser_options.add_argument("--headless") + browser_options.add_argument("user-agent=" + str("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36")) service = Service(executable_path="/usr/local/bin/geckodriver", log_output="/dev/null") - browser = webdriver.Firefox(options=browser_options, service=service) # LOGIN @@ -66,71 +66,79 @@ def get_user_agent(): if browser.current_url == LOGIN_URL: - browser.find_element(by=By.NAME, value="username").send_keys(email) - browser.find_element(by=By.NAME, value="password").send_keys(password) - - login_button = False - - for button in browser.find_elements(by=By.TAG_NAME, value="button"): - if button.text == "Log In": - button.click() - login_button = True - break + try: + username_input = WebDriverWait(browser, 10).until(lambda browser: browser.find_element(by=By.ID, value="username")) + except TimeoutException: + print("Username input not found within the specified timeout.") + browser.quit() + exit(1) - if not login_button: - print("Login button has changed. Please contact support. ") + try: + password_input = WebDriverWait(browser, 10).until(lambda browser: browser.find_element(by=By.ID, value="password")) + except TimeoutException: + print("Password input not found within the specified timeout.") + browser.quit() exit(1) - sleep(2) + username_input.send_keys(email) + password_input.send_keys(password) - if str(browser.current_url).endswith("noip.com/"): + try: + login_button = WebDriverWait(browser, 10).until(lambda browser: browser.find_element(by=By.ID, value="clogs-captcha-button")) + login_button.click() + except TimeoutException: + print("Login button not found within the specified timeout.") + browser.quit() + exit(1) + wait = WebDriverWait(driver=browser, timeout=20) + try: + dashboard_nav = WebDriverWait(driver=browser, timeout=60, poll_frequency=3).until(expected_conditions.visibility_of(browser.find_element(by=By.ID, value="dashboard-nav"))) print("Login successful") - browser.get(HOST_URL) - sleep(1) + except TimeoutException: + print("Could not login. Check if account is blocked.") + browser.quit() + exit(1) - aux = 1 - while not browser.title.startswith("My No-IP") and aux < 3: - browser.get(HOST_URL) - sleep(3) - aux += 1 + browser.get(HOST_URL) - if browser.title.startswith("My No-IP") and aux < 4: - confirmed_hosts = 0 + try: + create_hostname_button = WebDriverWait(driver=browser, timeout=60, poll_frequency=3).until(expected_conditions.visibility_of(browser.find_element(by=By.ID, value="host-panel"))) + except TimeoutException: + print("Could not load NO-IP hostnames page.") + browser.quit() + exit(1) + + # CONFIRM HOSTS + try: + hosts = method2() + print("Confirming hosts phase") + confirmed_hosts = 0 - # RENEW HOSTS + for host in hosts: try: - hosts = method2() - print("Confirming hosts phase") - - for host in hosts: - try: - button = host.find_element(by=By.TAG_NAME, value="button") - except NoSuchElementException as e: - break - - if button.text == "Confirm" or translate(button.text) == "Confirm": - button.click() - confirmed_host = host.find_element(by=By.TAG_NAME, value="a").text - confirmed_hosts += 1 - print("Host \"" + confirmed_host + "\" confirmed") - sleep(0.25) - - if confirmed_hosts == 1: - print("1 host confirmed") - else: - print(str(confirmed_hosts) + " hosts confirmed") - - print("Finished") - - except Exception as e: - print("Error: ", e) - - finally: - print("Logging off\n\n") - browser.get(LOGOUT_URL) - else: - print("Error: cannot login. Check if account is not blocked.") + button = host.find_element(by=By.TAG_NAME, value="button") + except NoSuchElementException as e: + break + + if button.text == "Confirm" or translate(button.text) == "Confirm": + button.click() + confirmed_host = host.find_element(by=By.TAG_NAME, value="a").text + confirmed_hosts += 1 + print("Host \"" + confirmed_host + "\" confirmed") + sleep(0.25) + + if confirmed_hosts == 1: + print("1 host confirmed") + else: + print(str(confirmed_hosts) + " hosts confirmed") + + print("Finished") + + except Exception as e: + print("Error: ", e) + + finally: print("Logging off\n\n") browser.get(LOGOUT_URL) else: diff --git a/renovate.json b/renovate.json index 099d3e77..cc4a9ec1 100644 --- a/renovate.json +++ b/renovate.json @@ -14,13 +14,13 @@ "digest" ], "automerge": true, - "automergeType": "branch", + "automergeType": "pr", "ignoreTests": true } ], "regexManagers": [ { - "description": "Get versions for PyPI", + "description": "Get versions for PyPI and Geckodriver", "fileMatch": [ "^\\.github\\/workflows\\/[^/]+\\.ya?ml$" ],