From 065d82713201c4fef4488549cfa03a964a3662ec Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 12:43:45 -0500 Subject: [PATCH 01/12] move from python 2 to 3 wip --- Dockerfile | 2 +- Dockerfile.skaffold | 2 +- deploy/Dockerfile.prod | 2 +- okteto.Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 15d43186..161b1ddc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2 +FROM python:3 RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/Dockerfile.skaffold b/Dockerfile.skaffold index 6550bd40..955490af 100644 --- a/Dockerfile.skaffold +++ b/Dockerfile.skaffold @@ -1,4 +1,4 @@ -FROM python:2 +FROM python:3 RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/deploy/Dockerfile.prod b/deploy/Dockerfile.prod index b6b16306..1fc2f926 100644 --- a/deploy/Dockerfile.prod +++ b/deploy/Dockerfile.prod @@ -3,7 +3,7 @@ WORKDIR /docker-compose-generate COPY ./util/docker-compose-generate /docker-compose-generate RUN make build -FROM python:2 +FROM python:3 COPY --from=builder /docker-compose-generate/dcg /dcg RUN apt-get update \ diff --git a/okteto.Dockerfile b/okteto.Dockerfile index 8c418125..ad12a675 100644 --- a/okteto.Dockerfile +++ b/okteto.Dockerfile @@ -4,7 +4,7 @@ COPY ./util/docker-compose-generate /docker-compose-generate RUN make build -FROM python:2 +FROM python:3 COPY --from=builder /docker-compose-generate/dcg /dcg RUN apt-get update && \ From dae29228e16822048743e6b7d78e46b652b537cc Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 13:47:58 -0500 Subject: [PATCH 02/12] update okteto file --- okteto-pipeline.yml | 15 --------------- okteto.yml | 28 +++++++++++++++++++--------- 2 files changed, 19 insertions(+), 24 deletions(-) delete mode 100644 okteto-pipeline.yml diff --git a/okteto-pipeline.yml b/okteto-pipeline.yml deleted file mode 100644 index 39ba9f3d..00000000 --- a/okteto-pipeline.yml +++ /dev/null @@ -1,15 +0,0 @@ -icon: https://github.com/okteto/polling/raw/master/icon.png -deploy: - - envsubst < kustomize/overlays/okteto/deployment.tmpl.yaml > kustomize/overlays/okteto/deployment.yaml - - - okteto build - -f ./okteto.Dockerfile - -t okteto.dev/install-scripts:${OKTETO_GIT_COMMIT} - --build-arg DD_API_KEY=${DD_API_KEY} - . - - - cd kustomize/overlays/okteto && kustomize edit set image replicated/install-scripts=okteto.dev/install-scripts:${OKTETO_GIT_COMMIT} - - - kubectl apply -k kustomize/overlays/okteto -devs: - - okteto.yml diff --git a/okteto.yml b/okteto.yml index ab673cb6..d38deebf 100644 --- a/okteto.yml +++ b/okteto.yml @@ -1,9 +1,19 @@ -name: install-scripts -command: bash -workdir: /go/src/github.com/replicatedhq/install-scripts -sync: - - .:/go/src/github.com/replicatedhq/install-scripts -resources: - limits: - cpu: "2" - memory: 4Gi +icon: https://github.com/okteto/polling/raw/master/icon.png +deploy: + - envsubst < kustomize/overlays/okteto/deployment.tmpl.yaml > kustomize/overlays/okteto/deployment.yaml + + - okteto build + -f ./okteto.Dockerfile + -t okteto.dev/install-scripts:${OKTETO_GIT_COMMIT} + --build-arg DD_API_KEY=${DD_API_KEY} + . + + - cd kustomize/overlays/okteto && kustomize edit set image replicated/install-scripts=okteto.dev/install-scripts:${OKTETO_GIT_COMMIT} + + - kubectl apply -k kustomize/overlays/okteto +dev: + install-scripts: + command: bash + workdir: /go/src/github.com/replicatedhq/install-scripts + sync: + - .:/go/src/github.com/replicatedhq/install-scripts From f078986a11f8e9fe787b3fbfe4d1a064d4a04938 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 13:58:03 -0500 Subject: [PATCH 03/12] update PyYAML and uWSGI to fix compilation with Python 3.13 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bab8588a..92fc5281 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ Flask==0.12.3 mysql-connector==2.1.7 pytest==3.0.6 pytest-mock==1.5.0 -PyYAML==5.4 +PyYAML==6.0.2 git+https://github.com/emosbaugh/python-semver@f26d956e2ea13a8159e5629d5b7bf225f8ae5a61#egg=semver shellescape==3.8.1 -uWSGI==2.0.22 +uWSGI==2.0.28 From fb8824f6dbfe65782d76ebd2fee4aaecf62367ba Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 14:02:21 -0500 Subject: [PATCH 04/12] python 3.12 --- .circleci/config.yml | 2 +- Dockerfile | 2 +- Dockerfile.skaffold | 2 +- Makefile | 4 ++-- deploy/Dockerfile.prod | 2 +- okteto.Dockerfile | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8211c2d..10c37f84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ defaults: &defaults jobs: is_upstream: docker: - - image: cimg/python:3.10 + - image: cimg/python:3.12 steps: - run: | set -veuo pipefail diff --git a/Dockerfile b/Dockerfile index 161b1ddc..d7dc1879 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.12 RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/Dockerfile.skaffold b/Dockerfile.skaffold index 955490af..4c52a942 100644 --- a/Dockerfile.skaffold +++ b/Dockerfile.skaffold @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.12 RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/Makefile b/Makefile index d6f0d3ea..e914268a 100644 --- a/Makefile +++ b/Makefile @@ -113,12 +113,12 @@ shell_composer_linux: install-scripts-dev \ /bin/bash test: - python2 -m pytest -v tests + python3 -m pytest -v tests ./test.sh run: /dcg --raw > install_scripts/templates/swarm/docker-compose-generate-safe.sh - python2 main.py + python3 main.py .PHONY: scan scan: build diff --git a/deploy/Dockerfile.prod b/deploy/Dockerfile.prod index 1fc2f926..cde99dac 100644 --- a/deploy/Dockerfile.prod +++ b/deploy/Dockerfile.prod @@ -3,7 +3,7 @@ WORKDIR /docker-compose-generate COPY ./util/docker-compose-generate /docker-compose-generate RUN make build -FROM python:3 +FROM python:3.12 COPY --from=builder /docker-compose-generate/dcg /dcg RUN apt-get update \ diff --git a/okteto.Dockerfile b/okteto.Dockerfile index ad12a675..c3fecd50 100644 --- a/okteto.Dockerfile +++ b/okteto.Dockerfile @@ -4,7 +4,7 @@ COPY ./util/docker-compose-generate /docker-compose-generate RUN make build -FROM python:3 +FROM python:3.12 COPY --from=builder /docker-compose-generate/dcg /dcg RUN apt-get update && \ From b871525c8b09ff61da6d1b3716e141696053b03b Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 14:05:36 -0500 Subject: [PATCH 05/12] update boto3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 92fc5281..2946c038 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -boto3==1.7.25 +boto3==1.35.78 Flask==0.12.3 mysql-connector==2.1.7 pytest==3.0.6 From b2733993790bd281f1ff8954c7879ae97ee9d008 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 14:13:38 -0500 Subject: [PATCH 06/12] update import --- install_scripts/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install_scripts/app.py b/install_scripts/app.py index 1d9b3524..2547a356 100644 --- a/install_scripts/app.py +++ b/install_scripts/app.py @@ -1,6 +1,5 @@ from __future__ import print_function -import constant from flask import Flask, Response, abort, redirect, render_template, request, \ jsonify import semver @@ -13,7 +12,7 @@ from shellescape import quote import tempfile -from . import db, helpers, param, images +from . import db, helpers, param, images, constant app = Flask(__name__) From cdb6f74756930b52e664d8dc30704034ccd65ccc Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 14:21:22 -0500 Subject: [PATCH 07/12] update flask --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2946c038..6faff3a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ boto3==1.35.78 -Flask==0.12.3 +Flask==3.1.0 mysql-connector==2.1.7 pytest==3.0.6 pytest-mock==1.5.0 From 692dc84ae7c5b31d1fd1808127de3e60ac9cf18c Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 17:54:36 -0500 Subject: [PATCH 08/12] code updates for python 3 --- install_scripts/app.py | 4 +++- install_scripts/db.py | 3 ++- install_scripts/helpers.py | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/install_scripts/app.py b/install_scripts/app.py index 2547a356..ed98cf38 100644 --- a/install_scripts/app.py +++ b/install_scripts/app.py @@ -14,7 +14,9 @@ from . import db, helpers, param, images, constant -app = Flask(__name__) +project_root = os.path.dirname(__file__) +template_path = os.path.join(project_root, 'templates') +app = Flask(__name__, template_folder=template_path) _images = images.get_default_images() diff --git a/install_scripts/db.py b/install_scripts/db.py index 2fecca14..d659bf62 100644 --- a/install_scripts/db.py +++ b/install_scripts/db.py @@ -26,7 +26,8 @@ def get(): port=param.lookup('MYSQL_PORT'), database=database, user=param.lookup('MYSQL_USER'), - password=password) + password=password, + auth_plugin='mysql_native_password') return db diff --git a/install_scripts/helpers.py b/install_scripts/helpers.py index ea821604..0138143f 100644 --- a/install_scripts/helpers.py +++ b/install_scripts/helpers.py @@ -478,9 +478,11 @@ def does_customer_exist(customer_id): # Produce base64 encoding with linebreaks. def base64_encode(data): + if len(data) == 0: + return '' encoded = base64.b64encode(data) return '\n'.join( - encoded[pos:pos + 76] for pos in xrange(0, len(encoded), 76)) + encoded[pos:pos + 76] for pos in range(0, len(encoded), 76)) def get_docker_deb_pkg_version(docker_version, lsb_dist, dist_version): @@ -567,4 +569,4 @@ def snapshots_use_overlay(replicated_version): def make_shell_safe(s): if type(s) == str or type(s) == unicode: return s.replace('$', '_').replace('`', '\'') - return s \ No newline at end of file + return s From 516b9336e3c69f73856e529c3332e19bdf4f09c7 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 17:55:30 -0500 Subject: [PATCH 09/12] add .stignore without 'Manifest' exclusion for okteto --- .stignore | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .stignore diff --git a/.stignore b/.stignore new file mode 100644 index 00000000..87c35ecb --- /dev/null +++ b/.stignore @@ -0,0 +1,100 @@ +.git +# Byte-compiled / optimized / DLL files +__pycache__ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build +develop-eggs +dist +downloads +eggs +.eggs +lib +lib64 +parts +sdist +var +wheels +pip-wheel-metadata +share/python-wheels +*.egg-info +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build + +# PyBuilder +target + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env +venv +ENV +env.bak +venv.bak + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mypy +.mypy_cache +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre From 24f319ce64a2d5f529ea09e85b627ce0eb809d6e Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 17:56:32 -0500 Subject: [PATCH 10/12] use pip3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e914268a..07188848 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ else endif deps: - pip install -r requirements.txt + pip3 install -r requirements.txt build: docker build --pull -t install-scripts -f deploy/Dockerfile.prod . From a1f4f464b275846d7c595c3fafdd2a371263efa8 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 17:58:31 -0500 Subject: [PATCH 11/12] pip3? --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 10c37f84..7aef30c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,8 +34,8 @@ jobs: <<: *defaults steps: - checkout - - run: sudo apt-get update; sudo apt-get install shunit2 - - run: sudo pip install pytest # todo cache these + - run: sudo apt-get update; sudo apt-get install shunit2 python3-pip + - run: sudo pip3 install pytest # todo cache these - run: sudo make deps # todo cache these - run: make test From 9324292945cfca387c1096bca059d1e12971e0b2 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 11 Dec 2024 18:10:55 -0500 Subject: [PATCH 12/12] update pytest --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6faff3a9..11613ae2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ boto3==1.35.78 Flask==3.1.0 mysql-connector==2.1.7 -pytest==3.0.6 -pytest-mock==1.5.0 +pytest==8.3.4 +pytest-mock==3.14.0 PyYAML==6.0.2 git+https://github.com/emosbaugh/python-semver@f26d956e2ea13a8159e5629d5b7bf225f8ae5a61#egg=semver shellescape==3.8.1