diff --git a/technologies/job/python/python-3.13-base/Dockerfile b/technologies/job/python/python-3.13-base/Dockerfile new file mode 100644 index 000000000..6f85ca4af --- /dev/null +++ b/technologies/job/python/python-3.13-base/Dockerfile @@ -0,0 +1,49 @@ +FROM python:3.13.0-slim-bullseye +ENV DEBIAN_FRONTEND noninteractive + +ENV LC_ALL C +ENV LD_RUN_PATH /usr/local/lib + +# LIGHT DEPENDENCIES START +RUN apt-get update -qq && \ + apt-get install -yqq --no-install-recommends \ + ftp wget curl unzip telnet openssh-client git apt-transport-https ca-certificates \ + krb5-user libpam-krb5 libpam-ccreds locales openjdk-11-jdk ca-certificates-java ant && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# LIGHT DEPENDENCIES END + +# UPGRADE PIP +RUN pip --no-cache-dir install --upgrade pip + +# Add Hadoop CLI +RUN cd / \ + && mkdir app \ + && cd app \ + && wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.6.5/hadoop-2.6.5.tar.gz \ + && tar xvf hadoop-2.6.5.tar.gz \ + && rm hadoop-2.6.5.tar.gz \ + && rm -rf hadoop-2.6.5/etc/hadoop \ + && ln -s /etc/hadoop/conf hadoop-2.6.5/etc/hadoop; + +# Fix Locale +RUN sed -i '/^#.* en_US.* /s/^#//' /etc/locale.gen \ + && locale-gen + +# Fix certificate issues +RUN update-ca-certificates -f; + +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/ +ENV HADOOP_HOME=/app/hadoop-2.6.5 +ENV HADOOP_CONF_DIR=/etc/hadoop/conf +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/app/hadoop-2.6.5/lib/native:/usr/lib/jvm/java-11-openjdk-amd64/lib" +ENV PATH "/app/hadoop-2.6.5/bin:${PATH}" +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 + +COPY entrypoint /entrypoint +RUN chmod 755 /entrypoint + +WORKDIR /sandbox + +ENTRYPOINT ["bash","/entrypoint"] diff --git a/technologies/job/python/python-3.13-base/build.gradle.kts b/technologies/job/python/python-3.13-base/build.gradle.kts new file mode 100644 index 000000000..b814e94a3 --- /dev/null +++ b/technologies/job/python/python-3.13-base/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2019-2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.bmuschko.gradle.docker.DockerRemoteApiPlugin +import com.saagie.technologies.SaagieTechnologiesGradlePlugin + +apply() +apply() diff --git a/technologies/job/python/python-3.13-base/dockerInfo.yaml b/technologies/job/python/python-3.13-base/dockerInfo.yaml new file mode 100644 index 000000000..5b78d91c9 --- /dev/null +++ b/technologies/job/python/python-3.13-base/dockerInfo.yaml @@ -0,0 +1,4 @@ +image: saagie/python +baseTag: 3.13-base +dynamicVersion: 1.193.0_SDKTECHNO-275 +version: 3.13-base-1.193.0 diff --git a/technologies/job/python/python-3.13-base/entrypoint b/technologies/job/python/python-3.13-base/entrypoint new file mode 100644 index 000000000..7f1ac6515 --- /dev/null +++ b/technologies/job/python/python-3.13-base/entrypoint @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +export CLASSPATH=`hadoop classpath --glob` + +if compgen -G "*.zip*" > /dev/null; then + unzip -q *.zip +fi +if test -f requirements.txt; +then + echo "*******PIP INSTALL BEGIN*******" + pip install -r requirements.txt; + echo "*******PIP INSTALL END*******" +fi; +if [ $? != 0 ]; then echo "Error during 'pip install -f 'requirements.txt''";exit 1; fi; +if test -f main_script; +then sh ./main_script; +else exec "$@" +fi; diff --git a/technologies/job/python/python-3.13-base/image_test.yaml b/technologies/job/python/python-3.13-base/image_test.yaml new file mode 100644 index 000000000..1dc9b4958 --- /dev/null +++ b/technologies/job/python/python-3.13-base/image_test.yaml @@ -0,0 +1,100 @@ +schemaVersion: "2.0.0" + +metadataTest: + env: + - key: LANG + value: "en_US.UTF-8" + - key: LC_ALL + value: "en_US.UTF-8" + - key: LD_RUN_PATH + value: "/usr/local/lib" + - key: PATH + value: "/app/hadoop-2.6.5/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + - key: "HADOOP_CONF_DIR" + value: "/etc/hadoop/conf" + - key: "JAVA_HOME" + value: "/usr/lib/jvm/java-11-openjdk-amd64/" + +fileExistenceTests: + - name: "unzip" + path: "/usr/bin/unzip" + shouldExist: true + permissions: "-rwxr-xr-x" + + - name: "entrypoint" + path: "/entrypoint" + shouldExist: true + permissions: "-rwxr-xr-x" + + - name: "kinit" + path: "/usr/bin/kinit" + shouldExist: true + permissions: "-rwxr-xr-x" + +fileContentTests: + - name: "entrypoint" + path: "/entrypoint" + expectedContents: + [ + "unzip -q \\*.zip", + "pip install -r requirements.txt", + "sh ./main_script", + ] + +commandTests: + - name: "java version" + command: "java" + args: [ "-version" ] + expectedError: [ 'openjdk version "11.*' ] + + - name: "ftp" + args: ["-h"] + command: "ftp" + exitCode: 0 + + - name: "wget" + args: ["--help"] + command: "wget" + exitCode: 0 + + - name: "curl" + args: ["--help"] + command: "curl" + exitCode: 0 + + - name: "unzip" + args: ["--help"] + command: "unzip" + exitCode: 0 + + - name: "tar" + args: ["--help"] + command: "tar" + exitCode: 0 + + - name: "telnet" + command: "which" + args: ["telnet"] + expectedOutput: ["/usr/bin/telnet"] + exitCode: 0 + + - name: "scp" + command: "which" + args: ["scp"] + expectedOutput: ["/usr/bin/scp"] + exitCode: 0 + + - name: "python installation" + command: "which" + args: ["python"] + expectedOutput: ["/usr/local/bin/python"] + + - name: "python version" + command: "python" + args: ["-V"] + expectedOutput: ["Python 3.12*"] + + - name: "krb5-user installation" + command: "kinit" + expectedError: ['kinit: Program lacks support for encryption type while getting initial credentials'] + exitCode: 1 diff --git a/technologies/job/python/python-3.13-lite/Dockerfile b/technologies/job/python/python-3.13-lite/Dockerfile new file mode 100644 index 000000000..da77ae82d --- /dev/null +++ b/technologies/job/python/python-3.13-lite/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.13.0-slim-bullseye + +ENV DEBIAN_FRONTEND noninteractive +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 + +# INSTALL DEPENDENCIES +RUN apt-get update -qq && \ + apt-get install -yqq --no-install-recommends unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY entrypoint /entrypoint +RUN chmod 755 /entrypoint + +WORKDIR /sandbox + +ENTRYPOINT ["bash","/entrypoint"] diff --git a/technologies/job/python/python-3.13-lite/build.gradle.kts b/technologies/job/python/python-3.13-lite/build.gradle.kts new file mode 100644 index 000000000..b814e94a3 --- /dev/null +++ b/technologies/job/python/python-3.13-lite/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2019-2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.bmuschko.gradle.docker.DockerRemoteApiPlugin +import com.saagie.technologies.SaagieTechnologiesGradlePlugin + +apply() +apply() diff --git a/technologies/job/python/python-3.13-lite/context.yaml b/technologies/job/python/python-3.13-lite/context.yaml new file mode 100644 index 000000000..ec51ee45e --- /dev/null +++ b/technologies/job/python/python-3.13-lite/context.yaml @@ -0,0 +1,23 @@ +id: "3.13-lite" +label: "3.13-lite" +available: true +recommended: false +trustLevel: stable +job: + features: + - type: COMMAND_LINE + label: Command line + mandatory: true + comment: Linux shell command to launch the job. + defaultValue: python {file} arg1 arg2 + - type: ARTIFACT + label: Package + mandatory: false + comment: "Compatible upload file : .py or .zip" + - type: SCHEDULER + label: Scheduled + mandatory: true + - type: AI_DESCRIPTION_GENERATOR + label: AI description generator enabled. + mandatory: true + comment: Activation of the AI-based automatic description generation function. diff --git a/technologies/job/python/python-3.13-lite/dockerInfo.yaml b/technologies/job/python/python-3.13-lite/dockerInfo.yaml new file mode 100644 index 000000000..68d0a1e2a --- /dev/null +++ b/technologies/job/python/python-3.13-lite/dockerInfo.yaml @@ -0,0 +1,4 @@ +image: saagie/python +baseTag: 3.13.0-bullseye +dynamicVersion: 1.193.0_SDKTECHNO-275 +version: 3.13-lite-1.193.0 diff --git a/technologies/job/python/python-3.13-lite/entrypoint b/technologies/job/python/python-3.13-lite/entrypoint new file mode 100644 index 000000000..d43d32f97 --- /dev/null +++ b/technologies/job/python/python-3.13-lite/entrypoint @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +if compgen -G "*.zip*" > /dev/null; then + unzip -q *.zip +fi + +if test -f main_script; then + sh ./main_script; +else + exec "$@" +fi diff --git a/technologies/job/python/python-3.13-lite/image_test.yaml b/technologies/job/python/python-3.13-lite/image_test.yaml new file mode 100644 index 000000000..6e211084c --- /dev/null +++ b/technologies/job/python/python-3.13-lite/image_test.yaml @@ -0,0 +1,44 @@ +schemaVersion: "2.0.0" + +metadataTest: + env: + - key: LANG + value: "en_US.UTF-8" + - key: LC_ALL + value: "en_US.UTF-8" + +fileExistenceTests: + - name: "unzip" + path: "/usr/bin/unzip" + shouldExist: true + permissions: "-rwxr-xr-x" + + - name: "entrypoint" + path: "/entrypoint" + shouldExist: true + permissions: "-rwxr-xr-x" + +fileContentTests: + - name: "entrypoint" + path: "/entrypoint" + expectedContents: + [ + "unzip -q \\*.zip", + "sh ./main_script", + ] + +commandTests: + - name: "unzip" + args: ["--help"] + command: "unzip" + exitCode: 0 + + - name: "python installation" + command: "which" + args: ["python"] + expectedOutput: ["/usr/local/bin/python"] + + - name: "python version" + command: "python" + args: ["-V"] + expectedOutput: ["Python 3.12*"] diff --git a/technologies/job/python/python-3.13/Dockerfile b/technologies/job/python/python-3.13/Dockerfile new file mode 100644 index 000000000..336fe33de --- /dev/null +++ b/technologies/job/python/python-3.13/Dockerfile @@ -0,0 +1,35 @@ +ARG base_img + +FROM ${base_img} AS BASE_IMG + +# LIBS PART BEGIN +RUN apt update -qq && apt install -qqy --no-install-recommends \ + python3-pip \ + gcc \ + g++ \ + libsasl2-2 \ + libsasl2-modules-ldap \ + build-essential \ + unixodbc \ + unixodbc-dev \ + libpq-dev \ + libsqlite3-dev \ + libkrb5-dev \ + libsasl2-dev \ + libssl-dev \ + libcurl4-openssl-dev \ + libgeos-dev \ + swig \ + python3-matplotlib \ + python3-lxml \ + cmake \ + && rm -rf /var/lib/apt/lists/*; +# LIBS PART END + +# Need to read requirements.txt file sequentially to ensure consistent installation order +COPY resources/requirements.txt /tmp/requirements.txt +RUN grep -v '^#' /tmp/requirements.txt | xargs -L 1 pip --no-cache-dir install \ + && rm -rf /root/.cachex \ + && rm -rf /boot/.cache/pip \ + && rm -rf ~/.cache/pip + diff --git a/technologies/job/python/python-3.13/build.gradle.kts b/technologies/job/python/python-3.13/build.gradle.kts new file mode 100644 index 000000000..30077e521 --- /dev/null +++ b/technologies/job/python/python-3.13/build.gradle.kts @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2019-2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.bmuschko.gradle.docker.DockerRemoteApiPlugin +import com.saagie.technologies.SaagieTechnologiesGradlePlugin +import com.saagie.technologies.readDockerInfo +import com.saagie.technologies.getVersionForDocker + + +apply() +apply() + +val dockerInfo = readDockerInfo(projectDir) + +tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) { + dependsOn(":${this.project.name}-base:testImage") + this.buildArgs.put( + "base_img", + "${dockerInfo?.image}:${dockerInfo?.baseTag}-base-${this.project.getVersionForDocker()}" + ) +} \ No newline at end of file diff --git a/technologies/job/python/python-3.13/context.yaml b/technologies/job/python/python-3.13/context.yaml new file mode 100644 index 000000000..9474b37ec --- /dev/null +++ b/technologies/job/python/python-3.13/context.yaml @@ -0,0 +1,23 @@ +id: "3.13" +label: "3.13" +available: true +recommended: true +trustLevel: stable +job: + features: + - type: COMMAND_LINE + label: Command line + mandatory: true + comment: Linux shell command to launch the job. + defaultValue: python {file} arg1 arg2 + - type: ARTIFACT + label: Package + mandatory: false + comment: "Compatible upload file : .py or .zip (with optional requirements.txt for pip install)" + - type: SCHEDULER + label: Scheduled + mandatory: true + - type: AI_DESCRIPTION_GENERATOR + label: AI description generator enabled. + mandatory: true + comment: Activation of the AI-based automatic description generation function. diff --git a/technologies/job/python/python-3.13/dockerInfo.yaml b/technologies/job/python/python-3.13/dockerInfo.yaml new file mode 100644 index 000000000..aae5858cd --- /dev/null +++ b/technologies/job/python/python-3.13/dockerInfo.yaml @@ -0,0 +1,4 @@ +image: saagie/python +baseTag: 3.13 +dynamicVersion: 1.193.0_SDKTECHNO-275 +version: 3.13-1.193.0 diff --git a/technologies/job/python/python-3.13/image_test.yaml b/technologies/job/python/python-3.13/image_test.yaml new file mode 100644 index 000000000..a0cfe3900 --- /dev/null +++ b/technologies/job/python/python-3.13/image_test.yaml @@ -0,0 +1,228 @@ +schemaVersion: "2.0.0" + +metadataTest: + env: + - key: LANG + value: "en_US.UTF-8" + - key: LC_ALL + value: "en_US.UTF-8" + - key: LD_RUN_PATH + value: "/usr/local/lib" + - key: PATH + value: "/app/hadoop-2.6.5/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + - key: "HADOOP_CONF_DIR" + value: "/etc/hadoop/conf" + +fileExistenceTests: + - name: "unzip" + path: "/usr/bin/unzip" + shouldExist: true + permissions: "-rwxr-xr-x" + + - name: "entrypoint" + path: "/entrypoint" + shouldExist: true + permissions: "-rwxr-xr-x" + + - name: "kinit" + path: "/usr/bin/kinit" + shouldExist: true + permissions: "-rwxr-xr-x" + +fileContentTests: + - name: "entrypoint" + path: "/entrypoint" + expectedContents: + [ + "unzip -q \\*.zip", + "pip install -r requirements.txt", + "sh ./main_script", + "export CLASSPATH=`hadoop classpath --glob`" + ] + +commandTests: + - name: "ftp" + args: ["-h"] + command: "ftp" + exitCode: 0 + + - name: "wget" + args: ["--help"] + command: "wget" + exitCode: 0 + + - name: "curl" + args: ["--help"] + command: "curl" + exitCode: 0 + + - name: "unzip" + args: ["--help"] + command: "unzip" + exitCode: 0 + + - name: "tar" + args: ["--help"] + command: "tar" + exitCode: 0 + + - name: "telnet" + command: "which" + args: ["telnet"] + expectedOutput: ["/usr/bin/telnet"] + exitCode: 0 + + - name: "scp" + command: "which" + args: ["scp"] + expectedOutput: ["/usr/bin/scp"] + exitCode: 0 + + - name: "python installation" + command: "which" + args: ["python"] + expectedOutput: ["/usr/local/bin/python"] + + - name: "python version" + command: "python" + args: ["-V"] + expectedOutput: ["Python 3.12*"] + + - name: "krb5-user installation" + command: "kinit" + expectedError: ['kinit: Program lacks support for encryption type while getting initial credentials'] + exitCode: 1 + + - name: import_numpy + command: "python" + args: [ + "-c", + "import numpy;"] + exitCode: 0 + + - name: import_pandas + command: "python" + args: [ + "-c", + "import pandas;"] + exitCode: 0 + + - name: import_matplotlib + command: "python" + args: [ + "-c", + "import matplotlib;"] + exitCode: 0 + + - name: import_scipy + command: "python" + args: [ + "-c", + "import scipy;"] + exitCode: 0 + + - name: import_scikit-learn + command: "python" + args: [ + "-c", + "from sklearn import datasets;"] + exitCode: 0 + + - name: import_pyodbc + command: "python" + args: [ + "-c", + "import pyodbc;"] + exitCode: 0 + + - name: import_impyla + command: "python" + args: [ + "-c", + "from impala.dbapi import connect;"] + exitCode: 0 + + - name: import_hdfs + command: "python" + args: [ + "-c", + "import hdfs;"] + exitCode: 0 + + - name: import_thrift_sasl + command: "python" + args: [ + "-c", + "import thrift_sasl;"] + exitCode: 0 + + - name: thrift_sasl version + command: "bash" + args: [ + "-c", + "pip freeze | grep thrift-sasl"] + expectedOutput: ["thrift-sasl==0.4.3"] + exitCode: 0 + + - name: import_sqlalchemy.create_engine + command: "python" + args: [ + "-c", + "from sqlalchemy import create_engine;"] + exitCode: 0 + + - name: import_ibis + command: "python" + args: [ + "-c", + "import ibis;"] + exitCode: 0 + + - name: import_pymongo + command: "python" + args: [ + "-c", + "import pymongo;"] + exitCode: 0 + + - name: import_mysql.connector + command: "python" + args: [ + "-c", + "import mysql.connector;"] + exitCode: 0 + + - name: import_cython + command: "python" + args: [ + "-c", + "import cython;"] + exitCode: 0 + + - name: import_psycopg2 + command: "python" + args: [ + "-c", + "import psycopg2;"] + exitCode: 0 + + - name: import_fastparquet.ParquetFile + command: "python" + args: [ + "-c", + "from fastparquet import ParquetFile;"] + exitCode: 0 + + - name: import_request_kerberos + command: "python" + args: [ + "-c", + "from requests_kerberos import *;"] + exitCode: 0 + + - name: import_pyarrow + command: "python" + args: [ + "-c", + "from pyarrow import *;" ] + exitCode: 0 diff --git a/technologies/job/python/python-3.13/resources/requirements.txt b/technologies/job/python/python-3.13/resources/requirements.txt new file mode 100644 index 000000000..582bd0e14 --- /dev/null +++ b/technologies/job/python/python-3.13/resources/requirements.txt @@ -0,0 +1,24 @@ +#Datalake libraries +cython==3.0.10 +pyarrow==16.1.0 +hdfs[avro,dataframe,kerberos]==2.7.3 +kerberos==1.3.1 +requests-kerberos==0.15.0 +pyodbc==5.1.0 +SQLAlchemy==2.0.31 +fastparquet==2024.5.0 +boto3==1.34.144 +ibis-framework[impala]==9.1.0 +trino==0.329.0 + +#Datamart libraries +pymongo==4.8.0 +mysql-connector-python==9.0.0 +psycopg2-binary==2.9.9 + +#Data Science commmon libraries +numpy==1.26.4 +pandas==2.2.2 +matplotlib==3.9.1 +scikit-learn==1.5.1 +#saagieapi==2.12.1