Sdktechno 264 - python 3.12

technologies/job/python/metadata.yaml

@@ -94,6 +94,33 @@ contexts:
  image: "saagie/python"
  baseTag: "3.11"
  version: "3.11-1.181.0_SDKTECHNO-235"
+ - id: "3.12"
+ label: "3.12"
+ available: true
+ recommended: true
+ trustLevel: stable
+ job:
+ features:
+ - type: COMMAND_LINE
+ label: Command line
+ mandatory: true
+ comment: Linux shell command to launch the job.
+ defaultValue: python {file} arg1 arg2
+ - type: ARTIFACT
+ label: Package
+ mandatory: true
+ comment: "Compatible upload file : .py or .zip (with optional requirements.txt for pip install)"
+ - type: SCHEDULER
+ label: Scheduled
+ mandatory: true
+ - type: AI_DESCRIPTION_GENERATOR
+ label: AI description generator enabled
+ mandatory: true
+ comment: Activation of the AI-based automatic description generation function.
+ dockerInfo:
+ image: "saagie/python"
+ baseTag: "3.12"
+ version: "3.12-1.183.0_SDKTECHNO-264"
  - id: "3.7"
  label: "3.7"
  available: true

technologies/job/python/python-3.12-base/Dockerfile

@@ -0,0 +1,49 @@
+FROM python:3.12.4-slim-bullseye
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV LC_ALL C
+ENV LD_RUN_PATH /usr/local/lib
+
+# LIGHT DEPENDENCIES START
+RUN apt-get update -qq && \
+ apt-get install -yqq --no-install-recommends \
+ ftp wget curl unzip telnet openssh-client git apt-transport-https ca-certificates \
+ krb5-user libpam-krb5 libpam-ccreds locales openjdk-11-jdk ca-certificates-java ant && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+# LIGHT DEPENDENCIES END
+
+# UPGRADE PIP
+RUN pip --no-cache-dir install --upgrade pip
+
+# Add Hadoop CLI
+RUN cd / \
+ && mkdir app \
+ && cd app \
+ && wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.6.5/hadoop-2.6.5.tar.gz \
+ && tar xvf hadoop-2.6.5.tar.gz \
+ && rm hadoop-2.6.5.tar.gz \
+ && rm -rf hadoop-2.6.5/etc/hadoop \
+ && ln -s /etc/hadoop/conf hadoop-2.6.5/etc/hadoop;
+
+# Fix Locale
+RUN sed -i '/^#.* en_US.* /s/^#//' /etc/locale.gen \
+ && locale-gen
+
+# Fix certificate issues
+RUN update-ca-certificates -f;
+
+ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
+ENV HADOOP_HOME=/app/hadoop-2.6.5
+ENV HADOOP_CONF_DIR=/etc/hadoop/conf
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/app/hadoop-2.6.5/lib/native:/usr/lib/jvm/java-11-openjdk-amd64/lib"
+ENV PATH "/app/hadoop-2.6.5/bin:${PATH}"
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+COPY entrypoint /entrypoint
+RUN chmod 755 /entrypoint
+
+WORKDIR /sandbox
+
+ENTRYPOINT ["bash","/entrypoint"]

technologies/job/python/python-3.12-base/build.gradle.kts

@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019-2021.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()

technologies/job/python/python-3.12-base/dockerInfo.yaml

@@ -0,0 +1,4 @@
+image: saagie/python
+baseTag: 3.12-base
+dynamicVersion: 1.183.0_SDKTECHNO-264
+version: 3.12-base-1.183.0_SDKTECHNO-264

technologies/job/python/python-3.12-base/entrypoint

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export CLASSPATH=`hadoop classpath --glob`
+
+if compgen -G "*.zip*" > /dev/null; then
+ unzip -q *.zip
+fi
+if test -f requirements.txt;
+then
+ echo "*******PIP INSTALL BEGIN*******"
+ pip install -r requirements.txt;
+ echo "*******PIP INSTALL END*******"
+fi;
+if [ $? != 0 ]; then echo "Error during 'pip install -f 'requirements.txt''";exit 1; fi;
+if test -f main_script;
+then sh ./main_script;
+else exec "$@"
+fi;

technologies/job/python/python-3.12-base/image_test.yaml

@@ -0,0 +1,100 @@
+schemaVersion: "2.0.0"
+
+metadataTest:
+ env:
+ - key: LANG
+ value: "en_US.UTF-8"
+ - key: LC_ALL
+ value: "en_US.UTF-8"
+ - key: LD_RUN_PATH
+ value: "/usr/local/lib"
+ - key: PATH
+ value: "/app/hadoop-2.6.5/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ - key: "HADOOP_CONF_DIR"
+ value: "/etc/hadoop/conf"
+ - key: "JAVA_HOME"
+ value: "/usr/lib/jvm/java-11-openjdk-amd64/"
+
+fileExistenceTests:
+ - name: "unzip"
+ path: "/usr/bin/unzip"
+ shouldExist: true
+ permissions: "-rwxr-xr-x"
+
+ - name: "entrypoint"
+ path: "/entrypoint"
+ shouldExist: true
+ permissions: "-rwxr-xr-x"
+
+ - name: "kinit"
+ path: "/usr/bin/kinit"
+ shouldExist: true
+ permissions: "-rwxr-xr-x"
+
+fileContentTests:
+ - name: "entrypoint"
+ path: "/entrypoint"
+ expectedContents:
+ [
+ "unzip -q \\*.zip",
+ "pip install -r requirements.txt",
+ "sh ./main_script",
+ ]
+
+commandTests:
+ - name: "java version"
+ command: "java"
+ args: [ "-version" ]
+ expectedError: [ 'openjdk version "11.*' ]
+
+ - name: "ftp"
+ args: ["-h"]
+ command: "ftp"
+ exitCode: 0
+
+ - name: "wget"
+ args: ["--help"]
+ command: "wget"
+ exitCode: 0
+
+ - name: "curl"
+ args: ["--help"]
+ command: "curl"
+ exitCode: 0
+
+ - name: "unzip"
+ args: ["--help"]
+ command: "unzip"
+ exitCode: 0
+
+ - name: "tar"
+ args: ["--help"]
+ command: "tar"
+ exitCode: 0
+
+ - name: "telnet"
+ command: "which"
+ args: ["telnet"]
+ expectedOutput: ["/usr/bin/telnet"]
+ exitCode: 0
+
+ - name: "scp"
+ command: "which"
+ args: ["scp"]
+ expectedOutput: ["/usr/bin/scp"]
+ exitCode: 0
+
+ - name: "python installation"
+ command: "which"
+ args: ["python"]
+ expectedOutput: ["/usr/local/bin/python"]
+
+ - name: "python version"
+ command: "python"
+ args: ["-V"]
+ expectedOutput: ["Python 3.12*"]
+
+ - name: "krb5-user installation"
+ command: "kinit"
+ expectedError: ['kinit: Program lacks support for encryption type while getting initial credentials']
+ exitCode: 1

technologies/job/python/python-3.12/Dockerfile

@@ -0,0 +1,35 @@
+ARG base_img
+
+FROM ${base_img} AS BASE_IMG
+
+# LIBS PART BEGIN
+RUN apt update -qq && apt install -qqy --no-install-recommends \
+ python3-pip \
+ gcc \
+ g++ \
+ libsasl2-2 \
+ libsasl2-modules-ldap \
+ build-essential \
+ unixodbc \
+ unixodbc-dev \
+ libpq-dev \
+ libsqlite3-dev \
+ libkrb5-dev \
+ libsasl2-dev \
+ libssl-dev \
+ libcurl4-openssl-dev \
+ libgeos-dev \
+ swig \
+ python3-matplotlib \
+ python3-lxml \
+ cmake \
+ && rm -rf /var/lib/apt/lists/*;
+# LIBS PART END
+
+# Need to read requirements.txt file sequentially to ensure consistent installation order
+COPY resources/requirements.txt /tmp/requirements.txt
+RUN grep -v '^#' /tmp/requirements.txt | xargs -L 1 pip --no-cache-dir install \
+ && rm -rf /root/.cachex \
+ && rm -rf /boot/.cache/pip \
+ && rm -rf ~/.cache/pip
+

technologies/job/python/python-3.12/build.gradle.kts

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019-2021.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+import com.saagie.technologies.readDockerInfo
+import com.saagie.technologies.getVersionForDocker
+
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()
+
+val dockerInfo = readDockerInfo(projectDir)
+
+tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
+ dependsOn(":${this.project.name}-base:testImage")
+ this.buildArgs.put(
+ "base_img",
+ "${dockerInfo?.image}:${dockerInfo?.baseTag}-base-${this.project.getVersionForDocker()}"
+ )
+}

technologies/job/python/python-3.12/context.yaml

@@ -0,0 +1,23 @@
+id: "3.12"
+label: "3.12"
+available: true
+recommended: true
+trustLevel: stable
+job:
+ features:
+ - type: COMMAND_LINE
+ label: Command line
+ mandatory: true
+ comment: Linux shell command to launch the job.
+ defaultValue: python {file} arg1 arg2
+ - type: ARTIFACT
+ label: Package
+ mandatory: true
+ comment: "Compatible upload file : .py or .zip (with optional requirements.txt for pip install)"
+ - type: SCHEDULER
+ label: Scheduled
+ mandatory: true
+ - type: AI_DESCRIPTION_GENERATOR
+ label: AI description generator enabled
+ mandatory: true
+ comment: Activation of the AI-based automatic description generation function.

technologies/job/python/python-3.12/dockerInfo.yaml

@@ -0,0 +1,4 @@
+image: saagie/python
+baseTag: 3.12
+dynamicVersion: 1.183.0_SDKTECHNO-264
+version: 3.12-1.183.0_SDKTECHNO-264