-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
821470b
commit 875aa89
Showing
17 changed files
with
667 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
FROM python:3.13.0-slim-bullseye | ||
ENV DEBIAN_FRONTEND noninteractive | ||
|
||
ENV LC_ALL C | ||
ENV LD_RUN_PATH /usr/local/lib | ||
|
||
# LIGHT DEPENDENCIES START | ||
RUN apt-get update -qq && \ | ||
apt-get install -yqq --no-install-recommends \ | ||
ftp wget curl unzip telnet openssh-client git apt-transport-https ca-certificates \ | ||
krb5-user libpam-krb5 libpam-ccreds locales openjdk-11-jdk ca-certificates-java ant && \ | ||
apt-get clean && \ | ||
rm -rf /var/lib/apt/lists/* | ||
# LIGHT DEPENDENCIES END | ||
|
||
# UPGRADE PIP | ||
RUN pip --no-cache-dir install --upgrade pip | ||
|
||
# Add Hadoop CLI | ||
RUN cd / \ | ||
&& mkdir app \ | ||
&& cd app \ | ||
&& wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.6.5/hadoop-2.6.5.tar.gz \ | ||
&& tar xvf hadoop-2.6.5.tar.gz \ | ||
&& rm hadoop-2.6.5.tar.gz \ | ||
&& rm -rf hadoop-2.6.5/etc/hadoop \ | ||
&& ln -s /etc/hadoop/conf hadoop-2.6.5/etc/hadoop; | ||
|
||
# Fix Locale | ||
RUN sed -i '/^#.* en_US.* /s/^#//' /etc/locale.gen \ | ||
&& locale-gen | ||
|
||
# Fix certificate issues | ||
RUN update-ca-certificates -f; | ||
|
||
ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/ | ||
ENV HADOOP_HOME=/app/hadoop-2.6.5 | ||
ENV HADOOP_CONF_DIR=/etc/hadoop/conf | ||
ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/app/hadoop-2.6.5/lib/native:/usr/lib/jvm/java-11-openjdk-amd64/lib" | ||
ENV PATH "/app/hadoop-2.6.5/bin:${PATH}" | ||
ENV LC_ALL en_US.UTF-8 | ||
ENV LANG en_US.UTF-8 | ||
|
||
COPY entrypoint /entrypoint | ||
RUN chmod 755 /entrypoint | ||
|
||
WORKDIR /sandbox | ||
|
||
ENTRYPOINT ["bash","/entrypoint"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* Copyright 2019-2021. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* https://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
import com.bmuschko.gradle.docker.DockerRemoteApiPlugin | ||
import com.saagie.technologies.SaagieTechnologiesGradlePlugin | ||
|
||
apply<DockerRemoteApiPlugin>() | ||
apply<SaagieTechnologiesGradlePlugin>() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
image: saagie/python | ||
baseTag: 3.13-base | ||
dynamicVersion: 1.193.0_SDKTECHNO-275 | ||
version: 3.13-base-1.193.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
export CLASSPATH=`hadoop classpath --glob` | ||
|
||
if compgen -G "*.zip*" > /dev/null; then | ||
unzip -q *.zip | ||
fi | ||
if test -f requirements.txt; | ||
then | ||
echo "*******PIP INSTALL BEGIN*******" | ||
pip install -r requirements.txt; | ||
echo "*******PIP INSTALL END*******" | ||
fi; | ||
if [ $? != 0 ]; then echo "Error during 'pip install -f 'requirements.txt''";exit 1; fi; | ||
if test -f main_script; | ||
then sh ./main_script; | ||
else exec "$@" | ||
fi; |
100 changes: 100 additions & 0 deletions
100
technologies/job/python/python-3.13-base/image_test.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
schemaVersion: "2.0.0" | ||
|
||
metadataTest: | ||
env: | ||
- key: LANG | ||
value: "en_US.UTF-8" | ||
- key: LC_ALL | ||
value: "en_US.UTF-8" | ||
- key: LD_RUN_PATH | ||
value: "/usr/local/lib" | ||
- key: PATH | ||
value: "/app/hadoop-2.6.5/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" | ||
- key: "HADOOP_CONF_DIR" | ||
value: "/etc/hadoop/conf" | ||
- key: "JAVA_HOME" | ||
value: "/usr/lib/jvm/java-11-openjdk-amd64/" | ||
|
||
fileExistenceTests: | ||
- name: "unzip" | ||
path: "/usr/bin/unzip" | ||
shouldExist: true | ||
permissions: "-rwxr-xr-x" | ||
|
||
- name: "entrypoint" | ||
path: "/entrypoint" | ||
shouldExist: true | ||
permissions: "-rwxr-xr-x" | ||
|
||
- name: "kinit" | ||
path: "/usr/bin/kinit" | ||
shouldExist: true | ||
permissions: "-rwxr-xr-x" | ||
|
||
fileContentTests: | ||
- name: "entrypoint" | ||
path: "/entrypoint" | ||
expectedContents: | ||
[ | ||
"unzip -q \\*.zip", | ||
"pip install -r requirements.txt", | ||
"sh ./main_script", | ||
] | ||
|
||
commandTests: | ||
- name: "java version" | ||
command: "java" | ||
args: [ "-version" ] | ||
expectedError: [ 'openjdk version "11.*' ] | ||
|
||
- name: "ftp" | ||
args: ["-h"] | ||
command: "ftp" | ||
exitCode: 0 | ||
|
||
- name: "wget" | ||
args: ["--help"] | ||
command: "wget" | ||
exitCode: 0 | ||
|
||
- name: "curl" | ||
args: ["--help"] | ||
command: "curl" | ||
exitCode: 0 | ||
|
||
- name: "unzip" | ||
args: ["--help"] | ||
command: "unzip" | ||
exitCode: 0 | ||
|
||
- name: "tar" | ||
args: ["--help"] | ||
command: "tar" | ||
exitCode: 0 | ||
|
||
- name: "telnet" | ||
command: "which" | ||
args: ["telnet"] | ||
expectedOutput: ["/usr/bin/telnet"] | ||
exitCode: 0 | ||
|
||
- name: "scp" | ||
command: "which" | ||
args: ["scp"] | ||
expectedOutput: ["/usr/bin/scp"] | ||
exitCode: 0 | ||
|
||
- name: "python installation" | ||
command: "which" | ||
args: ["python"] | ||
expectedOutput: ["/usr/local/bin/python"] | ||
|
||
- name: "python version" | ||
command: "python" | ||
args: ["-V"] | ||
expectedOutput: ["Python 3.12*"] | ||
|
||
- name: "krb5-user installation" | ||
command: "kinit" | ||
expectedError: ['kinit: Program lacks support for encryption type while getting initial credentials'] | ||
exitCode: 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
FROM python:3.13.0-slim-bullseye | ||
|
||
ENV DEBIAN_FRONTEND noninteractive | ||
ENV LC_ALL en_US.UTF-8 | ||
ENV LANG en_US.UTF-8 | ||
|
||
# INSTALL DEPENDENCIES | ||
RUN apt-get update -qq && \ | ||
apt-get install -yqq --no-install-recommends unzip && \ | ||
apt-get clean && \ | ||
rm -rf /var/lib/apt/lists/* | ||
|
||
COPY entrypoint /entrypoint | ||
RUN chmod 755 /entrypoint | ||
|
||
WORKDIR /sandbox | ||
|
||
ENTRYPOINT ["bash","/entrypoint"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* Copyright 2019-2021. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* https://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
import com.bmuschko.gradle.docker.DockerRemoteApiPlugin | ||
import com.saagie.technologies.SaagieTechnologiesGradlePlugin | ||
|
||
apply<DockerRemoteApiPlugin>() | ||
apply<SaagieTechnologiesGradlePlugin>() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
id: "3.13-lite" | ||
label: "3.13-lite" | ||
available: true | ||
recommended: false | ||
trustLevel: stable | ||
job: | ||
features: | ||
- type: COMMAND_LINE | ||
label: Command line | ||
mandatory: true | ||
comment: Linux shell command to launch the job. | ||
defaultValue: python {file} arg1 arg2 | ||
- type: ARTIFACT | ||
label: Package | ||
mandatory: false | ||
comment: "Compatible upload file : .py or .zip" | ||
- type: SCHEDULER | ||
label: Scheduled | ||
mandatory: true | ||
- type: AI_DESCRIPTION_GENERATOR | ||
label: AI description generator enabled. | ||
mandatory: true | ||
comment: Activation of the AI-based automatic description generation function. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
image: saagie/python | ||
baseTag: 3.13.0-bullseye | ||
dynamicVersion: 1.193.0_SDKTECHNO-275 | ||
version: 3.13-lite-1.193.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
#!/bin/bash | ||
set -euo pipefail | ||
|
||
if compgen -G "*.zip*" > /dev/null; then | ||
unzip -q *.zip | ||
fi | ||
|
||
if test -f main_script; then | ||
sh ./main_script; | ||
else | ||
exec "$@" | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
schemaVersion: "2.0.0" | ||
|
||
metadataTest: | ||
env: | ||
- key: LANG | ||
value: "en_US.UTF-8" | ||
- key: LC_ALL | ||
value: "en_US.UTF-8" | ||
|
||
fileExistenceTests: | ||
- name: "unzip" | ||
path: "/usr/bin/unzip" | ||
shouldExist: true | ||
permissions: "-rwxr-xr-x" | ||
|
||
- name: "entrypoint" | ||
path: "/entrypoint" | ||
shouldExist: true | ||
permissions: "-rwxr-xr-x" | ||
|
||
fileContentTests: | ||
- name: "entrypoint" | ||
path: "/entrypoint" | ||
expectedContents: | ||
[ | ||
"unzip -q \\*.zip", | ||
"sh ./main_script", | ||
] | ||
|
||
commandTests: | ||
- name: "unzip" | ||
args: ["--help"] | ||
command: "unzip" | ||
exitCode: 0 | ||
|
||
- name: "python installation" | ||
command: "which" | ||
args: ["python"] | ||
expectedOutput: ["/usr/local/bin/python"] | ||
|
||
- name: "python version" | ||
command: "python" | ||
args: ["-V"] | ||
expectedOutput: ["Python 3.12*"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
ARG base_img | ||
|
||
FROM ${base_img} AS BASE_IMG | ||
|
||
# LIBS PART BEGIN | ||
RUN apt update -qq && apt install -qqy --no-install-recommends \ | ||
python3-pip \ | ||
gcc \ | ||
g++ \ | ||
libsasl2-2 \ | ||
libsasl2-modules-ldap \ | ||
build-essential \ | ||
unixodbc \ | ||
unixodbc-dev \ | ||
libpq-dev \ | ||
libsqlite3-dev \ | ||
libkrb5-dev \ | ||
libsasl2-dev \ | ||
libssl-dev \ | ||
libcurl4-openssl-dev \ | ||
libgeos-dev \ | ||
swig \ | ||
python3-matplotlib \ | ||
python3-lxml \ | ||
cmake \ | ||
&& rm -rf /var/lib/apt/lists/*; | ||
# LIBS PART END | ||
|
||
# Need to read requirements.txt file sequentially to ensure consistent installation order | ||
COPY resources/requirements.txt /tmp/requirements.txt | ||
RUN grep -v '^#' /tmp/requirements.txt | xargs -L 1 pip --no-cache-dir install \ | ||
&& rm -rf /root/.cachex \ | ||
&& rm -rf /boot/.cache/pip \ | ||
&& rm -rf ~/.cache/pip | ||
|
Oops, something went wrong.