diff --git a/contents_docker_image_kpo/Dockerfile b/contents_docker_image_kpo/Dockerfile new file mode 100644 index 0000000..3799a8d --- /dev/null +++ b/contents_docker_image_kpo/Dockerfile @@ -0,0 +1,19 @@ +# Dockerfile for tjanif/docker-sandbox:snowpark-example +FROM python:3.8.14 + +WORKDIR / + +# installing packages +COPY requirements.txt ./ +RUN pip install --no-cache-dir --upgrade pip +RUN pip install -r requirements.txt + +# creating the file to write XComs to +RUN mkdir -p airflow/xcom +RUN echo "" > airflow/xcom/return.json + +# copy the script into the Docker container +COPY script.py ./ + +# run the script +CMD ["python", "./script.py"] \ No newline at end of file diff --git a/contents_docker_image_kpo/requirements.txt b/contents_docker_image_kpo/requirements.txt new file mode 100644 index 0000000..52f82c0 --- /dev/null +++ b/contents_docker_image_kpo/requirements.txt @@ -0,0 +1 @@ +snowflake-snowpark-python[pandas] \ No newline at end of file diff --git a/contents_docker_image_kpo/script.py b/contents_docker_image_kpo/script.py new file mode 100644 index 0000000..95b59ff --- /dev/null +++ b/contents_docker_image_kpo/script.py @@ -0,0 +1,33 @@ +import os +import json +from snowflake.snowpark import Session + +# connection parameters are stored as variables in .env +connection_parameters = { + "account": os.environ['SNOWFLAKE_ACCOUNT'], + "user": os.environ['SNOWFLAKE_USER'], + "password": os.environ['SNOWFLAKE_PASSWORD'], + "role": os.environ['SNOWFLAKE_ROLE'], + "warehouse": os.environ['SNOWFLAKE_WAREHOUSE'], + "database": os.environ['SNOWFLAKE_DATABASE'], + "schema": os.environ['SNOWFLAKE_SCHEMA'], + "region": os.environ['SNOWFLAKE_REGION'] +} + +# create Snowpark session +session = Session.builder.configs(connection_parameters).create() +# query Snowflake database and print the result +df = session.sql(os.environ['query']) +print(df.collect()) + +# write result to XComs folder that was created in the Dockerfile +return_json = {"return_value": str({df.collect()[0]})} +json_object = json.dumps(return_json, indent=4) + +f = open('./airflow/xcom/return.json', 'w') +f.write(json_object) +f.close() + +# close Snowpark session +session.close() + diff --git a/dockerfile_just_pyvirtualenv/Dockerfile b/dockerfile_just_pyvirtualenv/Dockerfile new file mode 100644 index 0000000..44b0723 --- /dev/null +++ b/dockerfile_just_pyvirtualenv/Dockerfile @@ -0,0 +1,17 @@ +FROM quay.io/astronomer/astro-runtime:6.0.4 + +##### Docker Customizations below this line ##### + +## this is the default directory where pyenv will be installed, you can chose a different path as well +ENV PYENV_ROOT="/home/astro/.pyenv" +# it is important to add the folder where the python version will be installed to PATH in order to retrieve it in the PythonVirtualEnvOperator +ENV PATH=${PYENV_ROOT}/bin:/home/astro/.pyenv/versions/3.8.14/bin:${PATH} + +## if you ever want to check your dependency conflicts for extra packages that you may require for your venv, this requires you to install pip-tools +# RUN pip-compile -h +# RUN pip-compile snowpark_requirements.txt + +## install pyenv, install the required version +RUN curl https://pyenv.run | bash && \ + eval "$(pyenv init -)" && \ + pyenv install 3.8.14 \ No newline at end of file