Cookie cutter initial template (flyteorg#738)

* Add cookiecutter to requirements and pyflyte init Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Invoke cookiecutter with the overridden configuration. Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Comment the use of directory in the call to cookiecutter. Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Use the original repo and do not clobber existing files+directories Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Add example sub-command Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * make requirements Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Fix test_dataclass_transformer test Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Constrain mashmallow-jsonschema temporarily Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Add --template flag and project-name argument to command Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Revert changes to dataclass transformer tests Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Constrain marshmallow-jsonschema in dev-requirements Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Revert "Constrain marshmallow-jsonschema in dev-requirements" This reverts commit bf7f22d. * Fix typo in Makefile Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Fix regeneration of spark2 requirements file Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Use click for prompts Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Changes default workflow name to `my_wf` Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Remove the two inputs from simple-example Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Add more descriptive help for the template flag Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Set cookiecutter in setup.py Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Default top level directory to flyte (as opposed to myapp) Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> Co-authored-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> Signed-off-by: Robert Everson <reverson@lyft.com>
reverson · May 27, 2022 · b5a2689 · b5a2689
1 parent fe89f96
commit b5a2689
Show file tree

Hide file tree

Showing 10 changed files with 219 additions and 41 deletions.
diff --git a/Makefile b/Makefile
@@ -37,7 +37,7 @@ lint: ## Run linters
 	mypy flytekit/core || true
 	mypy flytekit/types || true
 	mypy tests/flytekit/unit/core || true
-	# Exclude setup.py to fix erorr: Duplicate module named "setup"
+	# Exclude setup.py to fix error: Duplicate module named "setup"
 	mypy plugins --exclude setup.py || true
 	pre-commit run --all-files
 

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -8,6 +8,10 @@
     # via
     #   -c requirements.txt
     #   pytest-flyte
+arrow==1.2.1
+    # via
+    #   -c requirements.txt
+    #   jinja2-time
 attrs==20.3.0
     # via
     #   -c requirements.txt
@@ -20,6 +24,10 @@ bcrypt==3.2.0
     # via
     #   -c requirements.txt
     #   paramiko
+binaryornot==0.4.4
+    # via
+    #   -c requirements.txt
+    #   cookiecutter
 certifi==2021.10.8
     # via
     #   -c requirements.txt
@@ -32,6 +40,10 @@ cffi==1.15.0
     #   pynacl
 cfgv==3.3.1
     # via pre-commit
+chardet==4.0.0
+    # via
+    #   -c requirements.txt
+    #   binaryornot
 charset-normalizer==2.0.7
     # via
     #   -c requirements.txt
@@ -43,13 +55,18 @@ checksumdir==1.2.0
 click==7.1.2
     # via
     #   -c requirements.txt
+    #   cookiecutter
     #   flytekit
 cloudpickle==2.0.0
     # via
     #   -c requirements.txt
     #   flytekit
 codespell==2.1.0
     # via -r dev-requirements.in
+cookiecutter==1.7.3
+    # via
+    #   -c requirements.txt
+    #   flytekit
 coverage[toml]==6.1.1
     # via -r dev-requirements.in
 croniter==1.0.15
@@ -60,6 +77,7 @@ cryptography==35.0.0
     # via
     #   -c requirements.txt
     #   paramiko
+    #   secretstorage
 dataclasses-json==0.5.6
     # via
     #   -c requirements.txt
@@ -108,7 +126,7 @@ grpcio==1.41.1
     # via
     #   -c requirements.txt
     #   flytekit
-identify==2.3.3
+identify==2.3.4
     # via pre-commit
 idna==3.3
     # via
@@ -120,10 +138,21 @@ importlib-metadata==4.8.1
     #   keyring
 iniconfig==1.1.1
     # via pytest
+jeepney==0.7.1
+    # via
+    #   -c requirements.txt
+    #   keyring
+    #   secretstorage
 jinja2==3.0.2
     # via
     #   -c requirements.txt
+    #   cookiecutter
+    #   jinja2-time
     #   pytest-flyte
+jinja2-time==0.2.0
+    # via
+    #   -c requirements.txt
+    #   cookiecutter
 joblib==1.1.0
     # via -r dev-requirements.in
 jsonschema==3.2.0
@@ -190,6 +219,10 @@ platformdirs==2.4.0
     #   virtualenv
 pluggy==1.0.0
     # via pytest
+poyo==0.5.0
+    # via
+    #   -c requirements.txt
+    #   cookiecutter
 pre-commit==2.15.0
     # via -r dev-requirements.in
 protobuf==3.19.1
@@ -206,7 +239,7 @@ pyarrow==3.0.0
     # via
     #   -c requirements.txt
     #   flytekit
-pycparser==2.20
+pycparser==2.21
     # via
     #   -c requirements.txt
     #   cffi
@@ -234,6 +267,7 @@ pytest-flyte @ git+git://github.com/flyteorg/pytest-flyte@main
 python-dateutil==2.8.1
     # via
     #   -c requirements.txt
+    #   arrow
     #   croniter
     #   flytekit
     #   pandas
@@ -243,6 +277,10 @@ python-json-logger==2.0.2
     # via
     #   -c requirements.txt
     #   flytekit
+python-slugify==5.0.2
+    # via
+    #   -c requirements.txt
+    #   cookiecutter
 pytimeparse==1.1.8
     # via
     #   -c requirements.txt
@@ -264,6 +302,7 @@ regex==2021.11.2
 requests==2.26.0
     # via
     #   -c requirements.txt
+    #   cookiecutter
     #   docker
     #   docker-compose
     #   flytekit
@@ -276,10 +315,15 @@ retry==0.9.2
     # via
     #   -c requirements.txt
     #   flytekit
+secretstorage==3.3.1
+    # via
+    #   -c requirements.txt
+    #   keyring
 six==1.16.0
     # via
     #   -c requirements.txt
     #   bcrypt
+    #   cookiecutter
     #   dockerpty
     #   flytekit
     #   grpcio
@@ -297,6 +341,10 @@ statsd==3.3.0
     # via
     #   -c requirements.txt
     #   flytekit
+text-unidecode==1.3
+    # via
+    #   -c requirements.txt
+    #   python-slugify
 texttable==1.6.4
     # via docker-compose
 toml==0.10.2

diff --git a/doc-requirements.txt b/doc-requirements.txt
@@ -10,10 +10,8 @@ alabaster==0.7.12
     # via sphinx
 ansiwrap==0.8.4
     # via papermill
-appnope==0.1.2
-    # via
-    #   ipykernel
-    #   ipython
+arrow==1.2.1
+    # via jinja2-time
 astroid==2.8.4
     # via sphinx-autoapi
 attrs==21.2.0
@@ -29,13 +27,15 @@ beautifulsoup4==4.10.0
     #   furo
     #   sphinx-code-include
     #   sphinx-material
+binaryornot==0.4.4
+    # via cookiecutter
 black==21.10b0
     # via papermill
 bleach==4.1.0
     # via nbconvert
-boto3==1.19.12
+boto3==1.20.0
     # via sagemaker-training
-botocore==1.22.12
+botocore==1.23.0
     # via
     #   boto3
     #   s3transfer
@@ -46,24 +46,30 @@ cffi==1.15.0
     #   bcrypt
     #   cryptography
     #   pynacl
+chardet==4.0.0
+    # via binaryornot
 charset-normalizer==2.0.7
     # via requests
 checksumdir==1.2.0
     # via flytekit
 click==7.1.2
     # via
     #   black
+    #   cookiecutter
     #   flytekit
     #   hmsclient
     #   papermill
 cloudpickle==2.0.0
     # via flytekit
+cookiecutter==1.7.3
+    # via flytekit
 croniter==1.0.15
     # via flytekit
 cryptography==35.0.0
     # via
     #   -r doc-requirements.in
     #   paramiko
+    #   secretstorage
 css-html-js-minify==2.5.5
     # via sphinx-material
 dataclasses-json==0.5.6
@@ -123,11 +129,19 @@ ipython-genutils==0.2.0
     #   nbformat
 jedi==0.18.0
     # via ipython
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.2
     # via
+    #   cookiecutter
+    #   jinja2-time
     #   nbconvert
     #   sphinx
     #   sphinx-autoapi
+jinja2-time==0.2.0
+    # via cookiecutter
 jmespath==0.10.0
     # via
     #   boto3
@@ -174,7 +188,7 @@ mypy-extensions==0.4.3
     #   typing-inspect
 natsort==8.0.0
     # via flytekit
-nbclient==0.5.4
+nbclient==0.5.5
     # via
     #   nbconvert
     #   papermill
@@ -218,6 +232,8 @@ pickleshare==0.7.5
     # via ipython
 platformdirs==2.4.0
     # via black
+poyo==0.5.0
+    # via cookiecutter
 prompt-toolkit==3.0.22
     # via ipython
 protobuf==3.19.1
@@ -236,7 +252,7 @@ py4j==0.10.9.2
     # via pyspark
 pyarrow==3.0.0
     # via flytekit
-pycparser==2.20
+pycparser==2.21
     # via cffi
 pygments==2.10.0
     # via
@@ -255,6 +271,7 @@ pyspark==3.2.0
     # via flytekit
 python-dateutil==2.8.1
     # via
+    #   arrow
     #   botocore
     #   croniter
     #   flytekit
@@ -263,7 +280,9 @@ python-dateutil==2.8.1
 python-json-logger==2.0.2
     # via flytekit
 python-slugify[unidecode]==5.0.2
-    # via sphinx-material
+    # via
+    #   cookiecutter
+    #   sphinx-material
 pytimeparse==1.1.8
     # via flytekit
 pytz==2018.4
@@ -283,6 +302,7 @@ regex==2021.11.2
     #   docker-image-py
 requests==2.26.0
     # via
+    #   cookiecutter
     #   flytekit
     #   papermill
     #   responses
@@ -297,12 +317,15 @@ s3transfer==0.5.0
     # via boto3
 sagemaker-training==3.9.2
     # via flytekit
-scipy==1.7.1
+scipy==1.7.2
     # via sagemaker-training
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   bcrypt
     #   bleach
+    #   cookiecutter
     #   flytekit
     #   grpcio
     #   pynacl

diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py
@@ -0,0 +1,38 @@
+import click
+from cookiecutter.main import cookiecutter
+
+
+@click.command("init")
+@click.option(
+    "--template",
+    default="simple-example",
+    help="cookiecutter template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git",
+)
+@click.argument("project-name")
+def init(template, project_name):
+    """
+    Create flyte-ready projects.
+    """
+    config = {
+        "project_name": project_name,
+        "app": "flyte",
+        "workflow": "my_wf",
+    }
+    cookiecutter(
+        "https://github.com/flyteorg/flytekit-python-template.git",
+        # TODO: remove this once we make the transition to cookie-cutter official.
+        checkout="cookie-cutter",
+        no_input=True,
+        # We do not want to clobber existing files/directories.
+        overwrite_if_exists=False,
+        extra_context=config,
+        # By specifying directory we can have multiple templates in the same repository,
+        # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html.
+        # The idea is to extend the number of templates, each in their own subdirectory, for example
+        # a tensorflow-based example.
+        directory=template,
+    )
+
+    click.echo(
+        f"Visit the {project_name} directory and follow the next steps in the Getting started guide (https://docs.flyte.org/en/latest/getting_started.html) to proceed."
+    )
diff --git a/flytekit/clis/sdk_in_container/pyflyte.py b/flytekit/clis/sdk_in_container/pyflyte.py
@@ -6,6 +6,7 @@
 
 from flytekit.clis.sdk_in_container.constants import CTX_PACKAGES
 from flytekit.clis.sdk_in_container.fast_register import fast_register
+from flytekit.clis.sdk_in_container.init import init
 from flytekit.clis.sdk_in_container.launch_plan import launch_plans
 from flytekit.clis.sdk_in_container.local_cache import local_cache
 from flytekit.clis.sdk_in_container.package import package
@@ -112,6 +113,7 @@ def update_configuration_file(config_file_path):
 main.add_command(launch_plans)
 main.add_command(package)
 main.add_command(local_cache)
+main.add_command(init)
 
 if __name__ == "__main__":
     main()
diff --git a/requirements-spark2.in b/requirements-spark2.in
@@ -1,2 +1,2 @@
 .[all-spark2.4]
--c requirements.in
+-r requirements.in