From df0e26c58eba48dd8eaf7eadec5775f617ce0dc9 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 18 Apr 2022 18:33:55 -0400 Subject: [PATCH] feat: introduce `tutor dev quickstart` Add `tutor dev quickstart` command, which is equivalent to `tutor local quickstart`, but uses dev containers instead of local production ones and includes some other small differences for the convience of Open edX developers. This should remove some friction from the Open edX development setup process, which previously required that users provision using local producation containers but then stop them and switch to dev containers: * tutor local quickstart * tutor local stop * tutor dev start -d Document the command and its improved workflow in ./docs/tutorials/nightly.rst Fixes overhangio/2u-tutor-adoption#58 --- CHANGELOG.md | 1 + docs/dev.rst | 80 +++++++++++++++++++++++++++++++--------- docs/install.rst | 3 +- docs/intro.rst | 2 +- tests/test_config.py | 3 +- tutor/commands/config.py | 4 +- tutor/commands/dev.py | 57 +++++++++++++++++++++++++++- tutor/commands/k8s.py | 11 ++++-- tutor/commands/local.py | 8 +++- tutor/interactive.py | 36 +++++++++--------- 10 files changed, 159 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3b31c5db8..aa661461f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +- [Feature] Add `tutor dev quickstart` command, which is similar to `tutor local quickstart`, except that it uses dev containers instead of local production ones and includes some other small differences for the convience of Open edX developers. This should remove some friction from the Open edX development setup process, which previously required that users provision using local producation containers (`tutor local quickstart`) but then stop them and switch to dev containers (`tutor local stop && tutor dev start -d`). - 💥[Improvement] Make it possible to run `tutor k8s exec ` (#636). As a consequence, it is no longer possible to run quoted commands: `tutor k8s exec ""`. Instead, you should remove the quotes: `tutor k8s exec `. - 💥[Deprecation] Drop support for the `TUTOR_EDX_PLATFORM_SETTINGS` environment variable. It is now recommended to create a plugin instead. - 💥[Improvement] Complete overhaul of the plugin extension mechanism. Tutor now has a hook-based Python API: actions can be triggered at different points of the application life cycle and data can be modified thanks to custom filters. The v0 plugin API is still supported, for backward compatibility, but plugin developers are encouraged to migrate their plugins to the new API. See the new plugin tutorial for more information. diff --git a/docs/dev.rst b/docs/dev.rst index 87a2600620..283bd4a7d6 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -5,37 +5,61 @@ Open edX development In addition to running Open edX in production, Tutor can be used for local development of Open edX. This means that it is possible to hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX. -The following commands assume you have previously launched a :ref:`local ` Open edX platform. If you have not done so already, you should run:: - tutor local quickstart +First-time setup +---------------- -To run the platform in development mode, you **must** answer no ("n") to the question "Are you configuring a production platform". +First, ensure you have already `installed Tutor `_ (for development against the named releases of Open edX) or `Tutor Nightly `_ (for development against Open edX's master branches). -Note that the local.overhang.io `domain `__ and its `subdomains `__ all point to 127.0.0.1. This is just a domain name that was set up to conveniently access a locally running Open edX platform. +Then, launch the developer platform setup process:: -Once the local platform has been configured, you should stop it so that it does not interfere with the development environment:: + tutor dev quickstart - tutor local stop +This will perform several tasks for you. It will: -Finally, you should build the ``openedx-dev`` docker image:: +* stop any existing locally-running Tutor containers, - tutor dev dc build lms +* disable HTTPS, -This ``openedx-dev`` development image differs from the ``openedx`` production image: +* set your ``LMS_HOST`` to `local.overhang.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), -- The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). -- Additional python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. -- The edx-platform `development requirements `__ are installed. +* prompt for a platform details (with suitable defaults), + +* build an ``openedx-dev`` image, which is based ``openedx`` production image but is `specialized for developer usage`_, + +* start LMS, CMS, supporting services, and any plugged-in services, + +* ensure databases are created and migrated, and + +* run service initialization scripts, such as service user creation and Waffle configuration. + +Once setup is complete, the platform will be running in the background: + +* LMS will be accessible at `http://local.overhang.io:8000 `_. +* CMS will be accessible at `http://studio.local.overhang.io:8001 `_. +* Plugged-in services should be accessible at their documented URLs. + + +Stopping the platform +--------------------- + +To bring down your platform's containers, simply run:: + + tutor dev stop + + +Starting the platform back up +----------------------------- -Since the ``openedx-dev`` is based upon the ``openedx`` docker image, it should be re-built every time the ``openedx`` docker image is modified. +Once you have used ``quickstart`` once, you can start the platform in the future with the lighter-weight ``start`` command, which brings up containers but does not perform any initialization tasks:: -Run a local development webserver ---------------------------------- + tutor dev start # Run platform in the same terminal ("attached") + tutor dev start -d # Or, run platform the in the background ("detached") -:: +Nonetheless, ``quickstart`` is idempotent, so it is always safe to run it again in the future without risk to your data. In fact, you may find it useful to use this command as a one-stop-shop for pulling images, running migrations, initializing new plugins you have enabled, and/or executing any new initialization steps that may have been introduced since you set up Tutor:: + + tutor dev quickstart --pullimages - tutor dev runserver lms # Access the lms at http://local.overhang.io:8000 - tutor dev runserver cms # Access the cms at http://studio.local.overhang.io:8001 Running arbitrary commands -------------------------- @@ -56,6 +80,26 @@ To collect assets, you can use the ``openedx-assets`` command that ships with Tu tutor dev run lms openedx-assets build --env=dev + +.. _specialized for developer usage: + +Rebuilding the openedx-dev image +-------------------------------- + +The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by ``tutor local ...`` to run LMS and CMS. However, it has a few differences to make it more convenient for developers: + +- The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). + +- Additional Python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. + +- The edx-platform `development requirements `__ are installed. + + +If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: + + tutor dev dc build lms + + .. _bind_mounts: Sharing directories with containers diff --git a/docs/install.rst b/docs/install.rst index 1ebc9c5d5e..f51e417186 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -91,9 +91,10 @@ To upgrade Open edX or benefit from the latest features and bug fixes, you shoul pip install --upgrade "tutor[full]" -Then run the ``quickstart`` command again. Depending on your deployment target, run either:: +Then run the ``quickstart`` command again. Depending on your deployment target, run one of:: tutor local quickstart # for local installations + tutor dev quickstart # for local development installations tutor k8s quickstart # for Kubernetes installation Upgrading with custom Docker images diff --git a/docs/intro.rst b/docs/intro.rst index 6e9918a33e..3f25e287f8 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -129,7 +129,7 @@ Many commands can be further parameterized to specify their target and options, tutor k8s logs mysql # View logs of MySQL in Kubernetes-managed deployment. tutor dev logs lms --tail 10 # View ten lines of logs of the LMS container in development mode. -And that's it! You do not need to understand Tutor's entire command-line interface to get started. Using the ``--help`` option that's availble on every command, it is easy to learn as you go. For an in-depth guide, you can also explore the `CLI Reference `_. +And that's it! You do not need to understand Tutor's entire command-line interface to get started. Using the ``--help`` option that's availble on every command, it is easy to learn as you go. For an in-depth guide, you can also explore the `CLI Reference `_. I'm ready, where do I start? ---------------------------- diff --git a/tests/test_config.py b/tests/test_config.py index b2e1be61ea..94b6f66bee 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -47,7 +47,8 @@ def mock_prompt(*_args: None, **kwargs: str) -> str: with temporary_root() as rootdir: with patch.object(click, "prompt", new=mock_prompt): with patch.object(click, "confirm", new=mock_prompt): - config = interactive.load_user_config(rootdir, interactive=True) + config = tutor_config.load_minimal(rootdir) + interactive.ask_questions(config) self.assertIn("MYSQL_ROOT_PASSWORD", config) self.assertEqual(8, len(get_typed(config, "MYSQL_ROOT_PASSWORD", str))) diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 644a5952e0..b329c95819 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -48,7 +48,9 @@ def save( unset_vars: List[str], env_only: bool, ) -> None: - config = interactive_config.load_user_config(context.root, interactive=interactive) + config = tutor_config.load_minimal(context.root) + if interactive: + interactive_config.ask_questions(config) if set_vars: for key, value in dict(set_vars).items(): config[key] = env.render_unknown(config, value) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 62fbaeaa46..c8a647ea07 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -4,7 +4,9 @@ from .. import config as tutor_config from .. import env as tutor_env -from .. import fmt +from .. import exceptions, fmt +from .. import interactive as interactive_config +from .. import utils from ..types import Config, get_typed from . import compose @@ -41,6 +43,58 @@ def dev(context: click.Context) -> None: context.obj = DevContext(context.obj.root) +@click.command(help="Configure and run Open edX from scratch, for development") +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.pass_context +def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) -> None: + try: + utils.check_macos_docker_memory() + except exceptions.TutorError as e: + fmt.echo_alert( + f"""Could not verify sufficient RAM allocation in Docker: + {e} +Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: + https://docs.tutor.overhang.io/install.html""" + ) + + click.echo(fmt.title("Interactive platform configuration")) + config = tutor_config.load_minimal(context.obj.root) + if not non_interactive: + interactive_config.ask_questions(config, run_for_prod=False) + tutor_config.save_config_file(context.obj.root, config) + config = tutor_config.load_full(context.obj.root) + tutor_env.save(context.obj.root, config) + + click.echo(fmt.title("Stopping any existing platform")) + context.invoke(compose.stop) + + if pullimages: + click.echo(fmt.title("Docker image updates")) + context.invoke(compose.dc_command, command="pull") + + click.echo(fmt.title("Building Docker image for LMS and CMS development")) + context.invoke(compose.dc_command, command="build", args=["lms"]) + + click.echo(fmt.title("Starting the platform in detached mode")) + context.invoke(compose.start, detach=True) + + click.echo(fmt.title("Database creation and migrations")) + context.invoke(compose.init) + + fmt.echo_info( + """The Open edX platform is now running in detached mode +Your Open edX platform is ready and can be accessed at the following urls: + {http}://{lms_host}:8000 + {http}://{cms_host}:8001 + """.format( + http="https" if config["ENABLE_HTTPS"] else "http", + lms_host=config["LMS_HOST"], + cms_host=config["CMS_HOST"], + ) + ) + + @click.command( help="Run a development server", context_settings={"ignore_unknown_options": True}, @@ -62,5 +116,6 @@ def runserver(context: click.Context, options: List[str], service: str) -> None: context.invoke(compose.run, args=args) +dev.add_command(quickstart) dev.add_command(runserver) compose.add_commands(dev) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 620f40c8f6..b4f42ce209 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -6,6 +6,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env +from tutor import interactive as interactive_config from tutor import exceptions, fmt, jobs, serialize, utils from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseJobContext @@ -171,10 +172,12 @@ def quickstart(context: click.Context, non_interactive: bool) -> None: ) click.echo(fmt.title("Interactive platform configuration")) - context.invoke( - config_save_command, - interactive=(not non_interactive), - ) + config = tutor_config.load_minimal(context.obj.root) + if not non_interactive: + interactive_config.ask_questions(config, run_for_prod=True) + tutor_config.save_config_file(context.obj.root, config) + config = tutor_config.load_full(context.obj.root) + tutor_env.save(context.obj.root, config) if run_upgrade_from_release and not non_interactive: question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. diff --git a/tutor/commands/local.py b/tutor/commands/local.py index cd4c668863..c9bd8fc414 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -5,6 +5,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env from tutor import exceptions, fmt, utils +from tutor import interactive as interactive_config from tutor.commands import compose from tutor.commands.config import save as config_save_command from tutor.commands.upgrade.local import upgrade_from @@ -83,7 +84,12 @@ def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) ) click.echo(fmt.title("Interactive platform configuration")) - context.invoke(config_save_command, interactive=(not non_interactive)) + config = tutor_config.load_minimal(context.obj.root) + if not non_interactive: + interactive_config.ask_questions(config) + tutor_config.save_config_file(context.obj.root, config) + config = tutor_config.load_full(context.obj.root) + tutor_env.save(context.obj.root, config) if run_upgrade_from_release and not non_interactive: question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. diff --git a/tutor/interactive.py b/tutor/interactive.py index d6d6fa50cc..0321b3f743 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional import click @@ -7,26 +7,26 @@ from .types import Config, get_typed -def load_user_config(root: str, interactive: bool = True) -> Config: +def ask_questions(config: Config, run_for_prod: Optional[bool] = None) -> None: """ - Load configuration and interactively ask questions to collect param values from the user. - """ - config = tutor_config.load_minimal(root) - if interactive: - ask_questions(config) - return config - + Interactively ask questions to collect configuration values from the user. -def ask_questions(config: Config) -> None: + Arguments: + config: Existing (or minimal) configuration. Modified in-place. + run_for_prod: Whether platform should be configured for production. + If None, then ask the user. + """ defaults = tutor_config.get_defaults() - run_for_prod = config.get("LMS_HOST") != "local.overhang.io" - run_for_prod = click.confirm( - fmt.question( - "Are you configuring a production platform? Type 'n' if you are just testing Tutor on your local computer" - ), - prompt_suffix=" ", - default=run_for_prod, - ) + if run_for_prod is None: + run_for_prod = config.get("LMS_HOST") != "local.overhang.io" + run_for_prod = click.confirm( + fmt.question( + "Are you configuring a production platform? " + "Type 'n' if you are just testing Tutor on your local computer" + ), + prompt_suffix=" ", + default=run_for_prod, + ) if not run_for_prod: dev_values: Config = { "LMS_HOST": "local.overhang.io",