diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index a10c1d74bda8d..84f9ecea169a2 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -638,75 +638,16 @@ def string_lower_type(val): default=False, ) -# webserver -ARG_PORT = Arg( - ("-p", "--port"), - default=conf.get("webserver", "WEB_SERVER_PORT"), - type=int, - help="The port on which to run the server", -) -ARG_SSL_CERT = Arg( - ("--ssl-cert",), - default=conf.get("webserver", "WEB_SERVER_SSL_CERT"), - help="Path to the SSL certificate for the webserver", -) -ARG_SSL_KEY = Arg( - ("--ssl-key",), - default=conf.get("webserver", "WEB_SERVER_SSL_KEY"), - help="Path to the key to use with the SSL certificate", -) -ARG_WORKERS = Arg( - ("-w", "--workers"), - default=conf.get("webserver", "WORKERS"), - type=int, - help="Number of workers to run the webserver on", -) -ARG_WORKERCLASS = Arg( - ("-k", "--workerclass"), - default=conf.get("webserver", "WORKER_CLASS"), - choices=["sync", "eventlet", "gevent", "tornado"], - help="The worker class to use for Gunicorn", -) -ARG_WORKER_TIMEOUT = Arg( - ("-t", "--worker-timeout"), - default=conf.get("webserver", "WEB_SERVER_WORKER_TIMEOUT"), - type=int, - help="The timeout for waiting on webserver workers", -) -ARG_HOSTNAME = Arg( - ("-H", "--hostname"), - default=conf.get("webserver", "WEB_SERVER_HOST"), - help="Set the hostname on which to run the web server", -) -ARG_DEBUG = Arg( - ("-d", "--debug"), help="Use the server that ships with Flask in debug mode", action="store_true" -) -ARG_ACCESS_LOGFILE = Arg( - ("-A", "--access-logfile"), - default=conf.get("webserver", "ACCESS_LOGFILE"), - help="The logfile to store the webserver access log. Use '-' to print to stdout", -) -ARG_ERROR_LOGFILE = Arg( - ("-E", "--error-logfile"), - default=conf.get("webserver", "ERROR_LOGFILE"), - help="The logfile to store the webserver error log. Use '-' to print to stderr", -) -ARG_ACCESS_LOGFORMAT = Arg( - ("-L", "--access-logformat"), - default=conf.get("webserver", "ACCESS_LOGFORMAT"), - help="The access log format for gunicorn logs", -) - # api-server ARG_API_SERVER_PORT = Arg( ("-p", "--port"), - default=9091, + default=conf.get("api", "port"), type=int, help="The port on which to run the API server", ) ARG_API_SERVER_WORKERS = Arg( ("-w", "--workers"), - default=4, + default=conf.get("api", "workers"), type=int, help="Number of workers to run on the API server", ) @@ -717,22 +658,15 @@ def string_lower_type(val): help="The timeout for waiting on API server workers", ) ARG_API_SERVER_HOSTNAME = Arg( - ("-H", "--hostname"), - default="0.0.0.0", # nosec - help="Set the hostname on which to run the API server", + ("-H", "--host"), + default=conf.get("api", "host"), + help="Set the host on which to run the API server", ) ARG_API_SERVER_ACCESS_LOGFILE = Arg( ("-A", "--access-logfile"), + default=conf.get("api", "access_logfile"), help="The logfile to store the access log. Use '-' to print to stdout", ) -ARG_API_SERVER_ERROR_LOGFILE = Arg( - ("-E", "--error-logfile"), - help="The logfile to store the error log. Use '-' to print to stderr", -) -ARG_API_SERVER_ACCESS_LOGFORMAT = Arg( - ("-L", "--access-logformat"), - help="The access log format for gunicorn logs", -) ARG_API_SERVER_APPS = Arg( ("--apps",), help="Applications to run (comma-separated). Default is all. Options: core, execution, all", @@ -743,7 +677,17 @@ def string_lower_type(val): help="Enable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info.", action="store_true", ) - +ARG_SSL_CERT = Arg( + ("--ssl-cert",), + default=conf.get("api", "ssl_cert"), + help="Path to the SSL certificate for the webserver", +) +ARG_SSL_KEY = Arg( + ("--ssl-key",), + default=conf.get("api", "ssl_key"), + help="Path to the key to use with the SSL certificate", +) +ARG_DEV = Arg(("-d", "--dev"), help="Start FastAPI in development mode", action="store_true") # scheduler ARG_NUM_RUNS = Arg( @@ -1871,13 +1815,11 @@ class GroupCommand(NamedTuple): ARG_STDOUT, ARG_STDERR, ARG_API_SERVER_ACCESS_LOGFILE, - ARG_API_SERVER_ERROR_LOGFILE, - ARG_API_SERVER_ACCESS_LOGFORMAT, ARG_API_SERVER_APPS, ARG_LOG_FILE, ARG_SSL_CERT, ARG_SSL_KEY, - ARG_DEBUG, + ARG_DEV, ARG_API_SERVER_ALLOW_PROXY_FORWARDING, ), ), diff --git a/airflow/cli/commands/local_commands/api_server_command.py b/airflow/cli/commands/local_commands/api_server_command.py index 530c340840d14..4f3fb5ab07e59 100644 --- a/airflow/cli/commands/local_commands/api_server_command.py +++ b/airflow/cli/commands/local_commands/api_server_command.py @@ -48,13 +48,12 @@ def api_server(args): apps = args.apps access_logfile = args.access_logfile or "-" - access_logformat = args.access_logformat num_workers = args.workers worker_timeout = args.worker_timeout proxy_headers = args.proxy_headers - if args.debug: - print(f"Starting the API server on port {args.port} and host {args.hostname} debug.") + if args.dev: + print(f"Starting the API server on port {args.port} and host {args.host} in development mode.") log.warning("Running in dev mode, ignoring uvicorn args") run_args = [ @@ -64,13 +63,13 @@ def api_server(args): "--port", str(args.port), "--host", - str(args.hostname), + str(args.host), ] if args.proxy_headers: run_args.append("--proxy-headers") - # There is no way to pass the apps to airflow/api_fastapi/main.py in the debug mode + # There is no way to pass the apps to airflow/api_fastapi/main.py in the development mode # because fastapi dev command does not accept any additional arguments # so environment variable is being used to pass it os.environ["AIRFLOW_API_APPS"] = apps @@ -91,18 +90,17 @@ def api_server(args): Running the uvicorn with: Apps: {apps} Workers: {num_workers} - Host: {args.hostname}:{args.port} + Host: {args.host}:{args.port} Timeout: {worker_timeout} Logfiles: {access_logfile} - Access Logformat: {access_logformat} =================================================================""" ) ) ssl_cert, ssl_key = _get_ssl_cert_and_key_filepaths(args) - setproctitle(f"airflow api_server -- host:{args.hostname} port:{args.port}") + setproctitle(f"airflow api_server -- host:{args.host} port:{args.port}") uvicorn.run( "airflow.api_fastapi.main:app", - host=args.hostname, + host=args.host, port=args.port, workers=num_workers, timeout_keep_alive=worker_timeout, diff --git a/airflow/cli/commands/remote_commands/config_command.py b/airflow/cli/commands/remote_commands/config_command.py index 7ab85c5e62105..09e2d0614dbad 100644 --- a/airflow/cli/commands/remote_commands/config_command.py +++ b/airflow/cli/commands/remote_commands/config_command.py @@ -292,6 +292,42 @@ def message(self) -> str: config=ConfigParameter("webserver", "force_log_out_after"), renamed_to=ConfigParameter("webserver", "session_lifetime_minutes"), ), + ConfigChange( + config=ConfigParameter("webserver", "web_server_host"), + renamed_to=ConfigParameter("api", "host"), + ), + ConfigChange( + config=ConfigParameter("webserver", "web_server_port"), + renamed_to=ConfigParameter("api", "port"), + ), + ConfigChange( + config=ConfigParameter("webserver", "workers"), + renamed_to=ConfigParameter("api", "workers"), + ), + ConfigChange( + config=ConfigParameter("webserver", "web_server_worker_timeout"), + renamed_to=ConfigParameter("api", "worker_timeout"), + ), + ConfigChange( + config=ConfigParameter("webserver", "web_server_ssl_cert"), + renamed_to=ConfigParameter("api", "ssl_cert"), + ), + ConfigChange( + config=ConfigParameter("webserver", "web_server_ssl_key"), + renamed_to=ConfigParameter("api", "ssl_key"), + ), + ConfigChange( + config=ConfigParameter("webserver", "access_logfile"), + renamed_to=ConfigParameter("api", "access_logfile"), + ), + ConfigChange( + config=ConfigParameter("webserver", "error_logfile"), + was_deprecated=False, + ), + ConfigChange( + config=ConfigParameter("webserver", "access_logformat"), + was_deprecated=False, + ), # policy ConfigChange( config=ConfigParameter("policy", "airflow_local_settings"), diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index e7907a6c8d961..48f142a2875dc 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1322,6 +1322,57 @@ api: type: string example: ~ default: "http://localhost:9091" + host: + description: | + The ip specified when starting the api server + version_added: ~ + type: string + example: ~ + default: "0.0.0.0" + port: + description: | + The port on which to run the api server + version_added: ~ + type: string + example: ~ + default: "9091" + workers: + description: | + Number of workers to run on the API server + version_added: ~ + type: string + example: ~ + default: "4" + worker_timeout: + description: | + Number of seconds the API server waits before timing out on a worker + version_added: ~ + type: string + example: ~ + default: "120" + access_logfile: + description: | + Log files for the api server. '-' means log to stderr. + version_added: ~ + type: string + example: ~ + default: "-" + ssl_cert: + description: | + Paths to the SSL certificate and key for the api server. When both are + provided SSL will be enabled. This does not change the api server port. + version_added: ~ + type: string + example: ~ + default: "" + ssl_key: + description: | + Paths to the SSL certificate and key for the api server. When both are + provided SSL will be enabled. This does not change the api server port. + version_added: ~ + type: string + example: ~ + default: "" auth_backends: description: | Comma separated list of auth backends to authenticate users of the API. See @@ -1534,36 +1585,6 @@ webserver: example: "America/New_York" # Default is left as UTC for now so the date's don't "suddenly" change on upgrade default: "UTC" - web_server_host: - description: | - The ip specified when starting the web server - version_added: ~ - type: string - example: ~ - default: "0.0.0.0" - web_server_port: - description: | - The port on which to run the web server - version_added: ~ - type: string - example: ~ - default: "8080" - web_server_ssl_cert: - description: | - Paths to the SSL certificate and key for the web server. When both are - provided SSL will be enabled. This does not change the web server port. - version_added: ~ - type: string - example: ~ - default: "" - web_server_ssl_key: - description: | - Paths to the SSL certificate and key for the web server. When both are - provided SSL will be enabled. This does not change the web server port. - version_added: ~ - type: string - example: ~ - default: "" session_backend: description: | The type of backend used to store web session data, can be ``database`` or ``securecookie``. For the @@ -1590,13 +1611,6 @@ webserver: type: string example: ~ default: "120" - web_server_worker_timeout: - description: | - Number of seconds the gunicorn webserver waits before timing out on a worker - version_added: ~ - type: string - example: ~ - default: "120" worker_refresh_batch_size: description: | Number of workers to refresh at a time through Gunicorn's built-in worker management. @@ -1636,13 +1650,6 @@ webserver: sensitive: true example: ~ default: "{SECRET_KEY}" - workers: - description: | - Number of workers to run the Gunicorn web server - version_added: ~ - type: string - example: ~ - default: "4" worker_class: description: | The worker class gunicorn should use. Choices include @@ -1666,30 +1673,6 @@ webserver: type: string example: ~ default: "sync" - access_logfile: - description: | - Log files for the gunicorn webserver. '-' means log to stderr. - version_added: ~ - type: string - example: ~ - default: "-" - error_logfile: - description: | - Log files for the gunicorn webserver. '-' means log to stderr. - version_added: ~ - type: string - example: ~ - default: "-" - access_logformat: - description: | - Access log format for gunicorn webserver. - default format is ``%%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s"`` - See `Gunicorn Settings: 'access_log_format' Reference - `__ for more details - version_added: 2.0.0 - type: string - example: ~ - default: "" expose_config: description: | Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values diff --git a/airflow/configuration.py b/airflow/configuration.py index 350aa5fd3ca1a..affda6fc476b8 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -326,6 +326,13 @@ def sensitive_config_values(self) -> set[tuple[str, str]]: # DeprecationWarning will be issued and the old option will be used instead deprecated_options: dict[tuple[str, str], tuple[str, str, str]] = { ("dag_processor", "refresh_interval"): ("scheduler", "dag_dir_list_interval", "3.0"), + ("api", "host"): ("webserver", "web_server_host", "3.0"), + ("api", "port"): ("webserver", "web_server_port", "3.0"), + ("api", "workers"): ("webserver", "workers", "3.0"), + ("api", "worker_timeout"): ("webserver", "web_server_worker_timeout", "3.0"), + ("api", "ssl_cert"): ("webserver", "web_server_ssl_cert", "3.0"), + ("api", "ssl_key"): ("webserver", "web_server_ssl_key", "3.0"), + ("api", "access_logfile"): ("webserver", "access_logfile", "3.0"), } # A mapping of new section -> (old section, since_version). diff --git a/newsfragments/47083.significant.rst b/newsfragments/47083.significant.rst new file mode 100644 index 0000000000000..6d8401935991b --- /dev/null +++ b/newsfragments/47083.significant.rst @@ -0,0 +1,45 @@ +``airflow api-server`` has replaced ``airflow webserver`` cli command + +The new Airflow UI is now being served as part of the ``airflow api-server`` command and the ``airflow webserver`` command has been removed. + +The following configuration options have moved to the ``[api]`` section: + +- ``[webserver] web_server_host`` has been moved to ``[api] host`` +- ``[webserver] web_server_port`` has been moved to ``[api] port`` +- ``[webserver] workers`` has been moved to ``[api] workers`` +- ``[webserver] web_server_worker_timeout`` has been moved to ``[api] worker_timeout`` +- ``[webserver] web_server_ssl_cert`` has been moved to ``[api] ssl_cert`` +- ``[webserver] web_server_ssl_key`` has been moved to ``[api] ssl_key`` +- ``[webserver] access_logfile`` has been moved to ``[api] access_logfile`` + +The following configuration options have been removed: + +- ``[webserver] error_logfile`` +- ``[webserver] access_logformat`` + +* Types of change + + * [ ] Dag changes + * [x] Config changes + * [ ] API changes + * [x] CLI changes + * [ ] Behaviour changes + * [ ] Plugin changes + * [ ] Dependency changes + * [ ] Code interface changes + +.. List the migration rules needed for this change (see https://github.com/apache/airflow/issues/41641) + +* Migration rules needed + + * ``airflow config lint`` + + * [x] ``[webserver] web_server_host`` → ``[api] host`` + * [x] ``[webserver] web_server_port`` → ``[api] port`` + * [x] ``[webserver] workers`` → ``[api] workers`` + * [x] ``[webserver] web_server_worker_timeout`` → ``[api] worker_timeout`` + * [x] ``[webserver] web_server_ssl_cert`` → ``[api] ssl_cert`` + * [x] ``[webserver] web_server_ssl_key`` → ``[api] ssl_key`` + * [x] ``[webserver] access_logfile`` → ``[api] access_logfile`` + * [x] ``[webserver] error_logfile`` removed + * [x] ``[webserver] access_logformat`` removed diff --git a/tests/cli/commands/local_commands/test_api_server_command.py b/tests/cli/commands/local_commands/test_api_server_command.py index 0693e2ef6aff7..f15a0b3b51e34 100644 --- a/tests/cli/commands/local_commands/test_api_server_command.py +++ b/tests/cli/commands/local_commands/test_api_server_command.py @@ -30,14 +30,14 @@ @pytest.mark.db_test -class TestCliFastAPI(_CommonCLIGunicornTestClass): +class TestCliApiServer(_CommonCLIGunicornTestClass): main_process_regexp = r"airflow api-server" @pytest.mark.parametrize( "args, expected_command", [ ( - ["api-server", "--port", "9092", "--hostname", "somehost", "--debug"], + ["api-server", "--port", "9092", "--host", "somehost", "--dev"], [ "fastapi", "dev", @@ -49,7 +49,7 @@ class TestCliFastAPI(_CommonCLIGunicornTestClass): ], ), ( - ["api-server", "--port", "9092", "--hostname", "somehost", "--debug", "--proxy-headers"], + ["api-server", "--port", "9092", "--host", "somehost", "--dev", "--proxy-headers"], [ "fastapi", "dev", @@ -63,7 +63,7 @@ class TestCliFastAPI(_CommonCLIGunicornTestClass): ), ], ) - def test_cli_fastapi_api_debug(self, args, expected_command): + def test_dev_arg(self, args, expected_command): with ( mock.patch("subprocess.Popen") as Popen, ): @@ -75,7 +75,7 @@ def test_cli_fastapi_api_debug(self, args, expected_command): close_fds=True, ) - def test_cli_fastapi_api_env_var_set_unset(self): + def test_apps_env_var_set_unset(self): """ Test that AIRFLOW_API_APPS is set and unset in the environment when calling the airflow api-server command @@ -86,11 +86,11 @@ def test_cli_fastapi_api_env_var_set_unset(self): ): apps_value = "core,execution" port = "9092" - hostname = "somehost" + host = "somehost" # Parse the command line arguments args = self.parser.parse_args( - ["api-server", "--port", port, "--hostname", hostname, "--apps", apps_value, "--debug"] + ["api-server", "--port", port, "--host", host, "--apps", apps_value, "--dev"] ) # Ensure AIRFLOW_API_APPS is not set initially @@ -111,7 +111,7 @@ def test_cli_fastapi_api_env_var_set_unset(self): "--port", port, "--host", - hostname, + host, ], close_fds=True, ) @@ -119,7 +119,7 @@ def test_cli_fastapi_api_env_var_set_unset(self): # Assert that AIRFLOW_API_APPS was unset after subprocess mock_environ.pop.assert_called_with("AIRFLOW_API_APPS") - def test_cli_fastapi_api_args(self, ssl_cert_and_key): + def test_args_to_uvicorn(self, ssl_cert_and_key): cert_path, key_path = ssl_cert_and_key with ( @@ -128,8 +128,6 @@ def test_cli_fastapi_api_args(self, ssl_cert_and_key): args = self.parser.parse_args( [ "api-server", - "--access-logformat", - "custom_log_format", "--pid", "/tmp/x.pid", "--ssl-cert",