diff --git a/.github/workflows/cluster_tests.yaml b/.github/workflows/cluster_tests.yaml index e97b14607..c2eb218ea 100644 --- a/.github/workflows/cluster_tests.yaml +++ b/.github/workflows/cluster_tests.yaml @@ -2,9 +2,6 @@ name: cluster-tests on: workflow_dispatch -env: - RH_LOG_LEVEL: "INFO" - jobs: cluster-tests: runs-on: ubuntu-latest diff --git a/.github/workflows/copy_docs.yaml b/.github/workflows/copy_docs.yaml index 14e714916..b260c94c6 100644 --- a/.github/workflows/copy_docs.yaml +++ b/.github/workflows/copy_docs.yaml @@ -6,9 +6,6 @@ on: branches: - '*' -env: - RH_LOG_LEVEL: "INFO" - jobs: generate-docs: runs-on: ubuntu-latest diff --git a/.github/workflows/generate_docs_for_tag.yaml b/.github/workflows/generate_docs_for_tag.yaml index b930f4f57..9ab69dcfe 100644 --- a/.github/workflows/generate_docs_for_tag.yaml +++ b/.github/workflows/generate_docs_for_tag.yaml @@ -11,9 +11,6 @@ on: required: false # Allow empty for cases where the release event provides the tag default: '' -env: - RH_LOG_LEVEL: "INFO" - jobs: build-docs-for-tag: runs-on: ubuntu-latest diff --git a/.github/workflows/local_den_unit_tests.yaml b/.github/workflows/local_den_unit_tests.yaml index e07bbc608..1c6d77c21 100644 --- a/.github/workflows/local_den_unit_tests.yaml +++ b/.github/workflows/local_den_unit_tests.yaml @@ -2,9 +2,6 @@ name: Local Den Unit Tests on: workflow_dispatch -env: - RH_LOG_LEVEL: "INFO" - jobs: local-den-tests: runs-on: ubuntu-latest diff --git a/.github/workflows/local_tests.yaml b/.github/workflows/local_tests.yaml index f7501d4a9..deb325fe1 100644 --- a/.github/workflows/local_tests.yaml +++ b/.github/workflows/local_tests.yaml @@ -7,7 +7,6 @@ on: env: API_SERVER_URL: https://api.run.house - RH_LOG_LEVEL: "INFO" jobs: # TODO: THESE ARE ONLY SEPARATE JOBS BECAUSE THERE ARE @@ -43,7 +42,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local tests/test_servers/ env: @@ -82,7 +80,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest and not clustertest" env: @@ -123,7 +120,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "secrettest" env: @@ -149,7 +145,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "moduletest" env: @@ -175,7 +170,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "functiontest" env: @@ -201,7 +195,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "envtest" env: @@ -227,7 +220,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "clustertest" env: diff --git a/.github/workflows/local_tests_den_dev.yaml b/.github/workflows/local_tests_den_dev.yaml index 3787f7e61..d328aa8dc 100644 --- a/.github/workflows/local_tests_den_dev.yaml +++ b/.github/workflows/local_tests_den_dev.yaml @@ -5,7 +5,6 @@ on: env: API_SERVER_URL: https://api-dev.run.house - RH_LOG_LEVEL: "INFO" jobs: # TODO: THESE ARE ONLY SEPARATE JOBS BECAUSE THERE ARE @@ -41,7 +40,7 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} + - name: pytest -v --level local tests/test_servers/ env: @@ -80,7 +79,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest" env: @@ -121,7 +119,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "secrettest" env: @@ -147,7 +144,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "moduletest" env: @@ -173,7 +169,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "functiontest" env: @@ -199,7 +194,6 @@ jobs: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} api_server_url: ${{ env.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} - name: pytest -v --level local -k "envtest" env: diff --git a/.github/workflows/nightly_release_testing.yaml b/.github/workflows/nightly_release_testing.yaml index cd85d8f8b..50fe41a60 100644 --- a/.github/workflows/nightly_release_testing.yaml +++ b/.github/workflows/nightly_release_testing.yaml @@ -6,9 +6,6 @@ on: # Run every night at 2 AM UTC - cron: '0 2 * * *' -env: - RH_LOG_LEVEL: "INFO" - jobs: not-cluster-tests: runs-on: ubuntu-latest diff --git a/.github/workflows/sagemaker_tests.yaml b/.github/workflows/sagemaker_tests.yaml index 0c781d91e..25c1c81ec 100644 --- a/.github/workflows/sagemaker_tests.yaml +++ b/.github/workflows/sagemaker_tests.yaml @@ -2,9 +2,6 @@ name: sagemaker-tests on: workflow_dispatch -env: - RH_LOG_LEVEL: "INFO" - jobs: sagemaker-tests: runs-on: ubuntu-latest diff --git a/.github/workflows/setup_release_testing/action.yaml b/.github/workflows/setup_release_testing/action.yaml index 81c3b855d..fada8c122 100644 --- a/.github/workflows/setup_release_testing/action.yaml +++ b/.github/workflows/setup_release_testing/action.yaml @@ -88,4 +88,3 @@ runs: username: ${{ inputs.CI_ACCOUNT_USERNAME }} token: ${{ inputs.CI_ACCOUNT_TOKEN }} api_server_url: ${{ inputs.API_SERVER_URL }} - rh_log_level: ${{ env.RH_LOG_LEVEL }} diff --git a/.github/workflows/setup_rh_config/action.yaml b/.github/workflows/setup_rh_config/action.yaml index 18bbe422f..a344a711b 100644 --- a/.github/workflows/setup_rh_config/action.yaml +++ b/.github/workflows/setup_rh_config/action.yaml @@ -15,10 +15,6 @@ inputs: description: 'The den api server to send the requests to' required: true - rh_log_level: - description: 'Cluster log level' - required: true - runs: using: composite steps: @@ -31,4 +27,3 @@ runs: echo "token: ${{ inputs.token }}" >> ~/.rh/config.yaml echo "username: ${{ inputs.username }}" >> ~/.rh/config.yaml echo "api_server_url: ${{ inputs.api_server_url }}" >> ~/.rh/config.yaml - echo "rh_log_level: ${{ inputs.rh_log_level }}" >> ~/.rh/config.yaml diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 32bdf8d97..576e85bb5 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -5,9 +5,6 @@ on: branches: [ main ] pull_request: -env: - RH_LOG_LEVEL: "INFO" - jobs: all-tests-logged-out-level-unit: runs-on: ubuntu-latest diff --git a/docs/debugging-logging.rst b/docs/debugging-logging.rst index f13b602b9..47fec8a4b 100644 --- a/docs/debugging-logging.rst +++ b/docs/debugging-logging.rst @@ -24,8 +24,7 @@ Alternatively, to see logs on your local machine while running a remote function Log Levels ---------- You can set the log level to control the verbosity of the Runhouse logs. You can adjust the log level by -setting the environment variable ``RH_LOG_LEVEL`` to your desired level, or by setting the `rh_log_level` field -in your local `~/.rh/config.yaml`. +setting the environment variable ``RH_LOG_LEVEL`` to your desired level. Debugging ~~~~~~~~~ diff --git a/examples/llama3-8b-ec2/llama3_ec2.py b/examples/llama3-8b-ec2/llama3_ec2.py index bdf6b6a29..963c57474 100644 --- a/examples/llama3-8b-ec2/llama3_ec2.py +++ b/examples/llama3-8b-ec2/llama3_ec2.py @@ -39,12 +39,12 @@ import torch # Next, we define a class that will hold the model and allow us to send prompts to it. -# You'll notice this class inherits from `rh.Module`. +# We'll later wrap this with `rh.module`. # This is a Runhouse class that allows you to # run code in your class on a remote machine. # # Learn more in the [Runhouse docs on functions and modules](/docs/tutorials/api-modules). -class HFChatModel(rh.Module): +class HFChatModel: def __init__(self, model_id="meta-llama/Meta-Llama-3-8B-Instruct", **model_kwargs): super().__init__() self.model_id, self.model_kwargs = model_id, model_kwargs @@ -129,15 +129,16 @@ def predict(self, prompt_text, **inf_kwargs): ) # Finally, we define our module and run it on the remote cluster. We construct it normally and then call - # `get_or_to` to run it on the remote cluster. Using `get_or_to` allows us to load the exiting Module - # by the name `llama3-8b-model` if it was already put on the cluster. If we want to update the module each - # time we run this script, we can use `to` instead of `get_or_to`. + # `to` to run it on the remote cluster. Alternatively, we could first check for an existing instance on the cluster + # by calling `cluster.get(name="llama3-8b-model")`. This would return the remote model after an initial run. + # If we want to update the module each time we run this script, we prefer to use `to`. # - # Note that we also pass the `env` object to the `get_or_to` method, which will ensure that the environment is + # Note that we also pass the `env` object to the `to` method, which will ensure that the environment is # set up on the remote machine before the module is run. - remote_hf_chat_model = HFChatModel( - torch_dtype=torch.bfloat16, - ).get_or_to(gpu, env=env, name="llama3-8b-model") + RemoteChatModel = rh.module(HFChatModel).to(gpu, env=env, name="HFChatModel") + remote_hf_chat_model = RemoteChatModel( + torch_dtype=torch.bfloat16, name="llama3-8b-model" + ) # ## Calling our remote function # diff --git a/examples/llama3-fine-tuning-lora/llama3_fine_tuning.py b/examples/llama3-fine-tuning-lora/llama3_fine_tuning.py index 3907cf0fe..2106a0659 100644 --- a/examples/llama3-fine-tuning-lora/llama3_fine_tuning.py +++ b/examples/llama3-fine-tuning-lora/llama3_fine_tuning.py @@ -32,7 +32,7 @@ import runhouse as rh # Next, we define a class that will hold the various methods needed to fine-tune the model. -# You'll notice this class inherits from `rh.Module`. This is a Runhouse class that allows you to +# We'll later wrap this with `rh.module`. This is a Runhouse class that allows you to # run code in your class on a remote machine. # # Learn more in the [Runhouse docs on functions and modules](/docs/tutorials/api-modules). @@ -40,7 +40,7 @@ DEFAULT_MAX_LENGTH = 200 -class FineTuner(rh.Module): +class FineTuner: def __init__( self, dataset_name="Shekswess/medical_llama3_instruct_dataset_short", @@ -268,15 +268,14 @@ def generate(self, query: str, max_length: int = DEFAULT_MAX_LENGTH): ) # Finally, we define our module and run it on the remote cluster. We construct it normally and then call - # `get_or_to` to run it on the remote cluster. Using `get_or_to` allows us to load the exiting Module - # by the name `llama3-medical-model` if it was already put on the cluster. If we want to update the Module each - # time we run this script, we can use `to` instead of `get_or_to`. + # `to` to run it on the remote cluster. Alternatively, we could first check for an existing instance on the cluster + # by calling `cluster.get(name="llama3-medical-model")`. This would return the remote model after an initial run. + # If we want to update the module each time we run this script, we prefer to use `to`. # - # Note that we also pass the `env` object to the `get_or_to` method, which will ensure that the environment is + # Note that we also pass the `env` object to the `to` method, which will ensure that the environment is # set up on the remote machine before the module is run. - fine_tuner_remote = FineTuner().get_or_to( - cluster, env=env, name="llama3-medical-model" - ) + RemoteFineTuner = rh.module(FineTuner).to(cluster, env=env, name="FineTuner") + fine_tuner_remote = RemoteFineTuner(name="llama3-medical-model") # ## Fine-tune the model on the cluster # diff --git a/runhouse/logger.py b/runhouse/logger.py index 720304b36..4dfb11913 100644 --- a/runhouse/logger.py +++ b/runhouse/logger.py @@ -8,7 +8,6 @@ def get_logger(name: str = __name__): level = os.getenv("RH_LOG_LEVEL") if level: - # Set the logging level logger.setLevel(level.upper()) # Check if the logger already has handlers, add a StreamHandler if not diff --git a/runhouse/main.py b/runhouse/main.py index fe0e8f20f..25f62e265 100644 --- a/runhouse/main.py +++ b/runhouse/main.py @@ -538,7 +538,6 @@ def _start_server( default_env_name=None, conda_env=None, from_python=None, - log_level=None, ): ############################################ # Build CLI commands to start the server @@ -627,7 +626,6 @@ def _start_server( flags.append(conda_env_flag) flags.append(" --from-python" if from_python else "") - flags.append(f" --log-level {log_level}" if log_level else "") # Check if screen or nohup are available screen = screen and _check_if_command_exists("screen") @@ -745,10 +743,6 @@ def start( conda_env: str = typer.Option( None, help="Name of conda env corresponding to default env if it is a CondaEnv." ), - log_level: Optional[str] = typer.Option( - default=None, - help="Log level for the server", - ), ): """Start the HTTP or HTTPS server on the cluster.""" _start_server( @@ -766,7 +760,6 @@ def start( certs_address=certs_address, default_env_name=default_env_name, conda_env=conda_env, - log_level=log_level, ) @@ -833,10 +826,6 @@ def restart( False, help="Whether HTTP server started from inside a Python call rather than CLI.", ), - log_level: Optional[str] = typer.Option( - default=None, - help="Log level for the server", - ), ): """Restart the HTTP server on the cluster.""" if name: @@ -864,7 +853,6 @@ def restart( default_env_name=default_env_name, conda_env=conda_env, from_python=from_python, - log_level=log_level, ) diff --git a/runhouse/resources/hardware/cluster.py b/runhouse/resources/hardware/cluster.py index 0b8f0cdfa..e9dac12ff 100644 --- a/runhouse/resources/hardware/cluster.py +++ b/runhouse/resources/hardware/cluster.py @@ -957,7 +957,6 @@ def restart_server( + (f" --domain {domain}" if domain else "") + f" --port {self.server_port}" + f" --api-server-url {rns_client.api_server_url}" - + f" --log-level {rns_client.log_level}" + f" --default-env-name {self.default_env.name}" + ( f" --conda-env {self.default_env.env_name}" diff --git a/runhouse/resources/packages/package.py b/runhouse/resources/packages/package.py index 99cdd4114..b20058c99 100644 --- a/runhouse/resources/packages/package.py +++ b/runhouse/resources/packages/package.py @@ -68,10 +68,11 @@ class Package(Resource): def __init__( self, - name: str = None, - install_method: str = None, - install_target: Union[str, "Folder"] = None, - install_args: str = None, + name: Optional[str] = None, + install_method: Optional[str] = None, + install_target: Optional[Union[str, "Folder"]] = None, + install_args: Optional[str] = None, + preferred_version: Optional[str] = None, dryrun: bool = False, **kwargs, # We have this here to ignore extra arguments when calling from from_config ): @@ -88,6 +89,7 @@ def __init__( self.install_method = install_method self.install_target = install_target self.install_args = install_args + self.preferred_version = preferred_version def config(self, condensed=True): # If the package is just a simple Package.from_string string, no @@ -105,6 +107,7 @@ def config(self, condensed=True): else self.install_target ) config["install_args"] = self.install_args + config["preferred_version"] = self.preferred_version return config def __str__(self): @@ -251,6 +254,25 @@ def _install(self, env: Union[str, "Env"] = None, cluster: "Cluster" = None): return if self.install_method == "pip": + + # If this is a generic pip package, with no version pinned, we want to check if there is a version + # already installed. If there is, then we ignore preferred version and leave the existing version. + # The user can always force a version install by doing `numpy==2.0.0` for example. Else, we install + # the preferred version, that matches their local. + if ( + is_python_package_string(self.install_target) + and self.preferred_version is not None + ): + # Check if this is installed + retcode = run_setup_command( + f"python -c \"import importlib.util; exit(0) if importlib.util.find_spec('{self.install_target}') else exit(1)\"", + cluster=cluster, + )[0] + if retcode != 0: + self.install_target = ( + f"{self.install_target}=={self.preferred_version}" + ) + install_cmd = self._pip_install_cmd(env=env, cluster=cluster) logger.info(f"Running via install_method pip: {install_cmd}") retcode = run_setup_command(install_cmd, cluster=cluster)[0] @@ -495,6 +517,7 @@ def from_string(specifier: str, dryrun=False): # If we are just defaulting to pip, attempt to install the same version of the package # that is already installed locally # Check if the target is only letters, nothing else. This means its a string like 'numpy'. + preferred_version = None if install_method == "pip" and is_python_package_string(target): locally_installed_version = find_locally_installed_version(target) if locally_installed_version: @@ -503,6 +526,10 @@ def from_string(specifier: str, dryrun=False): if local_install_path and Path(local_install_path).exists(): target = (local_install_path, None) + else: + # We want to preferrably install this version of the package server-side + preferred_version = locally_installed_version + # "Local" install method is a special case where we just copy a local folder and add to path if install_method == "local": return Package( @@ -514,6 +541,7 @@ def from_string(specifier: str, dryrun=False): install_target=target, install_args=args, install_method=install_method, + preferred_version=preferred_version, dryrun=dryrun, ) elif install_method == "rh": diff --git a/runhouse/resources/provenance.py b/runhouse/resources/provenance.py index e33f3514f..669bc80a9 100644 --- a/runhouse/resources/provenance.py +++ b/runhouse/resources/provenance.py @@ -64,7 +64,6 @@ def __init__( run_type: RunType = RunType.CMD_RUN, error: Optional[str] = None, error_traceback: Optional[str] = None, - log_level: Optional[str] = None, overwrite: bool = False, dryrun: bool = False, **kwargs, @@ -113,7 +112,6 @@ def __init__( self.downstream_artifacts = downstream_artifacts or [] self.fn_name = fn_name self.cmds = cmds - self.log_level = log_level or "INFO" self.run_type = run_type or self._detect_run_type() self.error = error self.traceback = error_traceback @@ -135,7 +133,6 @@ def __enter__(self): # Add the stdout and stderr handlers to the root logger self._stdout_handler = logging.StreamHandler(sys.stdout) - self._stdout_handler.setLevel(self.log_level) logger.addHandler(self._stdout_handler) return self diff --git a/runhouse/rns/rns_client.py b/runhouse/rns/rns_client.py index b566fa64a..5a7dd1d14 100644 --- a/runhouse/rns/rns_client.py +++ b/runhouse/rns/rns_client.py @@ -106,12 +106,6 @@ def api_server_url(self): return url_as_env_var return self._configs.get("api_server_url", None) - @property - def log_level(self): - log_level = os.getenv("RH_LOG_LEVEL", self._configs.get("rh_log_level", None)) - if log_level: - return log_level.upper() - def _index_base_folders(self, lst): self.rns_base_folders = {} for folder in lst: diff --git a/runhouse/rns/top_level_rns_fns.py b/runhouse/rns/top_level_rns_fns.py index eede3e708..8d92891ed 100644 --- a/runhouse/rns/top_level_rns_fns.py +++ b/runhouse/rns/top_level_rns_fns.py @@ -92,7 +92,6 @@ async def get_local_cluster_object(): await obj_store.ainitialize( servlet_name=servlet_name, setup_cluster_servlet=ClusterServletSetupOption.GET_OR_FAIL, - log_level=rns_client.log_level, ) except ConnectionError: return "file" diff --git a/runhouse/servers/cluster_servlet.py b/runhouse/servers/cluster_servlet.py index 71ab69572..e9594d909 100644 --- a/runhouse/servers/cluster_servlet.py +++ b/runhouse/servers/cluster_servlet.py @@ -52,10 +52,6 @@ async def __init__( self._auth_cache: AuthCache = AuthCache(cluster_config) self.autostop_helper = None - log_level = kwargs.get("log_level") - if log_level: - logger.setLevel(log_level) - if cluster_config.get("resource_subtype", None) == "OnDemandCluster": self.autostop_helper = AutostopHelper() diff --git a/runhouse/servers/env_servlet.py b/runhouse/servers/env_servlet.py index 9e9a643b5..0ededb46e 100644 --- a/runhouse/servers/env_servlet.py +++ b/runhouse/servers/env_servlet.py @@ -69,15 +69,10 @@ class EnvServlet: async def __init__(self, env_name: str, *args, **kwargs): self.env_name = env_name - log_level = kwargs.get("log_level") - if log_level: - logger.setLevel(log_level) - await obj_store.ainitialize( self.env_name, has_local_storage=True, setup_cluster_servlet=ClusterServletSetupOption.GET_OR_FAIL, - log_level=log_level, ) # Ray defaults to setting OMP_NUM_THREADS to 1, which unexpectedly limit parallelism in user programs. diff --git a/runhouse/servers/http/http_server.py b/runhouse/servers/http/http_server.py index da8fb0569..100405c59 100644 --- a/runhouse/servers/http/http_server.py +++ b/runhouse/servers/http/http_server.py @@ -64,7 +64,7 @@ ObjStoreError, RaySetupOption, ) -from runhouse.utils import sync_function +from runhouse.utils import generate_default_name, sync_function app = FastAPI(docs_url=None, redoc_url=None) @@ -136,13 +136,9 @@ async def ainitialize( default_env_name=None, conda_env=None, from_test: bool = False, - log_level: str = None, *args, **kwargs, ): - if log_level: - logger.setLevel(log_level) - runtime_env = {"conda": conda_env} if conda_env else None default_env_name = default_env_name or EMPTY_DEFAULT_ENV_NAME @@ -156,7 +152,6 @@ async def ainitialize( default_env_name, setup_ray=RaySetupOption.TEST_PROCESS, runtime_env=runtime_env, - log_level=log_level, ) # TODO disabling due to latency, figure out what to do with this @@ -171,7 +166,6 @@ async def ainitialize( env_name=default_env_name, create=True, runtime_env=runtime_env, - log_level=log_level, ) if default_env_name == EMPTY_DEFAULT_ENV_NAME: @@ -422,8 +416,18 @@ async def get_call( # Default argument to json doesn't allow a user to pass in a serialization string if they want # But, if they didn't pass anything, we want it to be `json` by default. serialization = serialization or "json" - try: + if run_name is None and stream_logs: + raise ValueError( + "run_name is required for all calls when stream_logs is True." + ) + + if run_name is None: + run_name = generate_default_name( + prefix=key if method_name == "__call__" else f"{key}_{method_name}", + precision="ms", # Higher precision because we see collisions within the same second + sep="@", + ) # The types need to be explicitly specified as parameters first so that # we can cast Query params to the right type. @@ -950,13 +954,6 @@ async def main(): help="Whether HTTP server is called from Python rather than CLI.", ) - parser.add_argument( - "--log-level", - type=str, - default=None, - help="Log level for the server", - ) - parse_args = parser.parse_args() conda_name = parse_args.conda_env @@ -979,11 +976,9 @@ async def main(): # We only want to forcibly start a Ray cluster if asked. # We connect this to the "base" env, which we'll initialize later, # so writes to the obj_store within the server get proxied to the "base" env. - log_level = parse_args.log_level await obj_store.ainitialize( default_env_name, setup_cluster_servlet=ClusterServletSetupOption.FORCE_CREATE, - log_level=log_level, ) cluster_config = await obj_store.aget_cluster_config() @@ -1152,7 +1147,6 @@ async def main(): await HTTPServer.ainitialize( default_env_name=default_env_name, conda_env=conda_name, - log_level=log_level, ) if den_auth: diff --git a/runhouse/servers/obj_store.py b/runhouse/servers/obj_store.py index c53fde57a..51b38ead3 100644 --- a/runhouse/servers/obj_store.py +++ b/runhouse/servers/obj_store.py @@ -175,9 +175,6 @@ async def ainitialize( if self.servlet_name is not None: return - if log_level: - logger.setLevel(log_level) - from runhouse.resources.hardware.ray_utils import kill_actors # Only if ray is not initialized do we attempt a setup process. @@ -1097,7 +1094,6 @@ async def acall_local( run_name: Optional[str] = None, stream_logs: bool = False, remote: bool = False, - log_level: str = None, **kwargs, ): """Base call functionality: Load the module, and call a method on it with args and kwargs. Nothing else. @@ -1110,7 +1106,7 @@ async def acall_local( log_ctx = None if stream_logs and run_name is not None: - log_ctx = LogToFolder(name=run_name, log_level=log_level) + log_ctx = LogToFolder(name=run_name) log_ctx.__enter__() # Use a finally to track the active functions so that it is always removed @@ -1135,7 +1131,6 @@ async def acall_local( run_name=run_name, stream_logs=stream_logs, remote=remote, - log_level=log_level, **kwargs, ) finally: @@ -1153,15 +1148,11 @@ async def _acall_local_helper( run_name: Optional[str] = None, stream_logs: bool = False, remote: bool = False, - log_level: str = None, **kwargs, ): """acall_local primarily sets up the logging and tracking for the function call, then calls _acall_local_helper to actually do the work. This is so we can have a finally block in acall_local to clean up the active function calls tracking.""" - if log_level: - logger.setLevel(log_level) - obj = self.get_local(key, default=KeyError) from runhouse.resources.module import Module @@ -1348,7 +1339,6 @@ async def acall( run_name: Optional[str] = None, stream_logs: bool = False, remote: bool = False, - log_level: str = None, ): env_servlet_name_containing_key = await self.aget_env_servlet_name_for_key(key) if not env_servlet_name_containing_key: @@ -1373,7 +1363,6 @@ async def acall( run_name=run_name, stream_logs=stream_logs, remote=remote, - log_level=log_level, *args, **kwargs, ) diff --git a/runhouse/utils.py b/runhouse/utils.py index 074f23899..e698d52fe 100644 --- a/runhouse/utils.py +++ b/runhouse/utils.py @@ -376,11 +376,10 @@ def __getattr__(self, item): class LogToFolder: - def __init__(self, name: str, log_level: str = None): + def __init__(self, name: str): self.name = name self.directory = self._base_local_folder_path(name) self.root_logger = logging.getLogger("") - self.log_level = log_level if log_level else "INFO" # We do exist_ok=True here because generator runs are separate calls to the same directory. os.makedirs(self.directory, exist_ok=True) @@ -391,10 +390,7 @@ def __enter__(self): # Add the stdout and stderr handlers to the root logger self._stdout_handler = logging.StreamHandler(sys.stdout) - self._stdout_handler.setLevel(self.log_level) - self.root_logger.addHandler(self._stdout_handler) - self.root_logger.setLevel(self.log_level) return self diff --git a/tests/fixtures/docker_cluster_fixtures.py b/tests/fixtures/docker_cluster_fixtures.py index 7f70c85d6..15a1d4f08 100644 --- a/tests/fixtures/docker_cluster_fixtures.py +++ b/tests/fixtures/docker_cluster_fixtures.py @@ -1,5 +1,6 @@ import importlib import logging +import os import shlex import subprocess import time @@ -298,8 +299,6 @@ def docker_cluster_pk_tls_exposed(request, test_rns_folder): - Public key authentication - Caddy set up on startup to forward Runhouse HTTP server to port 443 """ - import os - # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") @@ -355,8 +354,6 @@ def docker_cluster_pk_ssh(request, test_org_rns_folder): - Nginx set up on startup to forward Runhouse HTTP server to port 443 - Default env with Ray 2.30.0 """ - import os - # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") @@ -379,6 +376,7 @@ def docker_cluster_pk_ssh(request, test_org_rns_folder): ], working_dir=None, name="default_env", + env_vars={"RH_LOG_LEVEL": os.getenv("RH_LOG_LEVEL", "INFO")}, ) local_cluster, cleanup = set_up_local_cluster( @@ -459,8 +457,6 @@ def docker_cluster_pk_http_exposed(request, test_rns_folder): - Caddy set up on startup to forward Runhouse HTTP Server to port 80 - Default conda_env with Python 3.11 and Ray 2.30.0 """ - import os - # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") @@ -483,7 +479,10 @@ def docker_cluster_pk_http_exposed(request, test_rns_folder): "numpy<=1.26.4", ], conda_env={"dependencies": ["python=3.11"], "name": "default_env"}, - env_vars={"OMP_NUM_THREADS": "8"}, + env_vars={ + "OMP_NUM_THREADS": "8", + "RH_LOG_LEVEL": os.getenv("RH_LOG_LEVEL", "INFO"), + }, working_dir=None, name="default_env", ) @@ -530,8 +529,6 @@ def docker_cluster_pwd_ssh_no_auth(request, test_rns_folder): - No Den Auth - No caddy/port forwarding set up """ - import os - # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") @@ -579,8 +576,6 @@ def friend_account_logged_in_docker_cluster_pk_ssh(request, test_rns_folder): This fixture is not parameterized for every test; it is a separate cluster started with a test account (username: kitchen_tester) in order to test sharing resources with other users. """ - import os - # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") diff --git a/tests/fixtures/on_demand_cluster_fixtures.py b/tests/fixtures/on_demand_cluster_fixtures.py index 1187cb397..dbdf17d76 100644 --- a/tests/fixtures/on_demand_cluster_fixtures.py +++ b/tests/fixtures/on_demand_cluster_fixtures.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import pytest @@ -85,7 +86,11 @@ def ondemand_gcp_cluster(request): """ Note: Also used to test conda default env. """ - env_vars = {"var1": "val1", "var2": "val2"} + env_vars = { + "var1": "val1", + "var2": "val2", + "RH_LOG_LEVEL": os.getenv("RH_LOG_LEVEL", "INFO"), + } default_env = rh.conda_env( name="default_env", reqs=test_env().reqs + ["ray==2.30.0"], diff --git a/tests/test_resources/test_data/test_package.py b/tests/test_resources/test_data/test_package.py index 2639adb54..030425a58 100644 --- a/tests/test_resources/test_data/test_package.py +++ b/tests/test_resources/test_data/test_package.py @@ -9,10 +9,10 @@ from runhouse.utils import run_with_logs -def get_plotly_version(): - import plotly +def get_bs4_version(): + import bs4 - return plotly.__version__ + return bs4.__version__ class TestPackage(tests.test_resources.test_resource.TestResource): @@ -152,13 +152,12 @@ def test_local_reqs_on_cluster(self, cluster, local_package): assert isinstance(remote_package.install_target, InstallTarget) @pytest.mark.level("local") - @pytest.mark.skip("Feature deprecated for now") def test_local_package_version_gets_installed(self, cluster): - run_with_logs("pip install plotly==5.9.0") - env = rh.env(name="temp_env", reqs=["plotly"]) + run_with_logs("pip install beautifulsoup4==4.11.1") + env = rh.env(name="temp_env", reqs=["beautifulsoup4"]) - remote_fn = rh.function(get_plotly_version, env=env).to(cluster) - assert remote_fn() == "5.9.0" + remote_fn = rh.function(get_bs4_version, env=env).to(cluster) + assert remote_fn() == "4.11.1" # --------- basic torch index-url testing --------- @pytest.mark.level("unit")