ADS changes for MS enhancements

ads/aqua/app.py

@@ -22,10 +22,9 @@
     is_valid_ocid,
     load_config,
 )
-from ads.aqua.constants import UNKNOWN
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
-from ads.common.utils import extract_region, is_path_exists
+from ads.common.utils import UNKNOWN, extract_region, is_path_exists
 from ads.config import (
     AQUA_TELEMETRY_BUCKET,
     AQUA_TELEMETRY_BUCKET_NS,

ads/aqua/common/utils.py

@@ -19,7 +19,6 @@
 from string import Template
 from typing import List, Union
 
-import fsspec
 import oci
 from cachetools import TTLCache, cached
 from huggingface_hub.constants import HF_HUB_CACHE
@@ -58,7 +57,6 @@
     SUPPORTED_FILE_FORMATS,
     TEI_CONTAINER_DEFAULT_HOST,
     TGI_INFERENCE_RESTRICTED_PARAMS,
-    UNKNOWN,
     UNKNOWN_JSON_STR,
     VLLM_INFERENCE_RESTRICTED_PARAMS,
 )
@@ -68,7 +66,13 @@
 from ads.common.extended_enum import ExtendedEnum
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.oci_resource import SEARCH_TYPE, OCIResource
-from ads.common.utils import copy_file, get_console_link, upload_to_os
+from ads.common.utils import (
+    UNKNOWN,
+    copy_file,
+    get_console_link,
+    read_file,
+    upload_to_os,
+)
 from ads.config import (
     AQUA_MODEL_DEPLOYMENT_FOLDER,
     AQUA_SERVICE_MODELS_BUCKET,
@@ -228,15 +232,6 @@ def get_artifact_path(custom_metadata_list: List) -> str:
     return UNKNOWN
 
 
-def read_file(file_path: str, **kwargs) -> str:
-    try:
-        with fsspec.open(file_path, "r", **kwargs.get("auth", {})) as f:
-            return f.read()
-    except Exception as e:
-        logger.debug(f"Failed to read file {file_path}. {e}")
-        return UNKNOWN
-
-
 @threaded()
 def load_config(file_path: str, config_file_name: str, **kwargs) -> dict:
     artifact_path = f"{file_path.rstrip('/')}/{config_file_name}"

ads/aqua/constants.py

@@ -3,7 +3,6 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 """This module defines constants used in ads.aqua module."""
 
-UNKNOWN = ""
 UNKNOWN_VALUE = ""
 READY_TO_IMPORT_STATUS = "TRUE"
 UNKNOWN_DICT = {}

ads/aqua/evaluation/evaluation.py

@@ -55,7 +55,6 @@
     JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING,
     LIFECYCLE_DETAILS_MISSING_JOBRUN,
     NB_SESSION_IDENTIFIER,
-    UNKNOWN,
 )
 from ads.aqua.evaluation.constants import (
     EVAL_TERMINATION_STATE,
@@ -79,7 +78,7 @@
 from ads.aqua.evaluation.errors import EVALUATION_JOB_EXIT_CODE_MESSAGE
 from ads.common.auth import default_signer
 from ads.common.object_storage_details import ObjectStorageDetails
-from ads.common.utils import get_console_link, get_files, get_log_links
+from ads.common.utils import UNKNOWN, get_console_link, get_files, get_log_links
 from ads.config import (
     AQUA_JOB_SUBNET_ID,
     COMPARTMENT_OCID,

ads/aqua/finetuning/finetuning.py

@@ -30,7 +30,6 @@
     DEFAULT_FT_REPLICA,
     DEFAULT_FT_VALIDATION_SET_SIZE,
     JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING,
-    UNKNOWN,
     UNKNOWN_DICT,
 )
 from ads.aqua.data import AquaResourceIdentifier
@@ -45,7 +44,7 @@
 )
 from ads.common.auth import default_signer
 from ads.common.object_storage_details import ObjectStorageDetails
-from ads.common.utils import get_console_link
+from ads.common.utils import UNKNOWN, get_console_link
 from ads.config import (
     AQUA_FINETUNING_CONTAINER_OVERRIDE_FLAG_METADATA_NAME,
     AQUA_JOB_SUBNET_ID,

ads/aqua/model/model.py

@@ -59,7 +59,6 @@
     READY_TO_IMPORT_STATUS,
     TRAINING_METRICS_FINAL,
     TRINING_METRICS,
-    UNKNOWN,
     VALIDATION_METRICS,
     VALIDATION_METRICS_FINAL,
 )
@@ -80,7 +79,7 @@
 )
 from ads.common.auth import default_signer
 from ads.common.oci_resource import SEARCH_TYPE, OCIResource
-from ads.common.utils import get_console_link
+from ads.common.utils import UNKNOWN, get_console_link
 from ads.config import (
     AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
     AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,

ads/aqua/modeldeployment/deployment.py

@@ -27,15 +27,14 @@
     AQUA_MODEL_TYPE_CUSTOM,
     AQUA_MODEL_TYPE_SERVICE,
     MODEL_BY_REFERENCE_OSS_PATH_KEY,
-    UNKNOWN,
     UNKNOWN_DICT,
 )
 from ads.aqua.data import AquaResourceIdentifier
 from ads.aqua.finetuning.finetuning import FineTuneCustomMetadata
 from ads.aqua.model import AquaModelApp
 from ads.aqua.modeldeployment.entities import AquaDeployment, AquaDeploymentDetail
 from ads.common.object_storage_details import ObjectStorageDetails
-from ads.common.utils import get_log_links
+from ads.common.utils import UNKNOWN, get_log_links
 from ads.config import (
     AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
     AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,

ads/aqua/modeldeployment/entities.py

@@ -11,10 +11,10 @@
 )
 
 from ads.aqua.common.enums import Tags
-from ads.aqua.constants import UNKNOWN, UNKNOWN_DICT
+from ads.aqua.constants import UNKNOWN_DICT
 from ads.aqua.data import AquaResourceIdentifier
 from ads.common.serializer import DataClassSerializable
-from ads.common.utils import get_console_link
+from ads.common.utils import UNKNOWN, get_console_link
 
 
 @dataclass

ads/common/utils.py

@@ -22,7 +22,7 @@
 from enum import Enum
 from io import DEFAULT_BUFFER_SIZE
 from textwrap import fill
-from typing import Dict, Optional, Tuple, Union
+from typing import Any, Dict, Optional, Tuple, Union
 from urllib import request
 from urllib.parse import urlparse
 
@@ -63,6 +63,8 @@
 # Maximum distinct values by cardinality will be used for plotting
 MAX_DISPLAY_VALUES = 10
 
+UNKNOWN = ""
+
 # par link of the index json file.
 PAR_LINK = "https://objectstorage.us-ashburn-1.oraclecloud.com/p/WyjtfVIG0uda-P3-2FmAfwaLlXYQZbvPZmfX1qg0-sbkwEQO6jpwabGr2hMDBmBp/n/ociodscdev/b/service-conda-packs/o/service_pack/index.json"
 
@@ -82,6 +84,7 @@
     color=["teal", "blueviolet", "forestgreen", "peru", "y", "dodgerblue", "r"]
 )
 
+
 # sqlalchemy engines
 _engines = {}
 
@@ -149,6 +152,22 @@ def oci_key_location():
     )
 
 
+def text_sanitizer(content):
+    if isinstance(content, str):
+        return (
+            content.replace("“", '"')
+            .replace("”", '"')
+            .replace("’", "'")
+            .replace("‘", "'")
+            .replace("—", "-")
+            .encode("utf-8", "ignore")
+            .decode("utf-8", "ignore")
+        )
+    if isinstance(content, dict):
+        return json.dumps(content)
+    return str(content)
+
+
 @deprecated(
     "2.5.10",
     details="Deprecated, use: from ads.common.auth import AuthState; AuthState().oci_config_path",
@@ -212,6 +231,37 @@ def random_valid_ocid(prefix="ocid1.dataflowapplication.oc1.iad"):
     return f"{left}.{fake}"
 
 
+def parse_bool(value: Any) -> bool:
+    """
+    Converts a value to boolean. For strings, it interprets 'true', '1', or 'yes'
+    (case insensitive) as True; everything else as False.
+
+    Parameters
+    ----------
+    value : Any
+        The value to convert to boolean.
+
+    Returns
+    -------
+    bool
+        The boolean interpretation of the value.
+    """
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in ("true", "1", "yes")
+    return bool(value)
+
+
+def read_file(file_path: str, **kwargs) -> str:
+    try:
+        with fsspec.open(file_path, "r", **kwargs.get("auth", {})) as f:
+            return f.read()
+    except Exception as e:
+        logger.debug(f"Failed to read file {file_path}. {e}")
+        return UNKNOWN
+
+
 def get_dataframe_styles(max_width=75):
     """Styles used for dataframe, example usage:
 

ads/config.py

@@ -80,6 +80,9 @@
 DEBUG_TELEMETRY = os.environ.get("DEBUG_TELEMETRY", None)
 AQUA_SERVICE_NAME = "aqua"
 DATA_SCIENCE_SERVICE_NAME = "data-science"
+USER = "USER"
+SERVICE = "SERVICE"
+
 
 
 THREADED_DEFAULT_TIMEOUT = 5

ads/model/common/utils.py

@@ -1,22 +1,34 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
 # Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json
 import os
 import tempfile
-import yaml
 from typing import Any, Dict, Optional
 from zipfile import ZipFile
-from ads.common import utils
 
+import yaml
+
+from ads.common import utils
+from ads.common.extended_enum import ExtendedEnum
 
 DEPRECATE_AS_ONNX_WARNING = "This attribute `as_onnx` will be deprecated in the future. You can choose specific format by setting `model_save_serializer`."
 DEPRECATE_USE_TORCH_SCRIPT_WARNING = "This attribute `use_torch_script` will be deprecated in the future. You can choose specific format by setting `model_save_serializer`."
 
 
+class MetadataArtifactPathType(ExtendedEnum):
+    """
+    Enum for defining metadata artifact path type.
+    Can be either local path or OSS path. It can also be the content itself.
+    """
+
+    LOCAL = "local"
+    OSS = "oss"
+    CONTENT = "content"
+
+
 def _extract_locals(
     locals: Dict[str, Any], filter_out_nulls: Optional[bool] = True
 ) -> Dict[str, Any]: