diff --git a/src/marqo/api/models/health_response.py b/src/marqo/api/models/health_response.py new file mode 100644 index 000000000..ee493f94b --- /dev/null +++ b/src/marqo/api/models/health_response.py @@ -0,0 +1,33 @@ +from typing import Optional + +from marqo.base_model import StrictBaseModel + + +class InferenceHealthResponse(StrictBaseModel): + status: str + + +class BackendHealthResponse(StrictBaseModel): + status: str + memoryIsAvailable: Optional[bool] + storageIsAvailable: Optional[bool] + + +class HealthResponse(StrictBaseModel): + status: str + inference: InferenceHealthResponse + backend: BackendHealthResponse + + @classmethod + def from_marqo_health_status(cls, marqo_health_status): + return cls( + status=marqo_health_status.status.value, + inference=InferenceHealthResponse( + status=marqo_health_status.inference.status.value + ), + backend=BackendHealthResponse( + status=marqo_health_status.backend.status.value, + memoryIsAvailable=marqo_health_status.backend.memory_is_available, + storageIsAvailable=marqo_health_status.backend.storage_is_available + ) + ) diff --git a/src/marqo/base_model.py b/src/marqo/base_model.py new file mode 100644 index 000000000..07819f9cf --- /dev/null +++ b/src/marqo/base_model.py @@ -0,0 +1,22 @@ +from pydantic import BaseModel + + +class MarqoBaseModel(BaseModel): + class Config: + allow_population_by_field_name = True # accept both real name and alias (if present) + validate_assignment = True + + +class StrictBaseModel(MarqoBaseModel): + class Config(MarqoBaseModel.Config): + extra = "forbid" + + +class ImmutableBaseModel(MarqoBaseModel): + class Config(MarqoBaseModel.Config): + allow_mutation = False + + +class ImmutableStrictBaseModel(StrictBaseModel, ImmutableBaseModel): + class Config(StrictBaseModel.Config, ImmutableBaseModel.Config): + pass diff --git a/src/marqo/core/index_management/index_management.py b/src/marqo/core/index_management/index_management.py index 91b166871..83e9ae4d3 100644 --- a/src/marqo/core/index_management/index_management.py +++ b/src/marqo/core/index_management/index_management.py @@ -256,7 +256,7 @@ def _remove_schema(self, app: str, name: str) -> None: def _add_schema_to_services(self, app: str, name: str) -> None: services_path = os.path.join(app, 'services.xml') - tree = ET.parse(services_path) # Replace 'path_to_file.xml' with the path to your XML file + tree = ET.parse(services_path) root = tree.getroot() documents_section = root.find(".//documents") @@ -273,6 +273,8 @@ def _remove_schema_from_services(self, app: str, name: str) -> None: tree = ET.parse(services_path) root = tree.getroot() + # TODO - Verify there is only one documents section (one content cluster) + # Error out otherwise as we don't know which one to use documents_section = root.find(".//documents") deleted = False diff --git a/src/marqo/core/models/marqo_index.py b/src/marqo/core/models/marqo_index.py index a1ece636d..d55644287 100644 --- a/src/marqo/core/models/marqo_index.py +++ b/src/marqo/core/models/marqo_index.py @@ -1,7 +1,7 @@ import re from abc import ABC, abstractmethod from enum import Enum -from typing import List, Optional, Dict, Any, Set, Type, Union +from typing import List, Optional, Dict, Any, Set, Union import pydantic from pydantic import PrivateAttr, root_validator @@ -10,7 +10,7 @@ from pydantic.utils import ROOT_KEY from marqo.core import constants -from marqo.core.models.strict_base_model import ImmutableStrictBaseModel, StrictBaseModel +from marqo.base_model import ImmutableStrictBaseModel, StrictBaseModel from marqo.exceptions import InvalidArgumentError from marqo.logging import get_logger from marqo.s2_inference import s2_inference @@ -95,18 +95,18 @@ class TensorField(ImmutableStrictBaseModel): class HnswConfig(ImmutableStrictBaseModel): - ef_construction: int = pydantic.Field(gt=0) + ef_construction: int = pydantic.Field(gt=0, alias='efConstruction') m: int = pydantic.Field(gt=0) class TextPreProcessing(ImmutableStrictBaseModel): - split_length: int = pydantic.Field(gt=0) - split_overlap: int = pydantic.Field(ge=0) - split_method: TextSplitMethod + split_length: int = pydantic.Field(gt=0, alias='splitLength') + split_overlap: int = pydantic.Field(ge=0, alias='splitOverlap') + split_method: TextSplitMethod = pydantic.Field(alias='splitMethod') class ImagePreProcessing(ImmutableStrictBaseModel): - patch_method: Optional[PatchMethod] + patch_method: Optional[PatchMethod] = pydantic.Field(alias='patchMethod') class Model(StrictBaseModel): diff --git a/src/marqo/core/models/marqo_index_health.py b/src/marqo/core/models/marqo_index_health.py index 21c51747a..641564b1b 100644 --- a/src/marqo/core/models/marqo_index_health.py +++ b/src/marqo/core/models/marqo_index_health.py @@ -1,7 +1,8 @@ from enum import Enum from types import DynamicClassAttribute +from typing import Optional -from marqo.core.models.strict_base_model import StrictBaseModel +from marqo.base_model import StrictBaseModel class HealthStatus(Enum): @@ -27,10 +28,17 @@ def __eq__(self, other): return self.priority == other.priority +class InferenceHealthStatus(StrictBaseModel): + status: HealthStatus + + class VespaHealthStatus(StrictBaseModel): status: HealthStatus + memory_is_available: Optional[bool] + storage_is_available: Optional[bool] class MarqoHealthStatus(StrictBaseModel): status: HealthStatus + inference: InferenceHealthStatus backend: VespaHealthStatus diff --git a/src/marqo/core/models/marqo_index_request.py b/src/marqo/core/models/marqo_index_request.py index 96e32953e..e8b32796a 100644 --- a/src/marqo/core/models/marqo_index_request.py +++ b/src/marqo/core/models/marqo_index_request.py @@ -1,10 +1,11 @@ from abc import ABC from typing import List, Dict, Optional +import pydantic from pydantic import root_validator, validator import marqo.core.models.marqo_index as marqo_index -from marqo.core.models.strict_base_model import StrictBaseModel, ImmutableStrictBaseModel +from marqo.base_model import StrictBaseModel, ImmutableStrictBaseModel class MarqoIndexRequest(ImmutableStrictBaseModel, ABC): @@ -42,7 +43,7 @@ class FieldRequest(StrictBaseModel): name: str type: marqo_index.FieldType features: List[marqo_index.FieldFeature] = [] - dependent_fields: Optional[Dict[str, float]] + dependent_fields: Optional[Dict[str, float]] = pydantic.Field(alias='dependentFields') @root_validator def check_all_fields(cls, values): diff --git a/src/marqo/core/models/marqo_index_stats.py b/src/marqo/core/models/marqo_index_stats.py index 2c014e966..024d40798 100644 --- a/src/marqo/core/models/marqo_index_stats.py +++ b/src/marqo/core/models/marqo_index_stats.py @@ -1,6 +1,14 @@ -from marqo.core.models.strict_base_model import StrictBaseModel +from typing import Optional + +from marqo.base_model import StrictBaseModel + + +class VespaStats(StrictBaseModel): + memory_used_percentage: Optional[float] + storage_used_percentage: Optional[float] class MarqoIndexStats(StrictBaseModel): number_of_documents: int number_of_vectors: int + backend: VespaStats diff --git a/src/marqo/core/models/marqo_query.py b/src/marqo/core/models/marqo_query.py index bb91141da..a96592592 100644 --- a/src/marqo/core/models/marqo_query.py +++ b/src/marqo/core/models/marqo_query.py @@ -4,7 +4,7 @@ from pydantic import validator -from marqo.core.models.strict_base_model import StrictBaseModel +from marqo.base_model import StrictBaseModel from marqo.core.search.search_filter import SearchFilter, MarqoFilterStringParser diff --git a/src/marqo/core/models/strict_base_model.py b/src/marqo/core/models/strict_base_model.py deleted file mode 100644 index afeea15e2..000000000 --- a/src/marqo/core/models/strict_base_model.py +++ /dev/null @@ -1,17 +0,0 @@ -from pydantic import BaseModel, Field - - -class StrictBaseModel(BaseModel): - class Config: - extra: str = "forbid" - validate_assignment: bool = True - - -class ImmutableBaseModel(BaseModel): - class Config: - allow_mutation: bool = False - - -class ImmutableStrictBaseModel(StrictBaseModel, ImmutableBaseModel): - class Config(StrictBaseModel.Config, ImmutableBaseModel.Config): - pass diff --git a/src/marqo/core/monitoring/monitoring.py b/src/marqo/core/monitoring/monitoring.py index aea0a94e2..b7eb2e243 100644 --- a/src/marqo/core/monitoring/monitoring.py +++ b/src/marqo/core/monitoring/monitoring.py @@ -1,11 +1,11 @@ from typing import Optional import marqo.logging -from marqo.core.exceptions import IndexNotFoundError from marqo.core.index_management.index_management import IndexManagement from marqo.core.models import MarqoIndex -from marqo.core.models.marqo_index_health import MarqoHealthStatus, HealthStatus, VespaHealthStatus -from marqo.core.models.marqo_index_stats import MarqoIndexStats +from marqo.core.models.marqo_index_health import MarqoHealthStatus, HealthStatus, VespaHealthStatus, \ + InferenceHealthStatus +from marqo.core.models.marqo_index_stats import MarqoIndexStats, VespaStats from marqo.core.vespa_index import for_marqo_index as vespa_index_factory from marqo.exceptions import InternalError from marqo.vespa.exceptions import VespaError @@ -49,9 +49,24 @@ def get_index_stats(self, marqo_index: MarqoIndex) -> MarqoIndexStats: except (TypeError, AttributeError, IndexError) as e: raise InternalError(f"Failed to get the number of vectors for index {marqo_index.name}: {e}") from e + metrics = self.vespa_client.get_metrics() + + memory_utilization = metrics.clusterController_resourceUsage_maxMemoryUtilization_max + disk_utilization = metrics.clusterController_resourceUsage_maxDiskUtilization_max + + # Occasionally Vespa returns empty metrics, often for the first call after a restart + if memory_utilization is None: + logger.warn(f'Vespa did not return a value for memory utilization metrics') + if disk_utilization is None: + logger.warn(f'Vespa did not return a value for disk utilization metrics') + return MarqoIndexStats( number_of_documents=doc_count_query_result.total_count, - number_of_vectors=number_of_vectors + number_of_vectors=number_of_vectors, + backend=VespaStats( + memory_used_percentage=memory_utilization * 100 if memory_utilization is not None else None, + storage_used_percentage=disk_utilization * 100 if disk_utilization is not None else None + ) ) def get_index_stats_by_name(self, index_name: str) -> MarqoIndexStats: @@ -80,34 +95,56 @@ def get_health(self, index_name: Optional[str] = None, hostname_filter: Optional Marqo index health status """ # TODO - Check index specific metrics such as memory and disk usage - marqo_status = self._get_marqo_health() - try: - vespa_status = self._get_vespa_health(hostname_filter=hostname_filter) - except VespaError as e: - logger.warning(f"Failed to get Vespa health: {e}") - vespa_status = HealthStatus.Red + inference_status = self._get_inference_health() + vespa_status = self._get_vespa_health(hostname_filter=hostname_filter) - aggregated_status = max(marqo_status, vespa_status) + aggregated_status = max(inference_status.status, vespa_status.status) return MarqoHealthStatus( status=aggregated_status, - backend=VespaHealthStatus(status=vespa_status) + inference=inference_status, + backend=vespa_status ) - def _get_marqo_health(self) -> HealthStatus: - return HealthStatus.Green + def _get_inference_health(self) -> InferenceHealthStatus: + return InferenceHealthStatus(status=HealthStatus.Green) - def _get_vespa_health(self, hostname_filter: Optional[str]) -> HealthStatus: - metrics = self.vespa_client.get_metrics() + def _get_vespa_health(self, hostname_filter: Optional[str]) -> VespaHealthStatus: + try: + metrics = self.vespa_client.get_metrics() + except VespaError as e: + logger.error(f"Failed to get Vespa metrics: {e}") + return VespaHealthStatus(status=HealthStatus.Red) - status = HealthStatus.Green + # Check service status + service_status = HealthStatus.Green for node in metrics.nodes: + if service_status == HealthStatus.Red: + break + if hostname_filter is not None and hostname_filter not in node.hostname: continue for service in node.services: if service.status.code != 'up': - status = HealthStatus.Red - break - - return status + service_status = HealthStatus.Red + + # Check feed block + feed_status = HealthStatus.Green + nodes_above_limit = metrics.clusterController_resourceUsage_nodesAboveLimit_max + memory_utilization = metrics.clusterController_resourceUsage_maxMemoryUtilization_max + disk_utilization = metrics.clusterController_resourceUsage_maxDiskUtilization_max + + if nodes_above_limit is None: + logger.warn(f'Vespa did not return a value for nodes_above_limit metric') + feed_status = HealthStatus.Yellow + elif nodes_above_limit > 0: + feed_status = HealthStatus.Yellow + + status = max(service_status, feed_status) + + return VespaHealthStatus( + status=status, + memory_is_available=memory_utilization is not None and memory_utilization < 1.0, + storage_is_available=disk_utilization is not None and disk_utilization < 1.0 + ) diff --git a/src/marqo/tensor_search/api.py b/src/marqo/tensor_search/api.py index 6d77f8e7e..5518d9899 100644 --- a/src/marqo/tensor_search/api.py +++ b/src/marqo/tensor_search/api.py @@ -10,6 +10,7 @@ from marqo import config, errors from marqo import version +from marqo.api.models.health_response import HealthResponse from marqo.core.exceptions import IndexExistsError, IndexNotFoundError from marqo.core.index_management.index_management import IndexManagement from marqo.errors import InvalidArgError, MarqoWebError, MarqoError @@ -192,13 +193,14 @@ def get_documents_by_ids( @app.get("/indexes/{index_name}/stats") def get_index_stats(index_name: str, marqo_config: config.Config = Depends(get_config)): stats = marqo_config.monitoring.get_index_stats_by_name(index_name) - return JSONResponse( - content={ - 'numberOfDocuments': stats.number_of_documents, - 'numberOfVectors': stats.number_of_vectors - }, - status_code=200 - ) + return { + 'numberOfDocuments': stats.number_of_documents, + 'numberOfVectors': stats.number_of_vectors, + 'backend': { + 'memoryUsedPercentage': stats.backend.memory_used_percentage, + 'storageUsedPercentage': stats.backend.storage_used_percentage + } + } @app.delete("/indexes/{index_name}") @@ -218,12 +220,16 @@ def delete_docs(index_name: str, documentIds: List[str], @app.get("/health") def check_health(marqo_config: config.Config = Depends(get_config)): - return marqo_config.monitoring.get_health() + health_status = marqo_config.monitoring.get_health() + + return HealthResponse.from_marqo_health_status(health_status) @app.get("/indexes/{index_name}/health") def check_index_health(index_name: str, marqo_config: config.Config = Depends(get_config)): - return marqo_config.monitoring.get_health(index_name=index_name) + health_status = marqo_config.monitoring.get_health(index_name=index_name) + + return HealthResponse.from_marqo_health_status(health_status) @app.get("/indexes") diff --git a/src/marqo/tensor_search/index_meta_cache.py b/src/marqo/tensor_search/index_meta_cache.py index 9d7e5d34e..8015c26c4 100644 --- a/src/marqo/tensor_search/index_meta_cache.py +++ b/src/marqo/tensor_search/index_meta_cache.py @@ -13,6 +13,7 @@ from marqo.core.index_management.index_management import IndexManagement from marqo.core.models import MarqoIndex from marqo.tensor_search.tensor_search_logging import get_logger +from marqo.vespa.exceptions import VespaError, VespaStatusError logger = get_logger(__name__) @@ -100,15 +101,32 @@ def refresh(): if time.time() - cache_refresh_last_logged_time > cache_refresh_log_interval: cache_refresh_last_logged_time = time.time() logger.info(f'Last index cache refresh at {cache_refresh_last_logged_time}') - - time.sleep(cache_refresh_interval) + except VespaError as e: + if isinstance(e, VespaStatusError) and e.status_code == 400: + # This can happen when settings schema doesn't exist + logger.warn( + 'Failed to populate index cache due to 400 error from Vespa. This is expected if you ' + f'have not created an index yet. Error: {e}' + ) + else: + logger.warn( + "Failed to connect to Vespa Document API. If you are using an external Vespa instance, " + "ensure that the VESPA_DOCUMENT_URL environment variable is set and your Vespa " + f"instance is running and healthy. Error: {e}" + ) except Exception as e: - logger.error(f'Error in index cache refresh thread: {e}') + logger.error(f'Unexpected error in index cache refresh thread: {e}') + + time.sleep(cache_refresh_interval) refresh_thread = threading.Thread(target=refresh, daemon=True) refresh_thread.start() +def start_refresh_thread(config: Config): + _check_refresh_thread(config) + + def populate_cache(config: Config): """ Refresh cache for all indexes diff --git a/src/marqo/tensor_search/models/index_settings.py b/src/marqo/tensor_search/models/index_settings.py index b48fbca96..528d07c2e 100644 --- a/src/marqo/tensor_search/models/index_settings.py +++ b/src/marqo/tensor_search/models/index_settings.py @@ -4,37 +4,37 @@ import marqo.core.models.marqo_index as core import marqo.errors as errors from marqo import version +from marqo.base_model import StrictBaseModel from marqo.core.models.marqo_index_request import FieldRequest, MarqoIndexRequest, StructuredMarqoIndexRequest, \ UnstructuredMarqoIndexRequest -from marqo.tensor_search.models.api_models import BaseMarqoModel -class AnnParameters(BaseMarqoModel): - space_type: core.DistanceMetric +class AnnParameters(StrictBaseModel): + spaceType: core.DistanceMetric parameters: core.HnswConfig -class IndexSettings(BaseMarqoModel): +class IndexSettings(StrictBaseModel): type: core.IndexType = core.IndexType.Unstructured - all_fields: Optional[List[FieldRequest]] - tensor_fields: Optional[List[str]] - treat_urls_and_pointers_as_images: Optional[bool] + allFields: Optional[List[FieldRequest]] + tensorFields: Optional[List[str]] + treatUrlsAndPointersAsImages: Optional[bool] model: str = 'hf/all_datasets_v4_MiniLM-L6' - model_properties: Optional[Dict[str, Any]] - normalize_embeddings: bool = True - text_preprocessing: core.TextPreProcessing = core.TextPreProcessing( - split_length=2, - split_overlap=0, - split_method=core.TextSplitMethod.Sentence + modelProperties: Optional[Dict[str, Any]] + normalizeEmbeddings: bool = True + textPreprocessing: core.TextPreProcessing = core.TextPreProcessing( + splitLength=2, + splitOverlap=0, + splitMethod=core.TextSplitMethod.Sentence ) - image_preprocessing: core.ImagePreProcessing = core.ImagePreProcessing( - patch_method=None + imagePreprocessing: core.ImagePreProcessing = core.ImagePreProcessing( + patchMethod=None ) - vector_numeric_type: core.VectorNumericType = core.VectorNumericType.Float - ann_parameters: AnnParameters = AnnParameters( - space_type=core.DistanceMetric.Angular, + vectorNumericType: core.VectorNumericType = core.VectorNumericType.Float + annParameters: AnnParameters = AnnParameters( + spaceType=core.DistanceMetric.Angular, parameters=core.HnswConfig( - ef_construction=128, + efConstruction=128, m=16 ) ) @@ -42,69 +42,69 @@ class IndexSettings(BaseMarqoModel): def to_marqo_index_request(self, index_name: str) -> MarqoIndexRequest: marqo_fields = None if self.type == core.IndexType.Structured: - if self.treat_urls_and_pointers_as_images is not None: + if self.treatUrlsAndPointersAsImages is not None: raise errors.InvalidArgError( "treat_urls_and_pointers_as_images is not a valid parameter for structured indexes" ) - if self.all_fields is not None: + if self.allFields is not None: marqo_fields = [ FieldRequest( name=field.name, type=field.type, features=field.features, dependent_fields=field.dependent_fields - ) for field in self.all_fields + ) for field in self.allFields ] return StructuredMarqoIndexRequest( name=index_name, model=core.Model( name=self.model, - properties=self.model_properties, - custom=self.model_properties is not None + properties=self.modelProperties, + custom=self.modelProperties is not None ), - normalize_embeddings=self.normalize_embeddings, - text_preprocessing=self.text_preprocessing, - image_preprocessing=self.image_preprocessing, - distance_metric=self.ann_parameters.space_type, - vector_numeric_type=self.vector_numeric_type, - hnsw_config=self.ann_parameters.parameters, + normalize_embeddings=self.normalizeEmbeddings, + text_preprocessing=self.textPreprocessing, + image_preprocessing=self.imagePreprocessing, + distance_metric=self.annParameters.spaceType, + vector_numeric_type=self.vectorNumericType, + hnsw_config=self.annParameters.parameters, fields=marqo_fields, - tensor_fields=self.tensor_fields, + tensor_fields=self.tensorFields, marqo_version=version.get_version(), created_at=time.time(), updated_at=time.time() ) elif self.type == core.IndexType.Unstructured: - if self.all_fields is not None: + if self.allFields is not None: raise errors.InvalidArgError( "all_fields is not a valid parameter for unstructured indexes" ) - if self.tensor_fields is not None: + if self.tensorFields is not None: raise errors.InvalidArgError( "tensor_fields is not a valid parameter for unstructured indexes" ) - if self.treat_urls_and_pointers_as_images is None: + if self.treatUrlsAndPointersAsImages is None: # Default value for treat_urls_and_pointers_as_images is False, but we can't set it in the model # as it is not a valid parameter for structured indexes - self.treat_urls_and_pointers_as_images = False + self.treatUrlsAndPointersAsImages = False return UnstructuredMarqoIndexRequest( name=index_name, model=core.Model( name=self.model, - properties=self.model_properties, - custom=self.model_properties is not None + properties=self.modelProperties, + custom=self.modelProperties is not None ), - normalize_embeddings=self.normalize_embeddings, - text_preprocessing=self.text_preprocessing, - image_preprocessing=self.image_preprocessing, - distance_metric=self.ann_parameters.space_type, - vector_numeric_type=self.vector_numeric_type, - hnsw_config=self.ann_parameters.parameters, - treat_urls_and_pointers_as_images=self.treat_urls_and_pointers_as_images, + normalize_embeddings=self.normalizeEmbeddings, + text_preprocessing=self.textPreprocessing, + image_preprocessing=self.imagePreprocessing, + distance_metric=self.annParameters.spaceType, + vector_numeric_type=self.vectorNumericType, + hnsw_config=self.annParameters.parameters, + treat_urls_and_pointers_as_images=self.treatUrlsAndPointersAsImages, marqo_version=version.get_version(), created_at=time.time(), updated_at=time.time() diff --git a/src/marqo/tensor_search/on_start_script.py b/src/marqo/tensor_search/on_start_script.py index 80b19bd01..c0ae734f0 100644 --- a/src/marqo/tensor_search/on_start_script.py +++ b/src/marqo/tensor_search/on_start_script.py @@ -12,7 +12,6 @@ from marqo.tensor_search import index_meta_cache, utils from marqo.tensor_search.enums import EnvVars from marqo.tensor_search.tensor_search_logging import get_logger -from marqo.vespa.exceptions import VespaError, VespaStatusError logger = get_logger(__name__) @@ -41,24 +40,7 @@ def __init__(self, config: config.Config): self.config = config def run(self): - try: - index_meta_cache.populate_cache(self.config) - except VespaError as e: - if isinstance(e, VespaStatusError) and e.status_code == 400: - # This can happen when settings schema doesn't exist - logger.warn('Failed to populate index cache due to 400 error from Vespa. This is expected if you have ' - 'not created an index yet. Error: {e}') - else: - logger.warn("Failed to connect to Vespa Document API. If you are using an external Vespa instance, " - "ensure that the VESPA_DOCUMENT_URL environment variable is set and your Vespa instance " - f"is running and healthy. Error: {e}") - # the following lines turns off auto create index - # connection = HttpRequests(c) - # connection.put( - # path="_cluster/settings", - # body={ - # "persistent": {"action.auto_create_index": "false"} - # }) + index_meta_cache.start_refresh_thread(self.config) class CUDAAvailable: diff --git a/src/marqo/tensor_search/tensor_search.py b/src/marqo/tensor_search/tensor_search.py index 18f39b4a1..d1069a763 100644 --- a/src/marqo/tensor_search/tensor_search.py +++ b/src/marqo/tensor_search/tensor_search.py @@ -1395,7 +1395,7 @@ def vectorise_multimodal_combination_field_structured( image_field_names = [] image_content_to_vectorise = [] - normalize_embeddings = marqo_index.normalize_embeddings + normalize_embeddings = marqo_index.normalizeEmbeddings image_fields = [field.name for field in marqo_index.field_map_by_type[FieldType.ImagePointer]] for sub_field_name, sub_content in multimodal_object.items(): diff --git a/src/marqo/vespa/models/application_metrics.py b/src/marqo/vespa/models/application_metrics.py index 44104c799..f45bf777a 100644 --- a/src/marqo/vespa/models/application_metrics.py +++ b/src/marqo/vespa/models/application_metrics.py @@ -1,4 +1,5 @@ -from typing import List, Any, Dict +from enum import Enum +from typing import List, Dict, Optional, Union, Any from pydantic import BaseModel @@ -8,11 +9,16 @@ class Status(BaseModel): description: str +class MetricSet(BaseModel): + dimensions: Dict[str, str] + values: Dict[str, Any] + + class Service(BaseModel): name: str timestamp: int status: Status - metrics: List[Dict[str, Any]] + metrics: List[MetricSet] class Node(BaseModel): @@ -21,5 +27,89 @@ class Node(BaseModel): services: List[Service] +class Aggregation(Enum): + Max = 'max' + Min = 'min' + Average = 'average' + Sum = 'sum' + Count = 'count' + Last = 'last' + + +# noinspection PyPep8Naming class ApplicationMetrics(BaseModel): nodes: List[Node] + + _SERVICE_CLUSTERCONTROLLER = 'vespa.container-clustercontroller' + + @property + def clusterController_resourceUsage_maxMemoryUtilization_max(self) -> Optional[Union[int, float]]: + return self._aggregate_metric( + metric_name='cluster-controller.resource_usage.max_memory_utilization.max', + aggregation=Aggregation.Max, + service_name=self._SERVICE_CLUSTERCONTROLLER + ) + + @property + def clusterController_resourceUsage_maxDiskUtilization_max(self) -> Optional[Union[int, float]]: + return self._aggregate_metric( + metric_name='cluster-controller.resource_usage.max_disk_utilization.max', + aggregation=Aggregation.Max, + service_name=self._SERVICE_CLUSTERCONTROLLER + ) + + @property + def clusterController_resourceUsage_nodesAboveLimit_max(self) -> Optional[Union[int, float]]: + return self._aggregate_metric( + metric_name='cluster-controller.resource_usage.nodes_above_limit.max', + aggregation=Aggregation.Max, + service_name=self._SERVICE_CLUSTERCONTROLLER + ) + + def _aggregate_metric( + self, + metric_name: str, + aggregation, + service_name: Optional[str] = None + ) -> Optional[Union[int, float]]: + """ + Aggregate a metric across all nodes and services. + + Args: + metric_name: Name of metric to aggregate + aggregation: Aggregation to use + service_name: Optional name of service to aggregate metric for. Provioding this will speed up the + aggregation process + + Returns: + Aggregated metric value. If no values are found and aggregation type is not Aggregation.Count, + None is returned + """ + values = [] + for node in self.nodes: + for service in node.services: + if service_name is not None and service.name != service_name: + continue + + for metric_set in service.metrics: + if metric_name in metric_set.values: + values.append(metric_set.values[metric_name]) + + if aggregation == Aggregation.Count: + return len(values) + + if len(values) == 0: + return None + + if aggregation == Aggregation.Max: + return max(values) + elif aggregation == Aggregation.Min: + return min(values) + elif aggregation == Aggregation.Average: + return sum(values) / len(values) + elif aggregation == Aggregation.Sum: + return sum(values) + elif aggregation == Aggregation.Last: + return values[-1] + else: + raise ValueError(f"Unknown aggregation {aggregation}") diff --git a/tests/core/monitoring/test_monitoring.py b/tests/core/monitoring/test_monitoring.py index 336894c94..da55d9c9e 100644 --- a/tests/core/monitoring/test_monitoring.py +++ b/tests/core/monitoring/test_monitoring.py @@ -1,7 +1,7 @@ from marqo.core.exceptions import IndexNotFoundError from marqo.core.models.marqo_index import FieldType, FieldFeature, TextPreProcessing, TextSplitMethod from marqo.core.models.marqo_index_request import FieldRequest -from marqo.core.models.marqo_index_stats import MarqoIndexStats +from marqo.core.models.marqo_index_stats import MarqoIndexStats, VespaStats from marqo.tensor_search import tensor_search from marqo.tensor_search.models.add_docs_objects import AddDocsParams from tests.marqo_test import MarqoTestCase @@ -85,8 +85,12 @@ def test_get_index_stats_emptyIndex_successful(self): """ for marqo_index in self.indexes_to_test: with self.subTest(f'{marqo_index.name} - {marqo_index.type.value}'): - self.assertEqual( - MarqoIndexStats(number_of_documents=0, number_of_vectors=0), + self.assertIndexStatsEqual( + MarqoIndexStats( + number_of_documents=0, + number_of_vectors=0, + backend=VespaStats() + ), self.monitoring.get_index_stats(marqo_index) ) @@ -103,8 +107,12 @@ def test_get_index_stats_docsWithTensorFields_successful(self): device="cpu" ) ) - self.assertEqual( - MarqoIndexStats(number_of_documents=3, number_of_vectors=3), + self.assertIndexStatsEqual( + MarqoIndexStats( + number_of_documents=3, + number_of_vectors=3, + backend=VespaStats() + ), self.monitoring.get_index_stats(marqo_index) ) @@ -127,8 +135,12 @@ def test_get_index_stats_structuredMultimodalIndex_successful(self): device="cpu" ) ) - self.assertEqual( - MarqoIndexStats(number_of_documents=3, number_of_vectors=4), + self.assertIndexStatsEqual( + MarqoIndexStats( + number_of_documents=3, + number_of_vectors=4, + backend=VespaStats() + ), self.monitoring.get_index_stats(marqo_index) ) @@ -145,8 +157,12 @@ def test_get_index_stats_docsWithoutTensorFields_successful(self): device="cpu" ) ) - self.assertEqual( - MarqoIndexStats(number_of_documents=3, number_of_vectors=0), + self.assertIndexStatsEqual( + MarqoIndexStats( + number_of_documents=3, + number_of_vectors=0, + backend=VespaStats() + ), self.monitoring.get_index_stats(marqo_index) ) @@ -163,8 +179,12 @@ def test_get_index_stats_mixedDocs_successful(self): device="cpu" ) ) - self.assertEqual( - MarqoIndexStats(number_of_documents=3, number_of_vectors=2), + self.assertIndexStatsEqual( + MarqoIndexStats( + number_of_documents=3, + number_of_vectors=2, + backend=VespaStats() + ), self.monitoring.get_index_stats(marqo_index) ) @@ -176,27 +196,27 @@ def test_get_index_stats_sequentialIndexingAndDeletion_successful(self): ( 'add', # add 3 docs, 3 have a vector [{"_id": "1", "title": "2"}, {"_id": "2", "title": "2"}, {"_id": "3", "title": "62"}], - MarqoIndexStats(number_of_documents=3, number_of_vectors=3) + MarqoIndexStats(number_of_documents=3, number_of_vectors=3, backend=VespaStats()) ), ( 'add', # add 3 docs, 1 has a vector [{"_id": "4", "desc": "2"}, {"_id": "5", "title": "2"}, {"_id": "6", "desc": "62"}], - MarqoIndexStats(number_of_documents=6, number_of_vectors=4) + MarqoIndexStats(number_of_documents=6, number_of_vectors=4, backend=VespaStats()) ), ( 'delete', # delete 2 docs, 1 has a vector ["1", "4"], - MarqoIndexStats(number_of_documents=4, number_of_vectors=3) + MarqoIndexStats(number_of_documents=4, number_of_vectors=3, backend=VespaStats()) ), ( 'add', # add 3 docs, 1 has a vector [{"_id": "7", "desc": "2"}, {"_id": "8", "title": "2"}, {"_id": "9", "desc": "62"}], - MarqoIndexStats(number_of_documents=7, number_of_vectors=4) + MarqoIndexStats(number_of_documents=7, number_of_vectors=4, backend=VespaStats()) ), ( 'delete', # delete all docs [f"{i}" for i in range(1, 10)], - MarqoIndexStats(number_of_documents=0, number_of_vectors=0) + MarqoIndexStats(number_of_documents=0, number_of_vectors=0, backend=VespaStats()) ) ] @@ -217,7 +237,7 @@ def test_get_index_stats_sequentialIndexingAndDeletion_successful(self): index_name=marqo_index.name, doc_ids=docs ) - self.assertEqual( + self.assertIndexStatsEqual( expected_stats, self.monitoring.get_index_stats(marqo_index) ) @@ -238,8 +258,12 @@ def test_get_index_stats_longText_successful(self): device="cpu" ) ) - self.assertEqual( - MarqoIndexStats(number_of_documents=2, number_of_vectors=4), + self.assertIndexStatsEqual( + MarqoIndexStats( + number_of_documents=2, + number_of_vectors=4, + backend=VespaStats() + ), self.monitoring.get_index_stats(marqo_index) ) @@ -256,8 +280,12 @@ def test_get_index_stats_by_name_docsWithTensorFields_successful(self): device="cpu" ) ) - self.assertEqual( - MarqoIndexStats(number_of_documents=3, number_of_vectors=3), + self.assertIndexStatsEqual( + MarqoIndexStats( + number_of_documents=3, + number_of_vectors=3, + backend=VespaStats() + ), self.monitoring.get_index_stats_by_name(marqo_index.name) ) @@ -267,3 +295,14 @@ def test_get_index_stats_by_name_indexDoesNotExist_fails(self): """ with self.assertRaises(IndexNotFoundError): self.monitoring.get_index_stats_by_name('index_does_not_exist') + + def assertIndexStatsEqual(self, expected: MarqoIndexStats, actual: MarqoIndexStats): + """ + Assert MarqoIndexStats equality, verifying only range validity for storage and memory stats. + """ + self.assertEqual(expected.number_of_documents, actual.number_of_documents) + self.assertEqual(expected.number_of_vectors, actual.number_of_vectors) + self.assertGreater(actual.backend.memory_used_percentage, 0.0) + self.assertLessEqual(actual.backend.memory_used_percentage, 100.0) + self.assertGreater(actual.backend.storage_used_percentage, 0.0) + self.assertLessEqual(actual.backend.storage_used_percentage, 100.0) diff --git a/tests/tensor_search/integ_tests/test_add_documents_structured.py b/tests/tensor_search/integ_tests/test_add_documents_structured.py index 671c12d58..d267653f7 100644 --- a/tests/tensor_search/integ_tests/test_add_documents_structured.py +++ b/tests/tensor_search/integ_tests/test_add_documents_structured.py @@ -719,7 +719,9 @@ def _check_get_docs(doc_count, title_value): ) self.assertEqual( c, - self.config.monitoring.get_index_stats(index_name=self.index_name_img_random).number_of_documents, + self.config.monitoring.get_index_stats_by_name( + index_name=self.index_name_img_random + ).number_of_documents, ) self.assertFalse(res1['errors']) self.assertTrue(_check_get_docs(doc_count=c, title_value='blah')) diff --git a/tests/tensor_search/test_context_vectors_search.py b/tests/tensor_search/test_context_vectors_search.py index a76e5f4c8..243781ca5 100644 --- a/tests/tensor_search/test_context_vectors_search.py +++ b/tests/tensor_search/test_context_vectors_search.py @@ -27,8 +27,8 @@ def setUp(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: True + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: True } }) add_docs_caller(config=self.config, index_name=self.index_name_1, docs=[ @@ -147,8 +147,8 @@ def setUp(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: True + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: True } }) add_docs_caller(config=self.config, index_name=self.index_name_1, docs=[ @@ -394,7 +394,7 @@ def test_context_vector_with_zero_vectors_search_in_bulk(self): image_index_config = { IndexSettingsField.index_defaults: { IndexSettingsField.model: "random/small", - IndexSettingsField.treat_urls_and_pointers_as_images: True + IndexSettingsField.treatUrlsAndPointersAsImages: True }, 'number_of_shards': 1, 'number_of_replicas': 0, diff --git a/tests/tensor_search/test_image_download_headers.py b/tests/tensor_search/test_image_download_headers.py index a8ab69f72..c0146811a 100644 --- a/tests/tensor_search/test_image_download_headers.py +++ b/tests/tensor_search/test_image_download_headers.py @@ -51,7 +51,7 @@ def image_index_settings(self) -> dict: return { IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, + IndexSettingsField.treatUrlsAndPointersAsImages: True, } } diff --git a/tests/tensor_search/test_index_info.py b/tests/tensor_search/test_index_info.py index 0b107682e..ba4e5e104 100644 --- a/tests/tensor_search/test_index_info.py +++ b/tests/tensor_search/test_index_info.py @@ -109,7 +109,7 @@ def test_get_ann_parameters__default_index_param(self): def test_get_ann_parameters__without_default_ann_parameters__use_defaults(self): index_settings = configs.get_default_index_settings() - del index_settings[NsFields.index_defaults][NsFields.ann_parameters] + del index_settings[NsFields.index_defaults][NsFields.annParameters] ii = IndexInfo( model_name='some model', @@ -120,7 +120,7 @@ def test_get_ann_parameters__without_default_ann_parameters__use_defaults(self): def test_get_ann_parameters__use_specified_index_settings__overide_defaults(self): index_settings = configs.get_default_index_settings() - index_settings[NsFields.index_defaults][NsFields.ann_parameters][NsFields.ann_method_name] = "not-hnsw" + index_settings[NsFields.index_defaults][NsFields.annParameters][NsFields.ann_method_name] = "not-hnsw" ii = IndexInfo( model_name='some model', @@ -138,7 +138,7 @@ def test_get_ann_parameters__use_specified_index_settings__overide_defaults(self def test_get_ann_parameters__use_specified_ann_method_parameters__default_unspecified_values(self): index_settings = configs.get_default_index_settings() - index_settings[NsFields.index_defaults][NsFields.ann_parameters][NsFields.ann_method_parameters] = { + index_settings[NsFields.index_defaults][NsFields.annParameters][NsFields.ann_method_parameters] = { NsFields.hnsw_ef_construction: 1, NsFields.hnsw_m: 2 } diff --git a/tests/tensor_search/test_multimodal_tensor_combination.py b/tests/tensor_search/test_multimodal_tensor_combination.py index 809128928..3a37d06c3 100644 --- a/tests/tensor_search/test_multimodal_tensor_combination.py +++ b/tests/tensor_search/test_multimodal_tensor_combination.py @@ -46,8 +46,8 @@ def test_add_documents(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -102,8 +102,8 @@ def get_score(document): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -145,8 +145,8 @@ def test_multimodal_tensor_combination_tensor_value(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -257,8 +257,8 @@ def get_score(document): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -296,8 +296,8 @@ def test_multimodal_tensor_combination_vectorise_call(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -372,8 +372,8 @@ def test_multimodal_field_content_dictionary_validation(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -529,8 +529,8 @@ def test_batched_vectorise_call(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -588,8 +588,8 @@ def test_batched_vectorise_call_infer_image_is_false(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: False, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: False, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -651,8 +651,8 @@ def test_concurrent_image_downloading(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -698,8 +698,8 @@ def test_lexical_search_on_multimodal_combination(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "random/small", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) tensor_search.add_documents(config=self.config, add_docs_params=AddDocsParams( @@ -761,8 +761,8 @@ def test_overwrite_multimodal_tensor_field(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "random/small", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -810,8 +810,8 @@ def test_search_with_filtering_and_infer_image_false(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: False, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: False, + IndexSettingsField.normalizeEmbeddings: False } }) @@ -867,8 +867,8 @@ def test_index_info_cache_update(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) tensor_search.add_documents( @@ -926,8 +926,8 @@ def test_duplication_in_child_fields(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "random/small", - IndexSettingsField.treat_urls_and_pointers_as_images: False, - IndexSettingsField.normalize_embeddings: True + IndexSettingsField.treatUrlsAndPointersAsImages: False, + IndexSettingsField.normalizeEmbeddings: True } }) @@ -997,8 +997,8 @@ def test_multimodal_combination_open_search_chunks(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "random/small", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: True + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: True } }) @@ -1049,8 +1049,8 @@ def test_multimodal_child_fields_order(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: True + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: True } }) @@ -1116,8 +1116,8 @@ def test_multimodal_child_fields_order_from_os(self): index_name=self.index_name_1, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: False + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: False } }) diff --git a/tests/tensor_search/test_score_modifiers_search.py b/tests/tensor_search/test_score_modifiers_search.py index 4263d1339..dd508eeff 100644 --- a/tests/tensor_search/test_score_modifiers_search.py +++ b/tests/tensor_search/test_score_modifiers_search.py @@ -28,8 +28,8 @@ def setUp(self): index_name=self.index_name, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: True + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: True } }) pass @@ -566,8 +566,8 @@ def setUp(self): index_name=self.index_name, config=self.config, index_settings={ IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/32", - IndexSettingsField.treat_urls_and_pointers_as_images: True, - IndexSettingsField.normalize_embeddings: True + IndexSettingsField.treatUrlsAndPointersAsImages: True, + IndexSettingsField.normalizeEmbeddings: True } }) pass diff --git a/tests/tensor_search/test_search.py b/tests/tensor_search/test_search.py index 5b786da92..cea659d32 100644 --- a/tests/tensor_search/test_search.py +++ b/tests/tensor_search/test_search.py @@ -1026,7 +1026,7 @@ def test_multi_search_images(self): image_index_config = { IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/16", - IndexSettingsField.treat_urls_and_pointers_as_images: True + IndexSettingsField.treatUrlsAndPointersAsImages: True } } tensor_search.delete_index(self.config, self.index_name_1) @@ -1077,7 +1077,7 @@ def test_multi_search_check_vector(self): image_index_config = { IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/16", - IndexSettingsField.treat_urls_and_pointers_as_images: True + IndexSettingsField.treatUrlsAndPointersAsImages: True } } tensor_search.delete_index(self.config, self.index_name_1) @@ -1157,7 +1157,7 @@ def test_multi_search_images_edge_cases(self): image_index_config = { IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/16", - IndexSettingsField.treat_urls_and_pointers_as_images: True + IndexSettingsField.treatUrlsAndPointersAsImages: True } } tensor_search.delete_index(self.config, self.index_name_1) @@ -1191,7 +1191,7 @@ def test_multi_search_images_ok_edge_cases(self): image_index_config = { IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/16", - IndexSettingsField.treat_urls_and_pointers_as_images: True + IndexSettingsField.treatUrlsAndPointersAsImages: True } } tensor_search.delete_index(self.config, self.index_name_1) @@ -1250,7 +1250,7 @@ def test_image_search(self): image_index_config = { IndexSettingsField.index_defaults: { IndexSettingsField.model: "ViT-B/16", - IndexSettingsField.treat_urls_and_pointers_as_images: True + IndexSettingsField.treatUrlsAndPointersAsImages: True } } tensor_search.delete_index(self.config, self.index_name_1)