Merge branch 'main' into chore/bump-version-to-0.14.1

langgenius · Dec 18, 2024 · b7de5d9 · b7de5d9
2 parents 5657107 + 56434db
commit b7de5d9
Show file tree

Hide file tree

Showing 18 changed files with 308 additions and 17 deletions.
diff --git a/api/core/model_runtime/model_providers/hunyuan/hunyuan.yaml b/api/core/model_runtime/model_providers/hunyuan/hunyuan.yaml
@@ -3,8 +3,8 @@ label:
   zh_Hans: 腾讯混元
   en_US: Hunyuan
 description:
-  en_US: Models provided by Tencent Hunyuan, such as hunyuan-standard, hunyuan-standard-256k, hunyuan-pro and hunyuan-lite.
-  zh_Hans: 腾讯混元提供的模型，例如 hunyuan-standard、 hunyuan-standard-256k, hunyuan-pro 和 hunyuan-lite。
+  en_US: Models provided by Tencent Hunyuan, such as hunyuan-standard, hunyuan-standard-256k, hunyuan-pro, hunyuan-role, hunyuan-large, hunyuan-large-role, hunyuan-turbo-latest, hunyuan-large-longcontext, hunyuan-turbo, hunyuan-vision, hunyuan-turbo-vision, hunyuan-functioncall and hunyuan-lite.
+  zh_Hans: 腾讯混元提供的模型，例如 hunyuan-standard、 hunyuan-standard-256k, hunyuan-pro, hunyuan-role, hunyuan-large, hunyuan-large-role, hunyuan-turbo-latest, hunyuan-large-longcontext, hunyuan-turbo, hunyuan-vision, hunyuan-turbo-vision, hunyuan-functioncall 和 hunyuan-lite。
 icon_small:
   en_US: icon_s_en.png
 icon_large:

diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/_position.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/_position.yaml
@@ -4,3 +4,10 @@
 - hunyuan-pro
 - hunyuan-turbo
 - hunyuan-vision
+- hunyuan-role
+- hunyuan-large
+- hunyuan-large-role
+- hunyuan-large-longcontext
+- hunyuan-turbo-latest
+- hunyuan-turbo-vision
+- hunyuan-functioncall
diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-functioncall.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-functioncall.yaml
@@ -0,0 +1,38 @@
+model: hunyuan-functioncall
+label:
+  zh_Hans: hunyuan-functioncall
+  en_US: hunyuan-functioncall
+model_type: llm
+features:
+  - agent-thought
+  - tool-call
+  - multi-tool-call
+  - stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 32000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    min: 1
+    max: 32000
+  - name: enable_enhance
+    label:
+      zh_Hans: 功能增强
+      en_US: Enable Enhancement
+    type: boolean
+    help:
+      zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
+      en_US: Allow the model to perform external search to enhance the generation results.
+    required: false
+    default: true
+pricing:
+  input: '0.004'
+  output: '0.008'
+  unit: '0.001'
+  currency: RMB
diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large-longcontext.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large-longcontext.yaml
@@ -0,0 +1,38 @@
+model: hunyuan-large-longcontext
+label:
+  zh_Hans: hunyuan-large-longcontext
+  en_US: hunyuan-large-longcontext
+model_type: llm
+features:
+  - agent-thought
+  - tool-call
+  - multi-tool-call
+  - stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 134000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    min: 1
+    max: 134000
+  - name: enable_enhance
+    label:
+      zh_Hans: 功能增强
+      en_US: Enable Enhancement
+    type: boolean
+    help:
+      zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
+      en_US: Allow the model to perform external search to enhance the generation results.
+    required: false
+    default: true
+pricing:
+  input: '0.006'
+  output: '0.018'
+  unit: '0.001'
+  currency: RMB
diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large-role.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large-role.yaml
@@ -0,0 +1,38 @@
+model: hunyuan-large-role
+label:
+  zh_Hans: hunyuan-large-role
+  en_US: hunyuan-large-role
+model_type: llm
+features:
+  - agent-thought
+  - tool-call
+  - multi-tool-call
+  - stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 32000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    min: 1
+    max: 32000
+  - name: enable_enhance
+    label:
+      zh_Hans: 功能增强
+      en_US: Enable Enhancement
+    type: boolean
+    help:
+      zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
+      en_US: Allow the model to perform external search to enhance the generation results.
+    required: false
+    default: true
+pricing:
+  input: '0.004'
+  output: '0.008'
+  unit: '0.001'
+  currency: RMB
diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large.yaml
@@ -0,0 +1,38 @@
+model: hunyuan-large
+label:
+  zh_Hans: hunyuan-large
+  en_US: hunyuan-large
+model_type: llm
+features:
+  - agent-thought
+  - tool-call
+  - multi-tool-call
+  - stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 32000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    min: 1
+    max: 32000
+  - name: enable_enhance
+    label:
+      zh_Hans: 功能增强
+      en_US: Enable Enhancement
+    type: boolean
+    help:
+      zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
+      en_US: Allow the model to perform external search to enhance the generation results.
+    required: false
+    default: true
+pricing:
+  input: '0.004'
+  output: '0.012'
+  unit: '0.001'
+  currency: RMB
diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-role.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-role.yaml
@@ -0,0 +1,38 @@
+model: hunyuan-role
+label:
+  zh_Hans: hunyuan-role
+  en_US: hunyuan-role
+model_type: llm
+features:
+  - agent-thought
+  - tool-call
+  - multi-tool-call
+  - stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 32000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    min: 1
+    max: 32000
+  - name: enable_enhance
+    label:
+      zh_Hans: 功能增强
+      en_US: Enable Enhancement
+    type: boolean
+    help:
+      zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
+      en_US: Allow the model to perform external search to enhance the generation results.
+    required: false
+    default: true
+pricing:
+  input: '0.004'
+  output: '0.008'
+  unit: '0.001'
+  currency: RMB
diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-turbo-latest.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-turbo-latest.yaml
@@ -0,0 +1,38 @@
+model: hunyuan-turbo-latest
+label:
+  zh_Hans: hunyuan-turbo-latest
+  en_US: hunyuan-turbo-latest
+model_type: llm
+features:
+  - agent-thought
+  - tool-call
+  - multi-tool-call
+  - stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 32000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    min: 1
+    max: 32000
+  - name: enable_enhance
+    label:
+      zh_Hans: 功能增强
+      en_US: Enable Enhancement
+    type: boolean
+    help:
+      zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
+      en_US: Allow the model to perform external search to enhance the generation results.
+    required: false
+    default: true
+pricing:
+  input: '0.015'
+  output: '0.05'
+  unit: '0.001'
+  currency: RMB
diff --git a/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-turbo-vision.yaml b/api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-turbo-vision.yaml
@@ -0,0 +1,39 @@
+model: hunyuan-turbo-vision
+label:
+  zh_Hans: hunyuan-turbo-vision
+  en_US: hunyuan-turbo-vision
+model_type: llm
+features:
+  - agent-thought
+  - tool-call
+  - multi-tool-call
+  - stream-tool-call
+  - vision
+model_properties:
+  mode: chat
+  context_size: 8000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    min: 1
+    max: 8000
+  - name: enable_enhance
+    label:
+      zh_Hans: 功能增强
+      en_US: Enable Enhancement
+    type: boolean
+    help:
+      zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
+      en_US: Allow the model to perform external search to enhance the generation results.
+    required: false
+    default: true
+pricing:
+  input: '0.08'
+  output: '0.08'
+  unit: '0.001'
+  currency: RMB
diff --git a/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py b/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py
@@ -49,10 +49,10 @@ def to_opensearch_params(self) -> dict[str, Any]:
 
 
 class LindormVectorStore(BaseVector):
-    def __init__(self, collection_name: str, config: LindormVectorStoreConfig, **kwargs):
+    def __init__(self, collection_name: str, config: LindormVectorStoreConfig, using_ugc: bool, **kwargs):
         self._routing = None
         self._routing_field = None
-        if config.using_ugc:
+        if using_ugc:
             routing_value: str = kwargs.get("routing_value")
             if routing_value is None:
                 raise ValueError("UGC index should init vector with valid 'routing_value' parameter value")
@@ -64,7 +64,7 @@ def __init__(self, collection_name: str, config: LindormVectorStoreConfig, **kwa
             super().__init__(collection_name.lower())
         self._client_config = config
         self._client = OpenSearch(**config.to_opensearch_params())
-        self._using_ugc = config.using_ugc
+        self._using_ugc = using_ugc
         self.kwargs = kwargs
 
     def get_type(self) -> str:
@@ -467,12 +467,16 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings
         using_ugc = dify_config.USING_UGC_INDEX
         routing_value = None
         if dataset.index_struct:
-            if using_ugc:
+            # if an existed record's index_struct_dict doesn't contain using_ugc field,
+            # it actually stores in the normal index format
+            stored_in_ugc = dataset.index_struct_dict.get("using_ugc", False)
+            using_ugc = stored_in_ugc
+            if stored_in_ugc:
                 dimension = dataset.index_struct_dict["dimension"]
                 index_type = dataset.index_struct_dict["index_type"]
                 distance_type = dataset.index_struct_dict["distance_type"]
-                index_name = f"{UGC_INDEX_PREFIX}_{dimension}_{index_type}_{distance_type}"
                 routing_value = dataset.index_struct_dict["vector_store"]["class_prefix"]
+                index_name = f"{UGC_INDEX_PREFIX}_{dimension}_{index_type}_{distance_type}"
             else:
                 index_name = dataset.index_struct_dict["vector_store"]["class_prefix"]
         else:
@@ -487,11 +491,12 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings
                 "index_type": index_type,
                 "dimension": dimension,
                 "distance_type": distance_type,
+                "using_ugc": using_ugc,
             }
             dataset.index_struct = json.dumps(index_struct_dict)
             if using_ugc:
                 index_name = f"{UGC_INDEX_PREFIX}_{dimension}_{index_type}_{distance_type}"
                 routing_value = class_prefix
             else:
                 index_name = class_prefix
-        return LindormVectorStore(index_name, lindorm_config, routing_value=routing_value)
+        return LindormVectorStore(index_name, lindorm_config, routing_value=routing_value, using_ugc=using_ugc)
diff --git a/api/extensions/storage/opendal_storage.py b/api/extensions/storage/opendal_storage.py
@@ -34,7 +34,6 @@ def __init__(self, scheme: str, **kwargs):
             root = kwargs.get("root", "storage")
             Path(root).mkdir(parents=True, exist_ok=True)
 
-        # self.op = opendal.Operator(scheme=scheme, **kwargs)
         self.op = opendal.Operator(scheme=scheme, **kwargs)
         logger.debug(f"opendal operator created with scheme {scheme}")
         retry_layer = opendal.layers.RetryLayer(max_times=3, factor=2.0, jitter=True)

diff --git a/api/services/feature_service.py b/api/services/feature_service.py
@@ -63,6 +63,7 @@ class SystemFeatureModel(BaseModel):
     enable_social_oauth_login: bool = False
     is_allow_register: bool = False
     is_allow_create_workspace: bool = False
+    is_email_setup: bool = False
     license: LicenseModel = LicenseModel()
 
 
@@ -98,6 +99,7 @@ def _fulfill_system_params_from_env(cls, system_features: SystemFeatureModel):
         system_features.enable_social_oauth_login = dify_config.ENABLE_SOCIAL_OAUTH_LOGIN
         system_features.is_allow_register = dify_config.ALLOW_REGISTER
         system_features.is_allow_create_workspace = dify_config.ALLOW_CREATE_WORKSPACE
+        system_features.is_email_setup = dify_config.MAIL_TYPE is not None and dify_config.MAIL_TYPE != ""
 
     @classmethod
     def _fulfill_params_from_env(cls, features: FeatureModel):

diff --git a/docker/.env.example b/docker/.env.example
@@ -284,8 +284,11 @@ CONSOLE_CORS_ALLOW_ORIGINS=*
 # The type of storage to use for storing user files.
 STORAGE_TYPE=opendal
 
-# Apache OpenDAL Configuration, refer to https://github.com/apache/opendal
-# The scheme for the OpenDAL storage.
+# Apache OpenDAL Configuration
+# The configuration for OpenDAL consists of the following format: OPENDAL_<SCHEME_NAME>_<CONFIG_NAME>.
+# You can find all the service configurations (CONFIG_NAME) in the repository at: https://github.com/apache/opendal/tree/main/core/src/services.
+# Dify will scan configurations starting with OPENDAL_<SCHEME_NAME> and automatically apply them.
+# The scheme name for the OpenDAL storage.
 OPENDAL_SCHEME=fs
 # Configurations for OpenDAL Local File System.
 OPENDAL_FS_ROOT=storage
@@ -919,4 +922,4 @@ MAX_SUBMIT_COUNT=100
 
 # Proxy
 HTTP_PROXY=
-HTTPS_PROXY=
+HTTPS_PROXY=