Merge branch 'master' into AMORO-1035

apache · Sep 14, 2023 · 17ef90c · 17ef90c
2 parents d2efb0b + 9abe9db
commit 17ef90c
Show file tree

Hide file tree

Showing 7 changed files with 31 additions and 6 deletions.
diff --git a/ams/server/src/main/java/com/netease/arctic/server/dashboard/PlatformFileManager.java b/ams/server/src/main/java/com/netease/arctic/server/dashboard/PlatformFileManager.java
@@ -27,8 +27,8 @@ public String getFileContentB64ById(Integer fileId) {
   /**
    * getRuntime file content
    */
-  public String getFileContentById(Integer fileId) {
+  public byte[] getFileContentById(Integer fileId) {
     String fileContent = getAs(PlatformFileMapper.class, e -> e.getFileById(fileId));
-    return new String(Base64.getDecoder().decode(fileContent));
+    return Base64.getDecoder().decode(fileContent);
   }
 }
diff --git a/...erver/src/main/java/com/netease/arctic/server/dashboard/controller/CatalogController.java b/...erver/src/main/java/com/netease/arctic/server/dashboard/controller/CatalogController.java
@@ -391,11 +391,11 @@ public void getCatalogConfFileContent(Context ctx) {
     if (CONFIG_TYPE_STORAGE.equalsIgnoreCase(confType)) {
       Map<String, String> storageConfig = catalogMeta.getStorageConfigs();
       String key = configKey.replaceAll("-", "\\.");
-      ctx.result(new String(Base64.getDecoder().decode(storageConfig.get(key))));
+      ctx.result(Base64.getDecoder().decode(storageConfig.get(key)));
     } else if (CONFIG_TYPE_AUTH.equalsIgnoreCase(confType)) {
       Map<String, String> storageConfig = catalogMeta.getAuthConfigs();
       String key = configKey.replaceAll("-", "\\.");
-      ctx.result(new String(Base64.getDecoder().decode(storageConfig.get(key))));
+      ctx.result(Base64.getDecoder().decode(storageConfig.get(key)));
     } else {
       throw new RuntimeException("Invalid request for " + confType);
     }

diff --git a/.../main/java/com/netease/arctic/server/dashboard/controller/PlatformFileInfoController.java b/.../main/java/com/netease/arctic/server/dashboard/controller/PlatformFileInfoController.java
@@ -65,7 +65,7 @@ public void uploadFile(Context ctx) throws IOException {
   public void downloadFile(Context ctx) {
     String fileId = ctx.pathParam("fileId");
     Preconditions.checkArgument(StringUtils.isNumeric(fileId), "Invalid file id");
-    String content = platformFileInfoService.getFileContentById(Integer.valueOf(fileId));
+    byte[] content = platformFileInfoService.getFileContentById(Integer.valueOf(fileId));
     ctx.result(content);
   }
 }
diff --git a/...ver/src/main/java/com/netease/arctic/server/optimizing/plan/CommonPartitionEvaluator.java b/...ver/src/main/java/com/netease/arctic/server/optimizing/plan/CommonPartitionEvaluator.java
@@ -209,7 +209,11 @@ private void addDelete(ContentFile<?> delete) {
   @Override
   public boolean isNecessary() {
     if (necessary == null) {
-      necessary = isFullNecessary() || isMajorNecessary() || isMinorNecessary();
+      if (isFullOptimizing()) {
+        necessary = isFullNecessary();
+      } else {
+        necessary = isMajorNecessary() || isMinorNecessary();
+      }
       LOG.debug("{} necessary = {}, {}", name(), necessary, this);
     }
     return necessary;

diff --git a/charts/amoro/templates/amoro-configmap.yaml b/charts/amoro/templates/amoro-configmap.yaml
@@ -22,6 +22,11 @@ metadata:
   labels:
     {{- include "amoro.labels" . | nindent 4 }}
 data:
+  {{- with .Values.amoroConf.amoroEnv }}
+  config.sh: |
+    #!/usr/bin/env bash
+    {{- tpl . $ | nindent 4 }}
+  {{- end }}
   ## Helm chart provided Amoro configurations
   config.yaml: |
     ams:

diff --git a/charts/amoro/templates/amoro-deployment.yaml b/charts/amoro/templates/amoro-deployment.yaml
@@ -92,6 +92,14 @@ spec:
                 {{- tpl (toYaml .) $ | nindent 12 }}
               {{- end }}
             {{- end }}
+            {{- if or .Values.amoroConf.amoroEnv }}
+              mountPath: {{ .Values.amoroDir }}/amoro-{{ .Chart.AppVersion }}/bin/config.sh
+              readOnly: true
+              subPath: "config.sh"
+              {{- with .Values.volumeMounts }}
+                {{- tpl (toYaml .) $ | nindent 12 }}
+              {{- end }}
+            {{- end }}
           {{- with .Values.containers }}
             {{- tpl (toYaml .) $ | nindent 8 }}
           {{- end }}

diff --git a/charts/amoro/values.yaml b/charts/amoro/values.yaml
@@ -75,6 +75,14 @@ server:
 amoroDir: "/usr/local/ams"
 # AMS configurations files
 amoroConf:
+  # ams config.sh set env
+  amoroEnv: ~
+#  amoroEnv: |
+#    XMX_CONFIG="8196"
+#    XMS_CONFIG="8196"
+#    JMX_REMOTE_PORT_CONFIG=""
+#    JVM_EXTRA_CONFIG="-XX:NewRatio=1 -XX:SurvivorRatio=3"
+#    LANG_CONFIG="en_US.UTF-8"
   # ams database properties,default is derby. If production,suggest use mysql
   database:
     type: derby