Add legacy connector larksheet

bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-larksheet/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>bitsail-connectors-legacy</artifactId>
+        <groupId>com.bytedance.bitsail</groupId>
+        <version>${revision}</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>bitsail-connector-larksheet</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.bytedance.bitsail</groupId>
+            <artifactId>bitsail-common</artifactId>
+            <version>${revision}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.bytedance.bitsail</groupId>
+            <artifactId>bitsail-connector-print</artifactId>
+            <version>${revision}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.bytedance.bitsail</groupId>
+            <artifactId>bitsail-connector-test</artifactId>
+            <version>${revision}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-larksheet/src/main/java/com/bytedance/bitsail/connector/legacy/larksheet/api/SheetConfig.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.bytedance.bitsail.connector.legacy.larksheet.api;
+
+import com.bytedance.bitsail.common.configuration.BitSailConfiguration;
+import com.bytedance.bitsail.connector.legacy.larksheet.option.LarkSheetReaderOptions;
+
+import com.google.common.collect.Sets;
+import lombok.NoArgsConstructor;
+
+import java.io.Serializable;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+@NoArgsConstructor
+public class SheetConfig implements Serializable {
+
+  private static final long serialVersionUID = 4138998289023711374L;
+
+  /**
+   * Parameters for request retry.
+   */
+  public static final int ATTEMPT_NUMBER = 3;
+  public static final int WAIT_MILLISECONDS = 10;
+
+  /**
+   * Response status code for Lark api.<br/>
+   * Ref: <a href="https://open.feishu.cn/document/ukTMukTMukTM/ugjM14COyUjL4ITN?lang=en-US">Server Error Codes</a>
+   */
+  public static final int REQUEST_SUCCESS = 0;
+  public static final int REQUEST_TOO_FREQUENT = 99991400;
+  public static final int REQUEST_FORBIDDEN = 91403;
+  public static final int REQUEST_TOO_MANY = 1000004;
+  public static final int INVALID_TENANT_ACCESS_TOKEN = 99991663;
+  public static final int INVALID_APP_ACCESS_TOKEN = 99991664;
+  public static final int INVALID_ACCESS_TOKEN = 99991671;
+
+  /**
+   * Error codes related to flow control.
+   */
+  public static final Set<Integer> FLOW_CONTROL_CODES = Sets.newHashSet(REQUEST_TOO_FREQUENT, REQUEST_TOO_MANY);
+
+  /**
+   * Error codes related to illegal token.
+   */
+  public static final Set<Integer> INVALID_ACCESS_TOKEN_CODES = Sets.newHashSet(INVALID_TENANT_ACCESS_TOKEN, INVALID_APP_ACCESS_TOKEN, INVALID_ACCESS_TOKEN);
+
+  /**
+   * Maximum reader parallelism.<br/>
+   * Reason: Feishu open platform has flow control.
+   */
+  public static final int MAX_READ_PARALLELISM = 5;
+
+  /**
+   * SheetId key in user-defined sheet url.
+   */
+  public static final String SHEET_ID_URL_PARAM_NAME = "sheet";
+
+  /**
+   * Regular expression used to parse sheetToken.
+   */
+  public static final Pattern SHEET_TOKEN_PATTERN = Pattern.compile("/sheets/([0-9a-zA-Z]+)");
+
+  /**
+   * Token defined in job conf.
+   */
+  public static String PRE_DEFINED_SHEET_TOKEN;
+
+  /**
+   * APP_ID
+   */
+  public static String APP_ID;
+
+  /**
+   * APP_SECRET
+   */
+  public static String APP_SECRET;
+
+  /**
+   * Feishu open api host.
+   */
+  public static String OPEN_API_HOST;
+
+  /**
+   * Api for custom applications get app_access_token.<br/>
+   * Ref: <a href="https://open.feishu.cn/document/ukTMukTMukTM/uADN14CM0UjLwQTN">app_access_token</a>
+   */
+  public static String APP_ACCESS_TOKEN_API;
+
+  /**
+   * Api for get sheet meta info.<br/>
+   * Ref: <a href="https://open.feishu.cn/document/ukTMukTMukTM/uETMzUjLxEzM14SMxMTN">Get spreadsheet metadata</a>
+   */
+  public static String META_INFO_API_FORMAT;
+
+  /**
+   * Api for get a range of data from sheet.<br/>
+   * Ref: <a href="https://open.feishu.cn/document/ukTMukTMukTM/ugTMzUjL4EzM14COxMTN">Read a single range</a>
+   */
+  public static String SINGLE_RANGE_API_FORMAT;
+
+  public SheetConfig configure(BitSailConfiguration jobConf) {
+    PRE_DEFINED_SHEET_TOKEN = jobConf.get(LarkSheetReaderOptions.SHEET_TOKEN);
+
+    APP_ID = jobConf.get(LarkSheetReaderOptions.APP_ID);
+    APP_SECRET = jobConf.get(LarkSheetReaderOptions.APP_SECRET);
+    OPEN_API_HOST = jobConf.get(LarkSheetReaderOptions.OPEN_API_HOST);
+    APP_ACCESS_TOKEN_API = jobConf.get(LarkSheetReaderOptions.APP_ACCESS_TOKEN_API);
+    META_INFO_API_FORMAT = jobConf.get(LarkSheetReaderOptions.META_INFO_API_FORMAT);
+    SINGLE_RANGE_API_FORMAT = jobConf.get(LarkSheetReaderOptions.SINGLE_RANGE_API_FORMAT);
+
+    return this;
+  }
+}

bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-larksheet/src/main/java/com/bytedance/bitsail/connector/legacy/larksheet/api/TokenHolder.java

@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.bytedance.bitsail.connector.legacy.larksheet.api;
+
+import com.bytedance.bitsail.common.util.FastJsonUtil;
+import com.bytedance.bitsail.common.util.HttpManager;
+import com.bytedance.bitsail.common.util.Preconditions;
+import com.bytedance.bitsail.connector.legacy.larksheet.api.response.AppAccessTokenResponse;
+import com.bytedance.bitsail.connector.legacy.larksheet.api.response.OpenApiBaseResponse;
+
+import com.github.rholder.retry.RetryException;
+import com.github.rholder.retry.Retryer;
+import com.github.rholder.retry.RetryerBuilder;
+import com.github.rholder.retry.StopStrategies;
+import com.github.rholder.retry.WaitStrategies;
+import com.google.common.collect.Maps;
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.entity.ContentType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+public class TokenHolder {
+  private static final Logger LOG = LoggerFactory.getLogger(TokenHolder.class);
+
+  private TokenHolder() {
+  }
+
+  private static volatile String appAccessToken;
+
+  @Getter
+  private static volatile boolean generateTokenByApi = false;
+
+  private static final Retryer<OpenApiBaseResponse> RETRYER = RetryerBuilder.<OpenApiBaseResponse>newBuilder()
+      .retryIfException()
+      .retryIfResult(OpenApiBaseResponse::isFlowLimited)
+      .withStopStrategy(StopStrategies.stopAfterAttempt(SheetConfig.ATTEMPT_NUMBER))
+      .withWaitStrategy(WaitStrategies.fixedWait(SheetConfig.WAIT_MILLISECONDS, TimeUnit.MILLISECONDS))
+      .build();
+
+  /**
+   * Init app_access_token (singleton mode).<br/>
+   * 1. If user defines sheet_token in job conf, then use it.<br/>
+   * 2. If sheet_token is not defined, then use lark open api to get token.<br/>
+   *
+   * @return app_access_token
+   */
+  public static String init(String preDefinedToken) {
+    if (StringUtils.isNotEmpty(preDefinedToken)) {
+      appAccessToken = preDefinedToken;
+      LOG.info("Use pre-defined token from job configuration.");
+    }
+
+    if (StringUtils.isBlank(appAccessToken)) {
+      synchronized (TokenHolder.class) {
+        if (StringUtils.isBlank(appAccessToken)) {
+          refreshToken();
+        }
+      }
+    }
+    LOG.info("TokenHolder has been initialized successfully!");
+    return appAccessToken;
+  }
+
+  /**
+   * Make sure init() has been executed before getToken().
+   *
+   * @return app_access_token
+   */
+  public static String getToken() {
+    Preconditions.checkArgument(StringUtils.isNotBlank(appAccessToken),
+        "app_access_token is empty, please make sure TokenHolder.init() has been invoked before");
+    return appAccessToken;
+  }
+
+  /**
+   * Generate token by API.<br/>
+   * Ref: <a href="https://open.feishu.cn/document/ukTMukTMukTM/uADN14CM0UjLwQTN?lang=en-US">Custom applications get app_access_token</a>
+   */
+  public static void refreshToken() {
+    LOG.info("Start to generate or refresh app_access_token...");
+    Map<String, Object> body = Maps.newHashMap();
+    body.put("app_id", SheetConfig.APP_ID);
+    body.put("app_secret", SheetConfig.APP_SECRET);
+
+    AppAccessTokenResponse response;
+
+    try {
+      response = (AppAccessTokenResponse) RETRYER.call(() -> {
+        HttpManager.WrappedResponse wrappedResponse;
+        String url = SheetConfig.OPEN_API_HOST + SheetConfig.APP_ACCESS_TOKEN_API;
+        wrappedResponse = HttpManager.sendPost(url, null, body, ContentType.APPLICATION_JSON);
+        AppAccessTokenResponse tmpResponse = FastJsonUtil.parseObject(wrappedResponse.getResult(),
+            AppAccessTokenResponse.class);
+        if (tmpResponse.isFlowLimited()) {
+          LOG.info("trigger flow control when generate app_access_token, maybe retry later...");
+        }
+        return tmpResponse;
+      });
+    } catch (ExecutionException | RetryException e) {
+      throw new RuntimeException(String.format("Error while get app_access_token from lark open api, caused by: %s",
+          e.getCause().getMessage()), e.getCause());
+    }
+
+    if (response == null || response.isFailed() || StringUtils.isBlank(response.getAppAccessToken())) {
+      throw new RuntimeException(String.format("generate app_access_token from lark open api failed." +
+          " please check your app_id and app_secret, response is :%s", response));
+    }
+    appAccessToken = response.getAppAccessToken();
+    generateTokenByApi = true;
+    LOG.info("Successfully generate or refresh app_access_token!");
+  }
+
+}
+

bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-larksheet/src/main/java/com/bytedance/bitsail/connector/legacy/larksheet/api/response/AppAccessTokenResponse.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.bytedance.bitsail.connector.legacy.larksheet.api.response;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@ToString(callSuper = true)
+public class AppAccessTokenResponse extends OpenApiBaseResponse {
+
+  /**
+   * app_access_token
+   */
+  private String appAccessToken;
+
+  /**
+   * expire time in seconds
+   */
+  private int expire;
+
+  /**
+   * tenant_access_token
+   */
+  private String tenantAccessToken;
+
+}