seperate aws-sdk && use new sdk

.bazelignore

@@ -0,0 +1 @@
+thirdparties/aws/aws-sdk-cpp

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "nbd"]
 	path = nbd
 	url = https://github.com/opencurve/curve-nbd
+[submodule "thirdparties/aws/aws-sdk-cpp"]
+	path = thirdparties/aws/aws-sdk-cpp
+	url = https://github.com/opencurve/aws-sdk-cpp

WORKSPACE

@@ -173,37 +173,12 @@ new_local_repository(
     path = "thirdparties/etcdclient",
 )
 
-http_archive(
+new_local_repository(
     name = "aws",
-    urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
-    sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
-    strip_prefix = "aws-sdk-cpp-1.7.340",
     build_file = "//:thirdparties/aws/aws.BUILD",
+    path = "thirdparties/aws/aws-sdk-cpp",
 )
 
-http_archive(
-    name = "aws_c_common",
-    urls = ["https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz"],
-    sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
-    strip_prefix = "aws-c-common-0.4.29",
-    build_file = "//:thirdparties/aws/aws-c-common.BUILD",
-)
-
-http_archive(
-    name = "aws_c_event_stream",
-    urls = ["https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz"],
-    sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
-    strip_prefix = "aws-c-event-stream-0.1.4",
-    build_file = "//:thirdparties/aws/aws-c-event-stream.BUILD",
-)
-
-http_archive(
-    name = "aws_checksums",
-    urls = ["https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz"],
-    sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
-    strip_prefix = "aws-checksums-0.1.5",
-    build_file = "//:thirdparties/aws/aws-checksums.BUILD",
-)
 
 # C++ rules for Bazel.
 http_archive(

curvefs/docker/debian10/Dockerfile

@@ -4,5 +4,7 @@ RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+COPY libaws-cpp-sdk-core.so /usr/lib/
+COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian11/Dockerfile

@@ -4,5 +4,7 @@ RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+COPY libaws-cpp-sdk-core.so /usr/lib/
+COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian9/Dockerfile

@@ -4,5 +4,7 @@ RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+COPY libaws-cpp-sdk-core.so /usr/lib/
+COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/src/client/fuse_s3_client.cpp

@@ -51,15 +51,15 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     SetFuseClientS3Option(&opt, fsS3Option);
 
     auto s3Client = std::make_shared<S3ClientImpl>();
-    auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
     s3Client->Init(opt.s3Opt.s3AdaptrOpt);
-    s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
         opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
         opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte);
     if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
         DiskCacheType::Disable) {
+        auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
+        s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
         auto wrapper = std::make_shared<PosixWrapper>();
         auto diskCacheRead = std::make_shared<DiskCacheRead>();
         auto diskCacheWrite = std::make_shared<DiskCacheWrite>();
@@ -82,6 +82,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
 void FuseS3Client::UnInit() {
     s3Adaptor_->Stop();
     FuseClient::UnInit();
+    curve::common::S3Adapter::Shutdown();
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,

curvefs/src/client/s3/disk_cache_manager_impl.cpp

@@ -178,6 +178,7 @@ int DiskCacheManagerImpl::UmountDiskCache() {
         return -1;
     }
     taskPool_.Stop();
+    client_->Deinit();
     return 0;
 }
 

curvefs/src/mds/main.cpp

@@ -86,6 +86,7 @@ int main(int argc, char **argv) {
     // stop server and background threads
     mds.Stop();
 
+    curve::common::S3Adapter::Shutdown();
     google::ShutdownGoogleLogging();
     return 0;
 }

curvefs/src/metaserver/main.cpp

@@ -144,6 +144,7 @@ int main(int argc, char **argv) {
     // stop server and background threads
     metaserver.Stop();
 
+    curve::common::S3Adapter::Shutdown();
     google::ShutdownGoogleLogging();
     return 0;
 }

curvefs/src/metaserver/metaserver.cpp

@@ -445,8 +445,8 @@ void Metaserver::Stop() {
     LOG_IF(ERROR, !copysetNodeManager_->Stop())
         << "Failed to stop copyset node manager";
 
+    s3Adaptor_ = nullptr;
     S3CompactManager::GetInstance().Stop();
-
     LOG(INFO) << "MetaServer stopped success";
 }
 

curvefs/src/metaserver/partition_clean_manager.cpp

@@ -80,6 +80,7 @@ void PartitionCleanManager::Fini() {
         thread_.join();
         partitonCleanerList_.clear();
         inProcessingCleaner_ = nullptr;
+        S3ClientAdaptor_ = nullptr;
     }
     LOG(INFO) << "stop PartitionCleanManager manager ok.";
 }

curvefs/src/metaserver/trash_manager.cpp

@@ -42,6 +42,7 @@ void TrashManager::Fini() {
         LOG(INFO) << "stop trash manager ...";
         sleeper_.interrupt();
         recycleThread_.join();
+        options_ = {};
     }
     LOG(INFO) << "stop trash manager ok.";
 }

curvefs/test/client/client_s3_test.cpp

@@ -40,10 +40,10 @@ class ClientS3Test : public testing::Test {
     ClientS3Test() {}
     ~ClientS3Test() {}
     virtual void SetUp() {
+        Aws::InitAPI(awsOptions_);
         client_ = new S3ClientImpl();
         s3Client_ = std::make_shared<MockS3Adapter>();
         client_->SetAdapter(s3Client_);
-        Aws::InitAPI(awsOptions_);
     }
 
     virtual void TearDown() {

curvefs/test/client/test_disk_cache_manager_impl.cpp

@@ -96,6 +96,9 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
 
 TEST_F(TestDiskCacheManagerImpl, Init) {
     S3ClientAdaptorOption s3AdaptorOption;
+    s3AdaptorOption.diskCacheOpt.forceFlush = false;
+    s3AdaptorOption.diskCacheOpt.threads = 0;
+
     EXPECT_CALL(*diskCacheManager_, Init(_, _)).WillOnce(Return(-1));
     int ret = diskCacheManagerImpl_->Init(s3AdaptorOption);
     ASSERT_EQ(-1, ret);

curvefs/test/metaserver/metaserver_test.cpp

@@ -56,6 +56,7 @@ class MetaserverTest : public ::testing::Test {
  protected:
     void SetUp() override {
         // run mds server
+        Aws::InitAPI(aws_sdk_options_);
         dataDir_ = RandomStoragePath();
         metaPath_ = "./meta.dat";
         ASSERT_EQ(0, server_.AddService(&mockTopologyService_,
@@ -86,6 +87,7 @@ class MetaserverTest : public ::testing::Test {
         server_.Join();
         auto output = execShell("rm -rf " + dataDir_);
         ASSERT_EQ(output.size(), 0);
+        Aws::ShutdownAPI(aws_sdk_options_);
         return;
     }
 
@@ -113,6 +115,7 @@ class MetaserverTest : public ::testing::Test {
     MockHeartbeatService mockHeartbeatService_;
 
     brpc::Server server_;
+    Aws::SDKOptions aws_sdk_options_;
 };
 
 template <typename RpcRequestType, typename RpcResponseType,

curvefs/test/metaserver/s3compact/mock_s3_adapter.h

@@ -56,8 +56,7 @@ class MockS3Adapter : public S3Adapter {
 
     MOCK_METHOD1(Init, void(const std::string&));
     MOCK_METHOD0(Deinit, void());
-    MOCK_METHOD4(Reinit, void(const std::string&, const std::string&,
-                              const std::string&, S3AdapterOption opt));
+    MOCK_METHOD1(Reinit, void(const S3AdapterOption& opt));
     MOCK_METHOD0(GetS3Ak, std::string());
     MOCK_METHOD0(GetS3Sk, std::string());
     MOCK_METHOD0(GetS3Endpoint, std::string());

curvefs/test/metaserver/s3compact/s3compact_test.cpp

@@ -107,6 +107,7 @@ class S3CompactTest : public ::testing::Test {
     void TearDown() override {
         ASSERT_TRUE(kvStorage_->Close());
         ASSERT_EQ(0, system(std::string{"rm -rf " + dataDir_}.c_str()));
+        curve::common::S3Adapter::Shutdown();
     }
 
  protected:
@@ -601,7 +602,7 @@ TEST_F(S3CompactTest, test_CompactChunks) {
     EXPECT_CALL(*s3adapter_, GetS3Ak()).WillRepeatedly(Return(v));
     EXPECT_CALL(*s3adapter_, GetS3Sk()).WillRepeatedly(Return(v));
     EXPECT_CALL(*s3adapter_, GetS3Endpoint()).WillRepeatedly(Return(v));
-    EXPECT_CALL(*s3adapter_, Reinit(_, _, _, _)).WillRepeatedly(Return());
+    EXPECT_CALL(*s3adapter_, Reinit(_)).WillRepeatedly(Return());
     EXPECT_CALL(*s3adapter_, GetBucketName()).WillRepeatedly(Return(v));
 
     struct CompactInodeJob::S3CompactTask t {

curvefs/util/build.sh

@@ -191,6 +191,8 @@ build_requirements() {
     fi
     g_rocksdb_root="$(dirname ${PWD})/thirdparties/rocksdb"
     (cd ${g_rocksdb_root} && make build from_source=${g_build_rocksdb} && make install prefix=${g_rocksdb_root})
+    g_aws_sdk_root="$(dirname ${PWD})/thirdparties/aws"
+    (cd ${g_aws_sdk_root} && make)
 }
 
 main() {

curvefs/util/image.sh

@@ -21,6 +21,8 @@ mkdir -p $prefix $prefix/conf
 make install prefix="$prefix"
 make install prefix="$prefix" only=etcd
 make install prefix="$prefix" only=monitor
+cp -f ../thirdparties/aws/aws-sdk-cpp/build/aws-cpp-sdk-core/libaws-cpp-sdk-core.so docker/$2
+cp -f ../thirdparties/aws/aws-sdk-cpp/build/aws-cpp-sdk-s3-crt/libaws-cpp-sdk-s3-crt.so docker/$2
 for file in `ls conf`;
 do
     dsv="="

docker/debian10/Dockerfile

@@ -4,5 +4,7 @@ RUN mkdir -p /curvebs /etc/curve /etc/nebd /core
 COPY curvebs /curvebs
 COPY entrypoint.sh /
 COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/
+COPY libaws-cpp-sdk-core.so /usr/lib/
+COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

docker/debian11/Dockerfile

@@ -4,5 +4,7 @@ RUN mkdir -p /curvebs /etc/curve /etc/nebd /core
 COPY curvebs /curvebs
 COPY entrypoint.sh /
 COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/
+COPY libaws-cpp-sdk-core.so /usr/lib/
+COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

docker/debian9/Dockerfile

@@ -4,5 +4,7 @@ RUN mkdir -p /curvebs /etc/curve /etc/nebd /core
 COPY curvebs /curvebs
 COPY entrypoint.sh /
 COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/
+COPY libaws-cpp-sdk-core.so /usr/lib/
+COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]