Compile cloud features with 6.4.tikv branch

cloud/aws/aws_env.cc

@@ -41,7 +41,7 @@ struct JobHandle {
 
 class JobExecutor {
  public:
- shared_ptr<JobHandle> ScheduleJob(std::chrono::steady_clock::time_point time,
+ std::shared_ptr<JobHandle> ScheduleJob(std::chrono::steady_clock::time_point time,
  std::function<void(void)> callback);
  void CancelJob(JobHandle* handle);
 
@@ -254,7 +254,7 @@ AwsEnv::AwsEnv(Env* underlying_env, const std::string& src_bucket_prefix,
  dest_object_prefix_ = trim(dest_object_prefix_);
  dest_bucket_region_ = trim(dest_bucket_region_);
 
- unique_ptr<Aws::Auth::AWSCredentials> creds;
+ std::unique_ptr<Aws::Auth::AWSCredentials> creds;
  if (!cloud_env_options.credentials.access_key_id.empty() &&
  !cloud_env_options.credentials.secret_key.empty()) {
  creds.reset(new Aws::Auth::AWSCredentials(
@@ -332,7 +332,7 @@ AwsEnv::AwsEnv(Env* underlying_env, const std::string& src_bucket_prefix,
  GetBucketLocationConstraintForName(config.region);
 
  {
- unique_ptr<Aws::S3::S3Client> s3client(
+ std::unique_ptr<Aws::S3::S3Client> s3client(
  creds ? new Aws::S3::S3Client(*creds, config)
  : new Aws::S3::S3Client(config));
 
@@ -484,10 +484,10 @@ Status AwsEnv::CheckOption(const EnvOptions& options) {
 // Ability to read a file directly from cloud storage
 Status AwsEnv::NewSequentialFileCloud(const std::string& bucket_prefix,
  const std::string& fname,
- unique_ptr<SequentialFile>* result,
+ std::unique_ptr<SequentialFile>* result,
  const EnvOptions& options) {
  assert(status().ok());
- unique_ptr<S3ReadableFile> file;
+ std::unique_ptr<S3ReadableFile> file;
  Status st = NewS3ReadableFile(bucket_prefix, fname, &file);
  if (!st.ok()) {
  return st;
@@ -499,7 +499,7 @@ Status AwsEnv::NewSequentialFileCloud(const std::string& bucket_prefix,
 
 // open a file for sequential reading
 Status AwsEnv::NewSequentialFile(const std::string& logical_fname,
- unique_ptr<SequentialFile>* result,
+ std::unique_ptr<SequentialFile>* result,
  const EnvOptions& options) {
  assert(status().ok());
  result->reset();
@@ -534,7 +534,7 @@ Status AwsEnv::NewSequentialFile(const std::string& logical_fname,
  st = base_env_->NewSequentialFile(fname, result, options);
  }
  } else {
- unique_ptr<S3ReadableFile> file;
+ std::unique_ptr<S3ReadableFile> file;
  if (!st.ok() && has_dest_bucket_) { // read from destination S3
  st = NewS3ReadableFile(GetDestBucketPrefix(), destname(fname), &file);
  }
@@ -574,7 +574,7 @@ Status AwsEnv::NewSequentialFile(const std::string& logical_fname,
 
 // open a file for random reading
 Status AwsEnv::NewRandomAccessFile(const std::string& logical_fname,
- unique_ptr<RandomAccessFile>* result,
+ std::unique_ptr<RandomAccessFile>* result,
  const EnvOptions& options) {
  assert(status().ok());
  result->reset();
@@ -648,7 +648,7 @@ Status AwsEnv::NewRandomAccessFile(const std::string& logical_fname,
  // Only execute this code path if keep_local_sst_files == false. If it's
  // true, we will never use S3ReadableFile to read; we copy the file
  // locally and read using base_env.
- unique_ptr<S3ReadableFile> file;
+ std::unique_ptr<S3ReadableFile> file;
  if (!st.ok() && has_dest_bucket_) {
  st = NewS3ReadableFile(GetDestBucketPrefix(), destname(fname), &file);
  }
@@ -688,7 +688,7 @@ Status AwsEnv::NewRandomAccessFile(const std::string& logical_fname,
 
 // create a new file for writing
 Status AwsEnv::NewWritableFile(const std::string& logical_fname,
- unique_ptr<WritableFile>* result,
+ std::unique_ptr<WritableFile>* result,
  const EnvOptions& options) {
  assert(status().ok());
  result->reset();
@@ -703,7 +703,7 @@ Status AwsEnv::NewWritableFile(const std::string& logical_fname,
  Status s;
 
  if (has_dest_bucket_ && (sstfile || identity || manifest)) {
- unique_ptr<S3WritableFile> f(
+ std::unique_ptr<S3WritableFile> f(
  new S3WritableFile(this, fname, GetDestBucketPrefix(), destname(fname),
  options, cloud_env_options));
  s = f->status();
@@ -755,23 +755,23 @@ class S3Directory : public Directory {
  AwsEnv* env_;
  std::string name_;
  Status status_;
- unique_ptr<Directory> posixDir;
+ std::unique_ptr<Directory> posixDir;
 };
 
 //
 // Returns success only if the directory-bucket exists in the
 // AWS S3 service and the posixEnv local directory exists as well.
 //
 Status AwsEnv::NewDirectory(const std::string& name,
- unique_ptr<Directory>* result) {
+ std::unique_ptr<Directory>* result) {
  assert(status().ok());
  result->reset(nullptr);
 
  Log(InfoLogLevel::DEBUG_LEVEL, info_log_, "[aws] NewDirectory name '%s'",
  name.c_str());
 
  // create new object.
- unique_ptr<S3Directory> d(new S3Directory(this, name));
+ std::unique_ptr<S3Directory> d(new S3Directory(this, name));
 
  // Check if the path exists in local dir
  if (!d->status().ok()) {
@@ -944,7 +944,7 @@ Status AwsEnv::HeadObject(const std::string& bucket_prefix,
 
 Status AwsEnv::NewS3ReadableFile(const std::string& bucket_prefix,
  const std::string& fname,
- unique_ptr<S3ReadableFile>* result) {
+ std::unique_ptr<S3ReadableFile>* result) {
  // First, check if the file exists and also find its size. We use size in
  // S3ReadableFile to make sure we always read the valid ranges of the file
  uint64_t size;
@@ -1785,7 +1785,7 @@ Status AwsEnv::LockFile(const std::string& fname, FileLock** lock) {
 
 Status AwsEnv::UnlockFile(FileLock* lock) { return Status::OK(); }
 
-Status AwsEnv::NewLogger(const std::string& fname, shared_ptr<Logger>* result) {
+Status AwsEnv::NewLogger(const std::string& fname, std::shared_ptr<Logger>* result) {
  return base_env_->NewLogger(fname, result);
 }
 
@@ -1804,7 +1804,7 @@ Status AwsEnv::NewAwsEnv(Env* base_env, const std::string& src_bucket_prefix,
  if (!base_env) {
  base_env = Env::Default();
  }
- unique_ptr<AwsEnv> aenv(
+ std::unique_ptr<AwsEnv> aenv(
  new AwsEnv(base_env, src_bucket_prefix, src_object_prefix,
  src_bucket_region, dest_bucket_prefix, dest_object_prefix,
  dest_bucket_region, cloud_options, info_log));

cloud/aws/aws_env.h

@@ -392,7 +392,7 @@ class AwsEnv : public CloudEnvImpl {
 
  Status NewS3ReadableFile(const std::string& bucket_prefix,
  const std::string& fname,
- unique_ptr<S3ReadableFile>* result);
+ std::unique_ptr<S3ReadableFile>* result);
 
  // Save IDENTITY file to S3. Update dbid registry.
  Status SaveIdentitytoS3(const std::string& localfile,

cloud/aws/aws_file.h

@@ -97,7 +97,7 @@ class S3WritableFile : public WritableFile {
  std::string fname_;
  std::string tmp_file_;
  Status status_;
- unique_ptr<WritableFile> local_file_;
+ std::unique_ptr<WritableFile> local_file_;
  std::string bucket_prefix_;
  std::string cloud_fname_;
  bool is_manifest_;

cloud/aws/aws_log.cc

@@ -85,13 +85,13 @@ Status CloudLogController::Apply(const Slice& in) {
 
  // If this file is not yet open, open it and store it in cache.
  if (iter == cache_fds_.end()) {
- unique_ptr<RandomRWFile> result;
+ std::unique_ptr<RandomRWFile> result;
  st = env_->GetPosixEnv()->NewRandomRWFile(
  pathname, &result, EnvOptions());
 
  if (!st.ok()) {
  // create the file
- unique_ptr<WritableFile> tmp_writable_file;
+ std::unique_ptr<WritableFile> tmp_writable_file;
  env_->GetPosixEnv()->NewWritableFile(pathname, &tmp_writable_file,
  EnvOptions());
  tmp_writable_file.reset();

cloud/cloud_env.cc

@@ -12,7 +12,7 @@
 #include "rocksdb/options.h"
 #include "rocksdb/status.h"
 #include "util/file_reader_writer.h"
-#include "util/filename.h"
+#include "file/filename.h"
 
 namespace rocksdb {
 
@@ -44,14 +44,14 @@ Status CloudEnvImpl::LoadLocalCloudManifest(const std::string& dbname) {
  if (cloud_manifest_) {
  cloud_manifest_.reset();
  }
- unique_ptr<SequentialFile> file;
+ std::unique_ptr<SequentialFile> file;
  auto cloudManifestFile = CloudManifestFile(dbname);
  auto s = GetBaseEnv()->NewSequentialFile(cloudManifestFile, &file, EnvOptions());
  if (!s.ok()) {
  return s;
  }
  return CloudManifest::LoadFromLog(
- unique_ptr<SequentialFileReader>(
+ std::unique_ptr<SequentialFileReader>(
  new SequentialFileReader(std::move(file), cloudManifestFile)),
  &cloud_manifest_);
 }

cloud/cloud_env_impl.h

@@ -94,7 +94,7 @@ class CloudEnvImpl : public CloudEnv {
  void StopPurger();
 
  private:
- unique_ptr<CloudManifest> cloud_manifest_;
+ std::unique_ptr<CloudManifest> cloud_manifest_;
  // This runs only in tests when we want to disable cloud manifest
  // functionality
  bool test_disable_cloud_manifest_{false};

cloud/cloud_env_wrapper.h

@@ -32,7 +32,7 @@ class CloudEnvWrapper : public CloudEnvImpl {
  }
  virtual Status NewSequentialFileCloud(const std::string& bucket_prefix,
  const std::string& fname,
- unique_ptr<SequentialFile>* result,
+ std::unique_ptr<SequentialFile>* result,
  const EnvOptions& options) override {
  return notsup_;
  }
@@ -61,38 +61,38 @@ class CloudEnvWrapper : public CloudEnvImpl {
 
  // Ability to read a file directly from cloud storage
  virtual Status NewSequentialFileCloud(const std::string& fname,
- unique_ptr<SequentialFile>* result,
+ std::unique_ptr<SequentialFile>* result,
  const EnvOptions& options) {
  return notsup_;
  }
 
  // The following text is boilerplate that forwards all methods to base_env
- Status NewSequentialFile(const std::string& f, unique_ptr<SequentialFile>* r,
+ Status NewSequentialFile(const std::string& f, std::unique_ptr<SequentialFile>* r,
  const EnvOptions& options) override {
  return base_env_->NewSequentialFile(f, r, options);
  }
  Status NewRandomAccessFile(const std::string& f,
- unique_ptr<RandomAccessFile>* r,
+ std::unique_ptr<RandomAccessFile>* r,
  const EnvOptions& options) override {
  return base_env_->NewRandomAccessFile(f, r, options);
  }
- Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
+ Status NewWritableFile(const std::string& f, std::unique_ptr<WritableFile>* r,
  const EnvOptions& options) override {
  return base_env_->NewWritableFile(f, r, options);
  }
  Status ReuseWritableFile(const std::string& fname,
  const std::string& old_fname,
- unique_ptr<WritableFile>* r,
+ std::unique_ptr<WritableFile>* r,
  const EnvOptions& options) override {
  return base_env_->ReuseWritableFile(fname, old_fname, r, options);
  }
  Status NewRandomRWFile(const std::string& fname,
- unique_ptr<RandomRWFile>* result,
+ std::unique_ptr<RandomRWFile>* result,
  const EnvOptions& options) override {
  return base_env_->NewRandomRWFile(fname, result, options);
  }
  virtual Status NewDirectory(const std::string& name,
- unique_ptr<Directory>* result) override {
+ std::unique_ptr<Directory>* result) override {
  return base_env_->NewDirectory(name, result);
  }
  Status FileExists(const std::string& f) override {
@@ -162,7 +162,7 @@ class CloudEnvWrapper : public CloudEnvImpl {
  return base_env_->GetTestDirectory(path);
  }
  virtual Status NewLogger(const std::string& fname,
- shared_ptr<Logger>* result) override {
+ std::shared_ptr<Logger>* result) override {
  return base_env_->NewLogger(fname, result);
  }
  uint64_t NowMicros() override { return base_env_->NowMicros(); }

cloud/cloud_manifest.cc

@@ -40,7 +40,7 @@ Status CloudManifest::LoadFromLog(std::unique_ptr<SequentialFileReader> log,
  CorruptionReporter reporter;
  reporter.status = &status;
  log::Reader reader(nullptr, std::move(log), &reporter, true /* checksum */,
- 0 /* log_num */, false /* retry_after_eof */);
+ 0 /* log_num */);
  Slice record;
  std::string scratch;
  bool headerRead = false;

cloud/cloud_manifest_test.cc

@@ -3,7 +3,7 @@
 #include "cloud/cloud_manifest.h"
 #include <rocksdb/env.h>
 #include "util/file_reader_writer.h"
-#include "util/testharness.h"
+#include "test_util/testharness.h"
 
 namespace rocksdb {
 

cloud/db_cloud_impl.cc

@@ -17,9 +17,9 @@
 #include "rocksdb/persistent_cache.h"
 #include "rocksdb/status.h"
 #include "rocksdb/table.h"
-#include "util/auto_roll_logger.h"
+#include "logging/auto_roll_logger.h"
+#include "file/file_util.h"
 #include "util/file_reader_writer.h"
-#include "util/file_util.h"
 #include "util/xxhash.h"
 
 namespace rocksdb {
@@ -60,7 +60,7 @@ Status writeCloudManifest(Env* local_env, CloudManifest* manifest,
  std::unique_ptr<WritableFile> file;
  Status s = local_env->NewWritableFile(tmp_fname, &file, EnvOptions());
  if (s.ok()) {
- s = manifest->WriteToLog(unique_ptr<WritableFileWriter>(
+ s = manifest->WriteToLog(std::unique_ptr<WritableFileWriter>(
  new WritableFileWriter(std::move(file), tmp_fname, EnvOptions())));
  }
  if (s.ok()) {
@@ -179,7 +179,7 @@ Status DBCloud::Open(const Options& opt, const std::string& local_dbname,
  if (st.ok()) {
  tableopt->persistent_cache = pcache;
  Log(InfoLogLevel::INFO_LEVEL, options.info_log,
- "Created persistent cache %s with size %ld GB",
+ "Created persistent cache %s with size %" PRIu64 "GB",
  persistent_cache_path.c_str(), persistent_cache_size_gb);
  } else {
  Log(InfoLogLevel::INFO_LEVEL, options.info_log,
@@ -313,7 +313,7 @@ Status DBCloudImpl::CreateNewIdentityFile(CloudEnv* cenv,
  Env* env = cenv->GetBaseEnv();
  Status st;
  {
- unique_ptr<WritableFile> destfile;
+ std::unique_ptr<WritableFile> destfile;
  st = env->NewWritableFile(tmp_identity_path, &destfile, soptions);
  if (!st.ok()) {
  Log(InfoLogLevel::ERROR_LEVEL, options.info_log,
@@ -339,7 +339,7 @@ Status DBCloudImpl::CreateNewIdentityFile(CloudEnv* cenv,
  if (!st.ok()) {
  Log(InfoLogLevel::ERROR_LEVEL, options.info_log,
  "[db_cloud_impl] Unable to rename newly created IDENTITY.tmp "
- " to IDENTITY. %S",
+ " to IDENTITY. %s",
  st.ToString().c_str());
  return st;
  }
@@ -863,7 +863,7 @@ Status DBCloudImpl::SanitizeDirectory(const Options& options,
  // remap the filename appropriately, this is just to fool the underyling
  // RocksDB)
  {
- unique_ptr<WritableFile> destfile;
+ std::unique_ptr<WritableFile> destfile;
  st = env->NewWritableFile(CurrentFileName(local_name), &destfile, soptions);
  if (!st.ok()) {
  Log(InfoLogLevel::ERROR_LEVEL, options.info_log,
@@ -952,7 +952,7 @@ Status DBCloudImpl::FetchCloudManifest(CloudEnv* cenv, const Options& options,
  local_dbname.c_str());
 
  // No cloud manifest, create an empty one
- unique_ptr<CloudManifest> manifest;
+ std::unique_ptr<CloudManifest> manifest;
  CloudManifest::CreateForEmptyDatabase("", &manifest);
  return writeCloudManifest(cenv->GetBaseEnv(), manifest.get(), cloudmanifest);
 }