diff --git a/oneflow/api/cpp/framework.h b/oneflow/api/cpp/framework.h
index efe1ac38b9a..214ac3d2f5e 100644
--- a/oneflow/api/cpp/framework.h
+++ b/oneflow/api/cpp/framework.h
@@ -21,5 +21,6 @@ limitations under the License.
 #include "framework/shape.h"
 #include "framework/dtype.h"
 #include "framework/tensor.h"
+#include "framework/ivalue.h"
 
-#endif  // !ONEFLOW_API_CPP_FRAMEWORK_H_
+#endif  // ONEFLOW_API_CPP_FRAMEWORK_H_
diff --git a/oneflow/api/cpp/framework/dtype.h b/oneflow/api/cpp/framework/dtype.h
index 19744eae174..217a70d928e 100644
--- a/oneflow/api/cpp/framework/dtype.h
+++ b/oneflow/api/cpp/framework/dtype.h
@@ -40,4 +40,4 @@ enum class DType {
 
 }  // namespace oneflow_api
 
-#endif  // !ONEFLOW_API_CPP_FRAMEWORK_DTYPE_H_
+#endif  // ONEFLOW_API_CPP_FRAMEWORK_DTYPE_H_
diff --git a/oneflow/api/cpp/framework/ivalue.cpp b/oneflow/api/cpp/framework/ivalue.cpp
new file mode 100644
index 00000000000..638bee4f124
--- /dev/null
+++ b/oneflow/api/cpp/framework/ivalue.cpp
@@ -0,0 +1,53 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#include "oneflow/api/cpp/framework/ivalue.h"
+#include <glog/logging.h>
+
+namespace oneflow_api {
+
+namespace of = oneflow;
+
+std::ostream& operator<<(std::ostream& os, const IValue::Tag& tag) {
+  os << static_cast<int>(tag);
+  return os;
+}
+
+int64_t IValue::ToInt() const {
+  CHECK_EQ(tag_, Tag::kInt) << "Current value is not an int.";
+  return payload_.i.v_int;
+}
+
+double IValue::ToDouble() const {
+  CHECK_EQ(tag_, Tag::kDouble) << "Current value is not a double.";
+  return payload_.i.v_double;
+}
+
+bool IValue::ToBool() const {
+  CHECK_EQ(tag_, Tag::kBool) << "Current value is not a bool.";
+  return payload_.i.v_bool;
+}
+
+const Tensor& IValue::ToTensor() const {
+  CHECK_EQ(tag_, Tag::kTensor) << "Current value is not a tensor.";
+  return payload_.v_tensor;
+}
+
+const std::vector<Tensor>& IValue::ToTensorVector() const {
+  CHECK_EQ(tag_, Tag::kTensorVector) << "Current value is not a vector of tensor.";
+  return payload_.v_tensor_vector;
+}
+
+}  // namespace oneflow_api
diff --git a/oneflow/api/cpp/framework/ivalue.h b/oneflow/api/cpp/framework/ivalue.h
new file mode 100644
index 00000000000..fad26be6aff
--- /dev/null
+++ b/oneflow/api/cpp/framework/ivalue.h
@@ -0,0 +1,149 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#ifndef ONEFLOW_API_CPP_FRAMEWORK_IVALUE_H_
+#define ONEFLOW_API_CPP_FRAMEWORK_IVALUE_H_
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+#include "tensor.h"
+
+namespace oneflow_api {
+
+class IValue {
+ public:
+  IValue() : tag_(IValue::Tag::kNone) {}
+  explicit IValue(int value) : tag_(IValue::Tag::kInt) { payload_.i.v_int = value; }
+
+  explicit IValue(int64_t value) : tag_(IValue::Tag::kInt) { payload_.i.v_int = value; }
+
+  explicit IValue(double value) : tag_(IValue::Tag::kDouble) { payload_.i.v_double = value; }
+
+  explicit IValue(bool value) : tag_(IValue::Tag::kBool) { payload_.i.v_bool = value; }
+
+  IValue(const Tensor& value) : tag_(IValue::Tag::kTensor) {  // NOLINT
+    new (&payload_.v_tensor) Tensor(value);
+  }
+
+  IValue(Tensor&& value) : tag_(IValue::Tag::kTensor) {  // NOLINT
+    new (&payload_.v_tensor) Tensor(std::move(value));
+  }
+
+  IValue(const std::vector<Tensor>& value) : tag_(IValue::Tag::kTensorVector) {  // NOLINT
+    new (&payload_.v_tensor_vector) std::vector<Tensor>(value);
+  }
+
+  IValue(std::vector<Tensor>&& value) : tag_(IValue::Tag::kTensorVector) {  // NOLINT
+    new (&payload_.v_tensor_vector) std::vector<Tensor>(std::move(value));
+  }
+
+  IValue(const IValue& value) : tag_(value.tag_) {
+    if (IsTensor()) {
+      new (&payload_.v_tensor) Tensor(value.payload_.v_tensor);
+    } else if (IsTensorVector()) {
+      new (&payload_.v_tensor_vector) std::vector<Tensor>(value.payload_.v_tensor_vector);
+    } else {
+      payload_.i = value.payload_.i;
+    }
+  }
+
+  IValue(IValue&& value) noexcept : tag_(value.tag_) { MoveFrom(std::move(value)); }
+
+  IValue& operator=(const IValue& value) {
+    if (&value == this) { return *this; }
+    this->tag_ = value.tag_;
+    *this = IValue(value);
+    return *this;
+  }
+
+  IValue& operator=(IValue&& value) noexcept {
+    if (&value == this) { return *this; }
+    Destory();
+    this->tag_ = value.tag_;
+    MoveFrom(std::move(value));
+    return *this;
+  }
+
+  ~IValue() { Destory(); }
+
+  bool IsNone() const { return tag_ == Tag::kNone; }
+
+  bool IsInt() const { return tag_ == Tag::kInt; }
+
+  bool IsDouble() const { return tag_ == Tag::kDouble; }
+
+  bool IsBool() const { return tag_ == Tag::kBool; }
+
+  bool IsTensor() const { return tag_ == Tag::kTensor; }
+
+  bool IsTensorVector() const { return tag_ == Tag::kTensorVector; }
+
+  int64_t ToInt() const;
+  double ToDouble() const;
+  bool ToBool() const;
+  const Tensor& ToTensor() const;
+  const std::vector<Tensor>& ToTensorVector() const;
+
+ private:
+  enum class Tag { kNone = 0, kInt = 1, kDouble = 2, kBool = 3, kTensor = 4, kTensorVector = 5 };
+  friend std::ostream& operator<<(std::ostream&, const Tag&);
+
+  union Payload {  // NOLINT
+    union InternalPayload {
+      InternalPayload() : v_int(0) {}
+
+      int64_t v_int;
+      double v_double;
+      bool v_bool;
+    } i;
+
+    Tensor v_tensor;
+    std::vector<Tensor> v_tensor_vector;
+
+    Payload() : i() {}
+    ~Payload() {}
+  };
+
+  Payload payload_;
+  Tag tag_;
+
+  inline void Destory() {
+    if (IsTensor()) { payload_.v_tensor.~Tensor(); }
+    if (IsTensorVector()) { payload_.v_tensor_vector.~vector(); }
+  }
+
+  inline void MoveFrom(IValue&& value) {
+    if (IsTensor()) {
+      new (&payload_.v_tensor) Tensor(std::move(value.payload_.v_tensor));
+    } else if (IsTensorVector()) {
+      new (&payload_.v_tensor_vector)
+          std::vector<Tensor>(std::move(value.payload_.v_tensor_vector));
+    } else {
+      payload_.i = value.payload_.i;
+    }
+    value.ClearToNone();
+  }
+
+  inline void ClearToNone() {
+    Destory();
+    payload_.i.v_int = 0;
+    tag_ = Tag::kNone;
+  }
+};
+
+}  // namespace oneflow_api
+
+#endif  // ONEFLOW_API_CPP_FRAMEWORK_IVALUE_H_
diff --git a/oneflow/api/cpp/framework/shape.cpp b/oneflow/api/cpp/framework/shape.cpp
index da713cf7362..e5365c9ce4a 100644
--- a/oneflow/api/cpp/framework/shape.cpp
+++ b/oneflow/api/cpp/framework/shape.cpp
@@ -60,4 +60,9 @@ int64_t Shape::Count(int64_t begin_axis, int64_t end_axis) const {
 
 int64_t Shape::Count(int64_t begin_axis) const { return shape_->Count(begin_axis); }
 
+std::ostream& operator<<(std::ostream& os, const Shape& shape) {
+  os << shape.shape_->DebugStr();
+  return os;
+}
+
 }  // namespace oneflow_api
diff --git a/oneflow/api/cpp/framework/shape.h b/oneflow/api/cpp/framework/shape.h
index ed2078bdf20..7465444ded2 100644
--- a/oneflow/api/cpp/framework/shape.h
+++ b/oneflow/api/cpp/framework/shape.h
@@ -50,7 +50,9 @@ class Shape final {
 
  private:
   std::shared_ptr<oneflow::Shape> shape_ = nullptr;
+
+  friend std::ostream& operator<<(std::ostream&, const Shape&);
 };
 }  // namespace oneflow_api
 
-#endif  // !ONEFLOW_API_CPP_FRAMEWORK_SHAPE_H_
+#endif  // ONEFLOW_API_CPP_FRAMEWORK_SHAPE_H_
diff --git a/oneflow/api/cpp/framework/tensor.cpp b/oneflow/api/cpp/framework/tensor.cpp
index fe62b2dfc61..ab01924b280 100644
--- a/oneflow/api/cpp/framework/tensor.cpp
+++ b/oneflow/api/cpp/framework/tensor.cpp
@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 #include "oneflow/api/cpp/framework/tensor.h"
-#include <memory>
 #include "oneflow/api/cpp/framework/device.h"
 #include "oneflow/api/cpp/framework/dtype.h"
 #include "oneflow/api/cpp/framework/shape.h"
@@ -42,17 +41,31 @@ Tensor::Tensor(const Shape& shape, const Device& device, const DType& dtype) {
 }
 Tensor::Tensor(const std::shared_ptr<oneflow::one::Tensor>& tensor) : tensor_(tensor) {}
 
-const Shape Tensor::shape() const {
+Tensor::Tensor(const Tensor& tensor) : tensor_(tensor.tensor_) {}
+Tensor::Tensor(Tensor&& tensor) noexcept : tensor_(std::move(tensor.tensor_)) {}
+
+Tensor& Tensor::operator=(const Tensor& tensor) {
+  if (&tensor == this) { return *this; }
+  tensor_ = tensor.tensor_;
+  return *this;
+}
+Tensor& Tensor::operator=(Tensor&& tensor) noexcept {
+  if (&tensor == this) { return *this; }
+  tensor_ = std::move(tensor.tensor_);
+  return *this;
+}
+
+Shape Tensor::shape() const {
   const auto shape_ = tensor_->shape();
   return Shape(std::vector<int64_t>(shape_->dim_vec().begin(), shape_->dim_vec().end()));
 }
 
-const Device Tensor::device() const {
+Device Tensor::device() const {
   const auto device_ = tensor_->device().GetOrThrow();
   return Device(device_->type(), device_->device_id());
 }
 
-const DType Tensor::dtype() const { return static_cast<DType>(tensor_->dtype()->data_type()); }
+DType Tensor::dtype() const { return static_cast<DType>(tensor_->dtype()->data_type()); }
 
 void Tensor::zeros_() {
   std::shared_ptr<of::one::MirroredTensor> local_tensor =
diff --git a/oneflow/api/cpp/framework/tensor.h b/oneflow/api/cpp/framework/tensor.h
index c25cc494325..2a99bca8c0f 100644
--- a/oneflow/api/cpp/framework/tensor.h
+++ b/oneflow/api/cpp/framework/tensor.h
@@ -37,9 +37,18 @@ class Tensor final {
   explicit Tensor(const Shape& shape = Shape(), const Device& device = Device("cpu"),
                   const DType& dtype = DType::kFloat);
   explicit Tensor(const std::shared_ptr<oneflow::one::Tensor>& tensor);
-  [[nodiscard]] const Shape shape() const;
-  [[nodiscard]] const Device device() const;
-  [[nodiscard]] const DType dtype() const;
+
+  Tensor(const Tensor& tensor);
+  Tensor(Tensor&& tensor) noexcept;
+
+  ~Tensor() = default;
+
+  Tensor& operator=(const Tensor& tensor);
+  Tensor& operator=(Tensor&& tensor) noexcept;
+
+  [[nodiscard]] Shape shape() const;
+  [[nodiscard]] Device device() const;
+  [[nodiscard]] DType dtype() const;
 
   void zeros_();
 
@@ -58,4 +67,4 @@ class Tensor final {
 
 }  // namespace oneflow_api
 
-#endif  // !ONEFLOW_API_CPP_FRAMEWORK_TENSOR_H_
+#endif  // ONEFLOW_API_CPP_FRAMEWORK_TENSOR_H_
diff --git a/oneflow/api/cpp/nn.h b/oneflow/api/cpp/nn.h
index ebff7bd7c5a..4dbfbef6d9d 100644
--- a/oneflow/api/cpp/nn.h
+++ b/oneflow/api/cpp/nn.h
@@ -19,4 +19,4 @@ limitations under the License.
 
 #include "nn/functional/activation.h"
 
-#endif  // !ONEFLOW_API_CPP_NN_H_
+#endif  // ONEFLOW_API_CPP_NN_H_
diff --git a/oneflow/api/cpp/nn/functional/activation.h b/oneflow/api/cpp/nn/functional/activation.h
index f22cde74645..dc334eb034a 100644
--- a/oneflow/api/cpp/nn/functional/activation.h
+++ b/oneflow/api/cpp/nn/functional/activation.h
@@ -27,4 +27,4 @@ Tensor relu(const Tensor& tensor);
 
 }  // namespace oneflow_api
 
-#endif  // !ONEFLOW_API_CPP_NN_FUNCTIONAL_ACTIVATION_H_
+#endif  // ONEFLOW_API_CPP_NN_FUNCTIONAL_ACTIVATION_H_
diff --git a/oneflow/api/cpp/tests/api_test.cpp b/oneflow/api/cpp/tests/api_test.cpp
index 6888b95487d..88619f3617b 100644
--- a/oneflow/api/cpp/tests/api_test.cpp
+++ b/oneflow/api/cpp/tests/api_test.cpp
@@ -15,7 +15,6 @@ limitations under the License.
 */
 
 #include "oneflow/api/cpp/tests/api_test.h"
-#include <cstdint>
 #include <random>
 
 namespace oneflow_api {
diff --git a/oneflow/api/cpp/tests/ivalue_test.cpp b/oneflow/api/cpp/tests/ivalue_test.cpp
new file mode 100644
index 00000000000..ff4d054061f
--- /dev/null
+++ b/oneflow/api/cpp/tests/ivalue_test.cpp
@@ -0,0 +1,132 @@
+/*
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <random>
+#include <gtest/gtest.h>
+#include "oneflow/api/cpp/framework/dtype.h"
+#include "oneflow/api/cpp/framework/ivalue.h"
+#include "oneflow/api/cpp/tests/api_test.h"
+
+namespace oneflow_api {
+
+namespace {
+
+std::mt19937 rng(std::random_device{}());
+
+}
+
+TEST(Api, ivalue) {
+  std::uniform_real_distribution<> dist(-100, 100);
+  std::uniform_int_distribution<> dist_bool(0, 1);
+
+  const auto v_int = static_cast<int>(dist(rng));
+  ASSERT_EQ(IValue(v_int).ToInt(), v_int);
+
+  const auto v_int64 = static_cast<int64_t>(dist(rng));
+  ASSERT_EQ(IValue(v_int64).ToInt(), v_int64);
+
+  const auto v_float = static_cast<float>(dist(rng));
+  ASSERT_EQ(IValue(v_float).ToDouble(), v_float);
+
+  const auto v_double = static_cast<double>(dist(rng));
+  ASSERT_EQ(IValue(v_double).ToDouble(), v_double);
+
+  const auto v_bool = static_cast<bool>(dist_bool(rng));
+  ASSERT_EQ(IValue(v_bool).ToBool(), v_bool);
+}
+
+TEST(Api, ivalue_tensor) {
+  EnvScope scope;
+
+  const auto device = Device("cpu");
+  const auto shape = RandomShape();
+  const auto dtype = DType::kDouble;
+
+  const IValue i_tensor(Tensor(shape, device, dtype));
+  const auto& tensor = i_tensor.ToTensor();
+
+  ASSERT_EQ(tensor.shape(), shape);
+  ASSERT_EQ(tensor.device(), device);
+  ASSERT_EQ(tensor.dtype(), dtype);
+}
+
+TEST(Api, ivalue_tensor_vector) {
+  EnvScope scope;
+
+  const auto device = Device("cpu");
+
+  const std::vector<Tensor> v_tensor_vector{Tensor(RandomShape(), device, DType::kDouble),
+                                            Tensor(RandomShape(), device, DType::kFloat)};
+  const auto i_tensor = IValue(v_tensor_vector);
+  const auto& tensor_vector = i_tensor.ToTensorVector();
+
+  ASSERT_EQ(v_tensor_vector.size(), tensor_vector.size());
+
+  for (size_t i = 0; i < tensor_vector.size(); ++i) {
+    ASSERT_EQ(v_tensor_vector[i].device(), tensor_vector[i].device());
+    ASSERT_EQ(v_tensor_vector[i].shape(), tensor_vector[i].shape());
+    ASSERT_EQ(v_tensor_vector[i].dtype(), tensor_vector[i].dtype());
+  }
+}
+
+TEST(Api, ivalue_copy) {
+  EnvScope scope;
+
+  const auto device = Device("cpu");
+  const auto shape = RandomShape();
+  const auto dtype = DType::kDouble;
+
+  const IValue i_tensor(Tensor(shape, device, dtype));
+  const auto i_tensor_a = i_tensor;  // NOLINT
+
+  ASSERT_EQ(i_tensor_a.ToTensor().shape(), shape);
+  ASSERT_EQ(i_tensor_a.ToTensor().device(), device);
+  ASSERT_EQ(i_tensor_a.ToTensor().dtype(), dtype);
+
+  IValue i_tensor_b;
+  i_tensor_b = i_tensor;
+
+  ASSERT_EQ(i_tensor_b.ToTensor().shape(), shape);
+  ASSERT_EQ(i_tensor_b.ToTensor().device(), device);
+  ASSERT_EQ(i_tensor_b.ToTensor().dtype(), dtype);
+}
+
+TEST(Api, ivalue_move) {
+  EnvScope scope;
+
+  const auto device = Device("cpu");
+  const auto shape = RandomShape();
+  const auto dtype = DType::kDouble;
+
+  IValue i_tensor_a = IValue(Tensor(shape, device, dtype));
+  IValue i_tensor_b = IValue(Tensor(shape, device, dtype));
+
+  IValue i_tensor_c = std::move(i_tensor_a);
+  ASSERT_EQ(i_tensor_c.ToTensor().shape(), shape);
+  ASSERT_EQ(i_tensor_c.ToTensor().device(), device);
+  ASSERT_EQ(i_tensor_c.ToTensor().dtype(), dtype);
+
+  IValue i_tensor_d;
+  i_tensor_d = std::move(i_tensor_b);
+  ASSERT_EQ(i_tensor_d.ToTensor().shape(), shape);
+  ASSERT_EQ(i_tensor_d.ToTensor().device(), device);
+  ASSERT_EQ(i_tensor_d.ToTensor().dtype(), dtype);
+
+  ASSERT_EQ(i_tensor_a.IsNone(), true);
+  ASSERT_EQ(i_tensor_b.IsNone(), true);
+}
+
+}  // namespace oneflow_api