[jit] jit support property.proto (#44337)

* add property.proto, can compiled * property get and deserilize * support get float * format code * format code * add unittest * add more set method * fix grammar error * Update paddle/fluid/jit/property.h Co-authored-by: Aurelius84 <zhangliujie@baidu.com> * Update paddle/fluid/jit/property.cc Co-authored-by: Aurelius84 <zhangliujie@baidu.com> * Update paddle/fluid/jit/property.cc Co-authored-by: Aurelius84 <zhangliujie@baidu.com> * Update paddle/fluid/jit/property.cc Co-authored-by: Aurelius84 <zhangliujie@baidu.com> * fix comment * fix error throw * fix property save unit test * fix error info * fix copyright and header import * reorder jit property tensor datatype Co-authored-by: Aurelius84 <zhangliujie@baidu.com>
PaddlePaddle · Jul 21, 2022 · 0aa344f · 0aa344f
1 parent 7ab0e33
commit 0aa344f
Show file tree

Hide file tree

Showing 10 changed files with 612 additions and 1 deletion.
diff --git a/paddle/fluid/jit/CMakeLists.txt b/paddle/fluid/jit/CMakeLists.txt
@@ -52,3 +52,10 @@ if(WITH_TESTING AND NOT WIN32)
     DEPS ${JIT_DEPS})
   add_dependencies(layer_test jit_download_program)
 endif()
+
+proto_library(paddle_jit_property_proto SRCS property.proto)
+
+cc_library(
+  jit_property
+  SRCS property.cc
+  DEPS paddle_jit_property_proto)
diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc
@@ -0,0 +1,173 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/jit/property.h"
+#include "glog/logging.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/errors.h"
+
+namespace paddle {
+namespace jit {
+
+int Property::Size() const { return property_.entrys_size(); }
+
+void Property::SetFloat(const float &f) {
+  auto type = proto::ValueProto::FLOAT;
+  auto entry = property_.add_entrys();
+  entry->set_type(type);
+  entry->set_f(f);
+  VLOG(3) << "Property: set_float " << f;
+}
+
+void Property::SetFloat(const std::string &name, const float &f) {
+  auto type = proto::ValueProto::FLOAT;
+  auto entry = property_.add_entrys();
+  entry->set_name(name);
+  entry->set_type(type);
+  entry->set_f(f);
+  VLOG(3) << "Property: set_float " << f << " name: " << name;
+}
+
+float Property::GetFloat(const std::string &name) const {
+  for (int i = 0; i < Size(); i++) {
+    auto e = property_.entrys(i);
+    if (e.has_name() && e.name() == name) {
+      return e.f();
+    }
+  }
+
+  PADDLE_THROW(phi::errors::NotFound(
+      "JIT::Property GetFloat: name: %s not found", name));
+  return 0;
+}
+
+float Property::GetFloat(const int &idx) const {
+  PADDLE_ENFORCE_EQ(
+      idx < Size() && idx >= 0,
+      true,
+      phi::errors::OutOfRange(
+          "JIT::Property GetFloat: idx=%d out of range %d", idx, Size()));
+
+  auto e = property_.entrys(idx);
+  if (e.has_f()) {
+    return e.f();
+  }
+
+  PADDLE_THROW(phi::errors::InvalidArgument(
+      "JIT::Property GetFloat: input idx (%d) element is not a float.", idx));
+  return 0;
+}
+
+void Property::SetFloats(const std::vector<float> &v) {
+  auto type = proto::ValueProto::FLOATS;
+  auto entry = property_.add_entrys();
+  entry->set_type(type);
+  for (auto i : v) {
+    entry->add_floats(i);
+  }
+  VLOG(3) << "Property: set_floats  with length: " << v.size();
+}
+
+void Property::SetFloats(const std::string &name, const std::vector<float> &v) {
+  auto type = proto::ValueProto::FLOATS;
+  auto entry = property_.add_entrys();
+  entry->set_name(name);
+  entry->set_type(type);
+  for (auto i : v) {
+    entry->add_floats(i);
+  }
+  VLOG(3) << "Property: set_floats  with length " << v.size()
+          << " for name: " << name;
+}
+
+void Property::SetInt64(const int64_t &i) {
+  auto type = proto::ValueProto::INT;
+  auto entry = property_.add_entrys();
+  entry->set_type(type);
+  entry->set_i(i);
+  VLOG(3) << "Property: set_int " << i;
+}
+
+void Property::SetInt64(const std::string &name, const int64_t &i) {
+  auto type = proto::ValueProto::INT;
+  auto entry = property_.add_entrys();
+  entry->set_name(name);
+  entry->set_type(type);
+  entry->set_i(i);
+  VLOG(3) << "Property: set_int " << i << " name: " << name;
+}
+
+void Property::SetInt64s(const std::vector<int64_t> &v) {
+  auto type = proto::ValueProto::INTS;
+  auto entry = property_.add_entrys();
+  entry->set_type(type);
+  for (auto e : v) {
+    entry->add_ints(e);
+  }
+  VLOG(3) << "Property: set_ints " << v.size();
+}
+
+void Property::SetInt64s(const std::string &name,
+                         const std::vector<int64_t> &v) {
+  auto type = proto::ValueProto::INTS;
+  auto entry = property_.add_entrys();
+  entry->set_name(name);
+  entry->set_type(type);
+  for (auto i : v) {
+    entry->add_ints(i);
+  }
+  VLOG(3) << "Property: set_ints " << v[0] << " name: " << name;
+}
+
+void Property::SetString(const std::string &s) {
+  auto type = proto::ValueProto::STRING;
+  auto entry = property_.add_entrys();
+  entry->set_type(type);
+  entry->set_s(s);
+  VLOG(3) << "Property: set_string with value : " << s;
+}
+
+void Property::SetString(const std::string &name, const std::string &s) {
+  auto type = proto::ValueProto::STRING;
+  auto entry = property_.add_entrys();
+  entry->set_name(name);
+  entry->set_type(type);
+  entry->set_s(s);
+  VLOG(3) << "Property: set_string " << s << " name: " << name;
+}
+
+void Property::SetStrings(const std::vector<std::string> &v) {
+  auto type = proto::ValueProto::STRINGS;
+  auto entry = property_.add_entrys();
+  entry->set_type(type);
+  for (auto i : v) {
+    entry->add_strings(i);
+  }
+  VLOG(3) << "Property: set_strings " << v.size();
+}
+
+void Property::SetStrings(const std::string &name,
+                          const std::vector<std::string> &v) {
+  auto type = proto::ValueProto::STRINGS;
+  auto entry = property_.add_entrys();
+  entry->set_name(name);
+  entry->set_type(type);
+  for (auto i : v) {
+    entry->add_strings(i);
+  }
+  VLOG(3) << "Property: set_strings " << v[0] << " name: " << name;
+}
+
+}  // namespace jit
+}  // namespace paddle
diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h
@@ -0,0 +1,93 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <atomic>
+#include <string>
+#include <vector>
+
+#include "paddle/fluid/jit/property.pb.h"
+
+namespace paddle {
+namespace jit {
+
+class Property {
+ public:
+  Property() {}
+
+  // Explicitly implement the copy constructor for auto parallel
+  explicit Property(const Property &other)
+      : property_(other.property_), original_id_(other.original_id_) {}
+
+  Property &operator=(const Property &other) {
+    property_ = other.property_;
+    original_id_ = other.original_id_;
+    return *this;
+  }
+
+  proto::PropertyVals *Proto() { return &property_; }
+
+  const proto::PropertyVals *Proto() const { return &property_; }
+
+  int Size() const;
+
+  void SetFloat(const float &f);
+  void SetFloat(const std::string &name, const float &f);
+
+  void SetFloats(const std::vector<float> &v);
+  void SetFloats(const std::string &name, const std::vector<float> &v);
+
+  float GetFloat(const std::string &name) const;
+  float GetFloat(const int &idx) const;
+
+  void SetInt64(const int64_t &i);
+  void SetInt64(const std::string &name, const int64_t &i);
+
+  void SetInt64s(const std::vector<int64_t> &v);
+  void SetInt64s(const std::string &name, const std::vector<int64_t> &v);
+
+  void SetString(const std::string &s);
+  void SetString(const std::string &name, const std::string &s);
+
+  void SetStrings(const std::vector<std::string> &v);
+  void SetStrings(const std::string &name, const std::vector<std::string> &v);
+
+  // The Id() and OriginalId() are only used for auto parallel.
+  uint64_t Id() const { return id_; }
+  uint64_t OriginalId() const { return original_id_; }
+  void SetOriginalId(uint64_t original_id) { original_id_ = original_id; }
+
+ private:
+  proto::PropertyVals property_;
+
+  // This thread-safe implementation seems to be redudent since the neural
+  // networks are usually constructed in a single thread.
+  static uint64_t GenerateId() {
+    static std::atomic<std::uint64_t> uid{0};
+    return ++uid;
+  }
+
+  // Note: the id_ is unique for all Property (only for auto parallel).
+  uint64_t id_ = GenerateId();
+  // Note: the orignal_id_ is used for referring to the original Property
+  // that the current Property is built from (only for auto parallel).
+  // The default original_id_ is same as the id_, which means the
+  // current Property is not built from the other one.
+  uint64_t original_id_ = id_;
+};
+
+}  // namespace jit
+}  // namespace paddle