From 114869e5133c76a7b14f7a41da93bc2be7bf3bdc Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Thu, 14 Jul 2022 06:58:53 +0000 Subject: [PATCH 01/18] add property.proto, can compiled --- paddle/fluid/jit/CMakeLists.txt | 8 ++ paddle/fluid/jit/property.cc | 21 +++++ paddle/fluid/jit/property.h | 119 ++++++++++++++++++++++++++ paddle/fluid/jit/property.proto | 144 ++++++++++++++++++++++++++++++++ paddle/fluid/jit/serializer.cc | 1 + paddle/fluid/jit/serializer.h | 3 + paddle/fluid/pybind/protobuf.cc | 54 ++++++++++++ paddle/fluid/pybind/protobuf.h | 1 + 8 files changed, 351 insertions(+) create mode 100644 paddle/fluid/jit/property.cc create mode 100644 paddle/fluid/jit/property.h create mode 100644 paddle/fluid/jit/property.proto diff --git a/paddle/fluid/jit/CMakeLists.txt b/paddle/fluid/jit/CMakeLists.txt index 75483ac6544f4..0c3e6043bbdd2 100644 --- a/paddle/fluid/jit/CMakeLists.txt +++ b/paddle/fluid/jit/CMakeLists.txt @@ -53,3 +53,11 @@ if(WITH_TESTING DEPS ${JIT_DEPS}) add_dependencies(layer_test jit_download_program) endif() + + +proto_library(paddle_jit_property_proto SRCS property.proto) + +cc_library( + property + SRCS property.cc + DEPS paddle_jit_property_proto) \ No newline at end of file diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc new file mode 100644 index 0000000000000..51f2fd3de0d42 --- /dev/null +++ b/paddle/fluid/jit/property.cc @@ -0,0 +1,21 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/jit/property.h" + +namespace paddle { +namespace jit { + +} // namespace jit +} // namespace paddle diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h new file mode 100644 index 0000000000000..3d68f42298a39 --- /dev/null +++ b/paddle/fluid/jit/property.h @@ -0,0 +1,119 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "glog/logging.h" +#include "paddle/fluid/jit/property.pb.h" + +namespace paddle { +namespace jit { + +// convert between std::vector and protobuf repeated. +template +inline std::vector RepeatedToVector( + const google::protobuf::RepeatedField &repeated_field) { + std::vector ret; + ret.reserve(repeated_field.size()); + std::copy( + repeated_field.begin(), repeated_field.end(), std::back_inserter(ret)); + return ret; +} + +template +inline void VectorToRepeated(const std::vector &vec, + RepeatedField *repeated_field) { + repeated_field->Clear(); + repeated_field->Reserve(vec.size()); + for (const auto &elem : vec) { + *repeated_field->Add() = elem; + } +} + +// Specialize vector. +template +inline void VectorToRepeated(const std::vector &vec, + RepeatedField *repeated_field) { + repeated_field->Clear(); + repeated_field->Reserve(vec.size()); + for (auto elem : vec) { + *repeated_field->Add() = elem; + } +} + + +class Property { + public: + explicit Property() {} + + // Explicitly implement the copy constructor for auto parallel + Property(const Property &other) + : property_(other.property_), + original_id_(other.original_id_) {} + + Property &operator=(const Property &other) { + property_ = other.property_; + original_id_ = other.original_id_; + return *this; + } + + proto::PropertyVals *Proto() { return &property_; } + + const proto::PropertyVals *Proto() const { return &property_; } + + bool SetFloat(const float& f, std::string name=""); + bool SetFloats(const std::vector& v, std::string name=""); + + bool SetInt64(const int64_t& f, std::string name=""); + bool SetInt64s(const std::vector& v, std::string name=""); + + bool SetString(const std::string& s, std::string name=""); + bool SetStrings(const std::vector& v, std::string name=""); + + // The Id() and OriginalId() are only used for auto parallel. + uint64_t Id() const { return id_; } + uint64_t OriginalId() const { return original_id_; } + void SetOriginalId(uint64_t original_id) { original_id_ = original_id; } + +private: + void AddEntry(); + +private: + proto::PropertyVals property_; + + // This thread-safe implementation seems to be redudent since the neural + // networks are usually constructed in a single thread. + static uint64_t GenerateId() { + static std::atomic uid{0}; + return ++uid; + } + + // Note: the id_ is unique for all Property (only for auto parallel). + uint64_t id_ = GenerateId(); + // Note: the orignal_id_ is used for referring to the original Property + // that the current Property is built from (only for auto parallel). + // The default original_id_ is same as the id_, which means the + // current Property is not built from the other one. + uint64_t original_id_ = id_; +}; + +bool operator==(const Property &left, const Property &right); + +} // namespace jit +} // namespace paddle diff --git a/paddle/fluid/jit/property.proto b/paddle/fluid/jit/property.proto new file mode 100644 index 0000000000000..1b95f622300ad --- /dev/null +++ b/paddle/fluid/jit/property.proto @@ -0,0 +1,144 @@ +/*Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +syntax = "proto2"; +package paddle.jit.proto; + +message TensorProto { + enum DataType { + UNDEFINED = 0; + // Basic types. + FLOAT = 1; // float + UINT8 = 2; // uint8_t + INT8 = 3; // int8_t + UINT16 = 4; // uint16_t + INT16 = 5; // int16_t + INT32 = 6; // int32_t + INT64 = 7; // int64_t + STRING = 8; // string + BOOL = 9; // bool + + // IEEE754 half-precision floating-point format (16 bits wide). + // This format has 1 sign bit, 5 exponent bits, and 10 mantissa bits. + FLOAT16 = 10; + + DOUBLE = 11; + UINT32 = 12; + UINT64 = 13; + COMPLEX64 = 14; // complex with float32 real and imaginary components + COMPLEX128 = 15; // complex with float64 real and imaginary components + + // Non-IEEE floating-point format based on IEEE754 single-precision + // floating-point number truncated to 16 bits. + // This format has 1 sign bit, 8 exponent bits, and 7 mantissa bits. + BFLOAT16 = 16; + } + + optional bool stop_gradient = 1; + + // The shape of the tensor. + repeated int64 dims = 2; + + // The data type of the tensor. + optional int32 data_type = 3; + + // Tensor content must be organized in row-major order. + // For float and complex64 values + // Complex64 tensors are encoded as a single array of floats, + // with the real components appearing in odd numbered positions, + // and the corresponding imaginary component appearing in the + // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i] + // is encoded as [1.0, 2.0 ,3.0 ,4.0] + // When this field is present, the data_type field MUST be FLOAT or COMPLEX64. + repeated float float_data = 4 [packed = true]; + + // For int32, uint8, int8, uint16, int16, bool, and float16 values + // float16 values must be bit-wise converted to an uint16_t prior + // to writing to the buffer. + // When this field is present, the data_type field MUST be + // INT32, INT16, INT8, UINT16, UINT8, BOOL, or FLOAT16 + repeated int32 int32_data = 5 [packed = true]; + + // For strings. + // Each element of string_data is a UTF-8 encoded Unicode + // string. No trailing null, no leading BOM. The protobuf "string" + // scalar type is not used to match ML community conventions. + // When this field is present, the data_type field MUST be STRING + repeated bytes string_data = 6; + + // For int64. + // When this field is present, the data_type field MUST be INT64 + repeated int64 int64_data = 7 [packed = true]; + + // For double + // Complex128 tensors are encoded as a single array of doubles, + // with the real components appearing in odd numbered positions, + // and the corresponding imaginary component appearing in the + // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i] + // is encoded as [1.0, 2.0 ,3.0 ,4.0] + // When this field is present, the data_type field MUST be DOUBLE or COMPLEX128 + repeated double double_data = 8 [packed = true]; + + // For uint64 and uint32 values + // When this field is present, the data_type field MUST be + // UINT32 or UINT64 + repeated uint64 uint64_data = 9 [packed = true]; + + // Serializations can either use one of the fields above, or use this + // raw bytes field. The only exception is the string case, where one is + // required to store the content in the repeated bytes string_data field. + // + // When this raw_data field is used to store tensor value, elements MUST + // be stored in as fixed-width, little-endian order. + // Floating-point data types MUST be stored in IEEE 754 format. + // Complex64 elements must be written as two consecutive FLOAT values, real component first. + // Complex128 elements must be written as two consecutive DOUBLE values, real component first. + // Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). + // + // Note: the advantage of specific field rather than the raw_data field is + // that in some cases (e.g. int data), protobuf does a better packing via + // variable length storage, and may lead to smaller binary footprint. + // When this field is present, the data_type field MUST NOT be STRING or UNDEFINED + optional bytes raw_data = 10; +} + +message ValueProto { + enum AttributeType { + UNDEFINED = 0; + FLOAT = 1; + INT = 2; + STRING = 3; + TENSOR = 4; + + FLOATS = 6; + INTS = 7; + STRINGS = 8; + TENSORS = 9; + } + optional string name = 1; + + optional AttributeType type = 2; // discriminator that indicates which field below is in use + + // Exactly ONE of the following fields must be present + optional float f = 3; // float + optional int64 i = 4; // int + optional bytes s = 5; // UTF-8 string + optional TensorProto t = 6; // tensor value + + repeated float floats = 7; // list of floats + repeated int64 ints = 8; // list of ints + repeated bytes strings = 9; // list of UTF-8 strings + repeated TensorProto tensors = 10; // list of tensors +} + +message PropertyVals { + repeated ValueProto entrys=1; +} \ No newline at end of file diff --git a/paddle/fluid/jit/serializer.cc b/paddle/fluid/jit/serializer.cc index a557f9edc6ce3..06be974ea2669 100644 --- a/paddle/fluid/jit/serializer.cc +++ b/paddle/fluid/jit/serializer.cc @@ -95,6 +95,7 @@ framework::ProgramDesc Deserializer::LoadProgram(const std::string& file_name) { return framework::ProgramDesc(buffer); } + Layer Load(const std::string& file_path, const phi::Place& place) { auto deserializer = Deserializer(); return deserializer(file_path, place); diff --git a/paddle/fluid/jit/serializer.h b/paddle/fluid/jit/serializer.h index 8f2697f724389..7927d95b9d25d 100644 --- a/paddle/fluid/jit/serializer.h +++ b/paddle/fluid/jit/serializer.h @@ -18,6 +18,7 @@ #include "paddle/fluid/framework/var_desc.h" #include "paddle/fluid/framework/variable.h" +#include "paddle/fluid/jit/property.pb.h" #include "paddle/fluid/jit/layer.h" @@ -47,10 +48,12 @@ class Deserializer { const phi::Place& place, Name2VariableMap* params_dict) const; + // property pb void ReadAttributeData(const std::string& file_path, Name2VariableMap* attrs_dict) const; // void ReadExtraInfo(const std::string& file_name) const; + // void ReadByteCode(const std::string& file_name) const; framework::ProgramDesc LoadProgram(const std::string& file_name); diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index 6d2d62625560d..18eea6253bc75 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -26,6 +26,7 @@ limitations under the License. */ #include "paddle/fluid/framework/var_desc.h" #include "paddle/fluid/framework/version.h" #include "paddle/fluid/pybind/pybind_boost_headers.h" +#include "paddle/fluid/jit/property.h" namespace paddle { namespace pybind { @@ -34,6 +35,7 @@ PyTypeObject *g_vartype_pytype = nullptr; PyTypeObject *g_blockdesc_pytype = nullptr; namespace pd = paddle::framework; +namespace jit = paddle::jit; template static pybind11::bytes SerializeMessage( @@ -342,5 +344,57 @@ void BindOpDesc(pybind11::module *m) { .def("outputs", &pd::OpDesc::Outputs); } + +// Serialize Class Property +void BindPropertyDesc(pybind11::module *m){ + pybind11::enum_(*m, "PropertyTensorDType", "") + .value("FLOAT", jit::proto::TensorProto::FLOAT) + .value("UINT8", jit::proto::TensorProto::UINT8) + .value("INT8", jit::proto::TensorProto::INT8) + .value("UINT16", jit::proto::TensorProto::UINT16) + .value("INT16", jit::proto::TensorProto::INT16) + .value("INT32", jit::proto::TensorProto::INT32) + .value("INT64", jit::proto::TensorProto::INT64) + .value("STRING", jit::proto::TensorProto::STRING) + .value("BOOL", jit::proto::TensorProto::BOOL) + .value("FLOAT16", jit::proto::TensorProto::FLOAT16) + .value("DOUBLE", jit::proto::TensorProto::DOUBLE) + .value("UINT32", jit::proto::TensorProto::UINT32) + .value("UINT64", jit::proto::TensorProto::UINT64) + .value("COMPLEX64", jit::proto::TensorProto::COMPLEX64) + .value("COMPLEX128", jit::proto::TensorProto::COMPLEX128) + .value("BFLOAT16", jit::proto::TensorProto::BFLOAT16); + + + pybind11::enum_(*m, "PropertyValueType", "") + .value("FLOAT", jit::proto::ValueProto::FLOAT) + .value("INT", jit::proto::ValueProto::INT) + .value("STRING", jit::proto::ValueProto::STRING) + .value("TENSOR", jit::proto::ValueProto::TENSOR) + .value("FLOATS", jit::proto::ValueProto::FLOATS) + .value("INTS", jit::proto::ValueProto::INTS) + .value("STRINGS", jit::proto::ValueProto::STRINGS) + .value("TENSORS", jit::proto::ValueProto::TENSORS); + + pybind11::class_ property(*m, "Property"); + property.def( + "__init__", + [](jit::Property &self){ new (&self) jit::Property(); }, + pybind11::return_value_policy::reference) + .def("set_float", &jit::Property::SetFloat) + .def("set_floats", &jit::Property::SetFloats) + .def("set_int64", &jit::Property::SetInt64) + .def("set_int64s", &jit::Property::SetInt64s) + .def("set_string", &jit::Property::SetString) + .def("set_strings", &jit::Property::SetStrings) + .def("set_tensor", [](pd::VarDesc& tensor){ + throw platform::errors::Unimplemented("Not implement set_tensor."); + }) + .def("set_tensors", [](pybind11::list& tensors){ + throw platform::errors::Unimplemented("Not implement set_tensors."); + }) + .def("serialize_to_string", SerializeMessage); +} + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/protobuf.h b/paddle/fluid/pybind/protobuf.h index 79f174b5eb607..8ff14bdfa5af8 100644 --- a/paddle/fluid/pybind/protobuf.h +++ b/paddle/fluid/pybind/protobuf.h @@ -34,6 +34,7 @@ void BindBlockDesc(pybind11::module* m); void BindVarDsec(pybind11::module* m); void BindOpDesc(pybind11::module* m); void BindProcessMeshDesc(pybind11::module* m); +void BindPropertyDesc(pybind11::module *m); } // namespace pybind } // namespace paddle From 3ae745cb680f1325236bba18bd2a1c3154a6f448 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Mon, 18 Jul 2022 12:42:25 +0000 Subject: [PATCH 02/18] property get and deserilize --- paddle/fluid/jit/CMakeLists.txt | 2 +- paddle/fluid/jit/property.cc | 97 ++++++++++++++++++++++++++++++ paddle/fluid/jit/property.h | 51 ++++------------ paddle/fluid/jit/serializer.h | 2 +- paddle/fluid/pybind/CMakeLists.txt | 3 +- paddle/fluid/pybind/protobuf.cc | 75 ++++++++++++----------- paddle/fluid/pybind/pybind.cc | 1 + 7 files changed, 151 insertions(+), 80 deletions(-) diff --git a/paddle/fluid/jit/CMakeLists.txt b/paddle/fluid/jit/CMakeLists.txt index 0c3e6043bbdd2..d12b037608e64 100644 --- a/paddle/fluid/jit/CMakeLists.txt +++ b/paddle/fluid/jit/CMakeLists.txt @@ -58,6 +58,6 @@ endif() proto_library(paddle_jit_property_proto SRCS property.proto) cc_library( - property + jit_property SRCS property.cc DEPS paddle_jit_property_proto) \ No newline at end of file diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 51f2fd3de0d42..2d910248b5b3b 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -13,9 +13,106 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/jit/property.h" +#include "paddle/phi/core/errors.h" namespace paddle { namespace jit { +size_t Property::Size() const{ + return property_.entrys_size(); +} + +void Property::SetFloat(const float& f) { + auto type = proto::ValueProto::FLOAT; + auto entry = property_.add_entrys(); + entry->set_type(type); + entry->set_f(f); + VLOG(3) << "Property: set_float " << f; +} + +void Property::SetFloat(const std::string& name, const float& f){ + auto type = proto::ValueProto::FLOAT; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + entry->set_f(f); + VLOG(3) << "Property: set_float " << f << " name: " << name; +} + +√ Property::GetFloat(const std::string& name) const{ + for (size_t i = 0; i < Size(); i++){ + auto e = property_.entrys(i); + if (e.has_name() && e.name() == name ){ + return e.f(); + } + } + phi::errors::NotFound("name not found"); +} + +float Property::GetFloat(const int& idx) const{ + if (idx >= Size()){ + phi::errors::OutOfRange("idx out of range"); + } + auto e = property_.entrys(idx); + if (e.has_f()){ + return e.f(); + } + phi::errors::NotFound("not has float"); +} + +void Property::SetFloats(const std::string& name, const std::vector& v){ + auto type = proto::ValueProto::FLOATS; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + for (auto i : v){ + entry->add_floats(i); + } + VLOG(3) << "Property: set_floats " << v[0] << "... name: " << name; + +} + +void Property::SetInt64(const std::string& name, const int64_t& i){ + auto type = proto::ValueProto::INT; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + entry->set_i(i); + VLOG(3) << "Property: set_int " << i << " name: " << name; +} + +void Property::SetInt64s(const std::string& name, const std::vector& v){ + auto type = proto::ValueProto::INTS; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + for (auto i : v){ + entry->add_ints(i); + } + VLOG(3) << "Property: set_ints " << v[0] << " name: " << name; + +} + +void Property::SetString(const std::string& name, const std::string& s){ + auto type = proto::ValueProto::STRING; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + entry->set_s(s); + VLOG(3) << "Property: set_string " << s << " name: " << name; +} + +void Property::SetStrings(const std::string& name, const std::vector& v){ + auto type = proto::ValueProto::STRINGS; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + for (auto i : v){ + entry->add_strings(i); + } + VLOG(3) << "Property: set_strings " << v[0] << " name: " << name; +} + + } // namespace jit } // namespace paddle diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h index 3d68f42298a39..15bffd1c34439 100644 --- a/paddle/fluid/jit/property.h +++ b/paddle/fluid/jit/property.h @@ -25,39 +25,6 @@ namespace paddle { namespace jit { -// convert between std::vector and protobuf repeated. -template -inline std::vector RepeatedToVector( - const google::protobuf::RepeatedField &repeated_field) { - std::vector ret; - ret.reserve(repeated_field.size()); - std::copy( - repeated_field.begin(), repeated_field.end(), std::back_inserter(ret)); - return ret; -} - -template -inline void VectorToRepeated(const std::vector &vec, - RepeatedField *repeated_field) { - repeated_field->Clear(); - repeated_field->Reserve(vec.size()); - for (const auto &elem : vec) { - *repeated_field->Add() = elem; - } -} - -// Specialize vector. -template -inline void VectorToRepeated(const std::vector &vec, - RepeatedField *repeated_field) { - repeated_field->Clear(); - repeated_field->Reserve(vec.size()); - for (auto elem : vec) { - *repeated_field->Add() = elem; - } -} - - class Property { public: explicit Property() {} @@ -77,14 +44,20 @@ class Property { const proto::PropertyVals *Proto() const { return &property_; } - bool SetFloat(const float& f, std::string name=""); - bool SetFloats(const std::vector& v, std::string name=""); + size_t Size() const; + + void SetFloat(const float& f); + void SetFloat(const std::string& name, const float& f); + void SetFloats(const std::string& name, const std::vector& v); + + float GetFloat(const std::string& name) const; + float GetFloat(const int& idx) const; - bool SetInt64(const int64_t& f, std::string name=""); - bool SetInt64s(const std::vector& v, std::string name=""); + void SetInt64(const std::string& name, const int64_t& f); + void SetInt64s(const std::string& name, const std::vector& v); - bool SetString(const std::string& s, std::string name=""); - bool SetStrings(const std::vector& v, std::string name=""); + void SetString(const std::string& name, const std::string& s); + void SetStrings(const std::string& name, const std::vector& v); // The Id() and OriginalId() are only used for auto parallel. uint64_t Id() const { return id_; } diff --git a/paddle/fluid/jit/serializer.h b/paddle/fluid/jit/serializer.h index 7927d95b9d25d..bdc3b81d55193 100644 --- a/paddle/fluid/jit/serializer.h +++ b/paddle/fluid/jit/serializer.h @@ -18,7 +18,7 @@ #include "paddle/fluid/framework/var_desc.h" #include "paddle/fluid/framework/variable.h" -#include "paddle/fluid/jit/property.pb.h" +#include "paddle/fluid/jit/property.h" #include "paddle/fluid/jit/layer.h" diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index b2ecf36c5d227..aac17e7ecdbc4 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -39,7 +39,8 @@ set(PYBIND_DEPS phi_utils tcp_store new_profiler - jit_layer) + jit_layer + jit_property) if(WITH_PSCORE) set(PYBIND_DEPS ${PYBIND_DEPS} ps_service) diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index 18eea6253bc75..a7ba8cee1db1e 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -28,6 +28,9 @@ limitations under the License. */ #include "paddle/fluid/pybind/pybind_boost_headers.h" #include "paddle/fluid/jit/property.h" + +namespace py = pybind11; + namespace paddle { namespace pybind { @@ -49,6 +52,14 @@ static pybind11::bytes SerializeMessage( return retv; } +template +static void DeserializeMessage(T &self, const std::string& str){ + PADDLE_ENFORCE_EQ(self.Proto()->ParsePartialFromString(str), + true, + platform::errors::InvalidArgument("Failed to parse pb from string")); + return; +} + // Bind Methods void BindProgramDesc(pybind11::module *m) { pybind11::class_(*m, "ProgramDesc", "") @@ -347,53 +358,41 @@ void BindOpDesc(pybind11::module *m) { // Serialize Class Property void BindPropertyDesc(pybind11::module *m){ - pybind11::enum_(*m, "PropertyTensorDType", "") - .value("FLOAT", jit::proto::TensorProto::FLOAT) - .value("UINT8", jit::proto::TensorProto::UINT8) - .value("INT8", jit::proto::TensorProto::INT8) - .value("UINT16", jit::proto::TensorProto::UINT16) - .value("INT16", jit::proto::TensorProto::INT16) - .value("INT32", jit::proto::TensorProto::INT32) - .value("INT64", jit::proto::TensorProto::INT64) - .value("STRING", jit::proto::TensorProto::STRING) - .value("BOOL", jit::proto::TensorProto::BOOL) - .value("FLOAT16", jit::proto::TensorProto::FLOAT16) - .value("DOUBLE", jit::proto::TensorProto::DOUBLE) - .value("UINT32", jit::proto::TensorProto::UINT32) - .value("UINT64", jit::proto::TensorProto::UINT64) - .value("COMPLEX64", jit::proto::TensorProto::COMPLEX64) - .value("COMPLEX128", jit::proto::TensorProto::COMPLEX128) - .value("BFLOAT16", jit::proto::TensorProto::BFLOAT16); - - - pybind11::enum_(*m, "PropertyValueType", "") - .value("FLOAT", jit::proto::ValueProto::FLOAT) - .value("INT", jit::proto::ValueProto::INT) - .value("STRING", jit::proto::ValueProto::STRING) - .value("TENSOR", jit::proto::ValueProto::TENSOR) - .value("FLOATS", jit::proto::ValueProto::FLOATS) - .value("INTS", jit::proto::ValueProto::INTS) - .value("STRINGS", jit::proto::ValueProto::STRINGS) - .value("TENSORS", jit::proto::ValueProto::TENSORS); - pybind11::class_ property(*m, "Property"); property.def( "__init__", [](jit::Property &self){ new (&self) jit::Property(); }, pybind11::return_value_policy::reference) - .def("set_float", &jit::Property::SetFloat) - .def("set_floats", &jit::Property::SetFloats) - .def("set_int64", &jit::Property::SetInt64) - .def("set_int64s", &jit::Property::SetInt64s) - .def("set_string", &jit::Property::SetString) - .def("set_strings", &jit::Property::SetStrings) - .def("set_tensor", [](pd::VarDesc& tensor){ + .def("size", + &jit::Property::Size + ) + .def("set_float", + py::overload_cast(&jit::Property::SetFloat), + "set float", + py::arg("val")) + .def("set_float", + py::overload_cast(&jit::Property::SetFloat), + "set float", + py::arg("val"), py::arg("name")) + .def("get_float", + py::overload_cast(&jit::Property::GetFloat, py::const_) + ) + .def("get_float", + py::overload_cast(&jit::Property::GetFloat, py::const_) + ) + .def("set_floats", &jit::Property::SetFloats, "set list of float", py::arg("val"), py::arg("name")) + .def("set_int64", &jit::Property::SetInt64, "set int", py::arg("val"), py::arg("name")) + .def("set_int64s", &jit::Property::SetInt64s, "set list of int", py::arg("val"), py::arg("name")) + .def("set_string", &jit::Property::SetString, "set string", py::arg("val"), py::arg("name")) + .def("set_strings", &jit::Property::SetStrings, "set list of string", py::arg("val"), py::arg("name")) + .def("set_tensor", [](const pd::VarDesc& tensor, const std::string name){ throw platform::errors::Unimplemented("Not implement set_tensor."); }) - .def("set_tensors", [](pybind11::list& tensors){ + .def("set_tensors", [](const pybind11::list& tensors, const std::string name){ throw platform::errors::Unimplemented("Not implement set_tensors."); }) - .def("serialize_to_string", SerializeMessage); + .def("serialize_to_string", SerializeMessage) + .def("parse_from_string", DeserializeMessage); } } // namespace pybind diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 62f0402bedc7a..093a132a09098 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -3220,6 +3220,7 @@ All parameter, weight, gradient are variables in Paddle. BindProcessMeshDesc(&m); BindFleetExecutor(&m); BindTCPStore(&m); + BindPropertyDesc(&m); py::class_(m, "LodRankTable") .def("items", [](framework::LoDRankTable &table) { From 5f628f43e970d7a066ca2edb4333f23463da98a6 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Tue, 19 Jul 2022 04:00:48 +0000 Subject: [PATCH 03/18] support get float --- paddle/fluid/jit/property.cc | 157 +++++++++++++++++------------------ paddle/fluid/jit/property.h | 29 ++++--- 2 files changed, 92 insertions(+), 94 deletions(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 2d910248b5b3b..4856e0cd80f23 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -18,101 +18,100 @@ limitations under the License. */ namespace paddle { namespace jit { -size_t Property::Size() const{ - return property_.entrys_size(); +int Property::Size() const { return property_.entrys_size(); } + +void Property::SetFloat(const float &f) { + auto type = proto::ValueProto::FLOAT; + auto entry = property_.add_entrys(); + entry->set_type(type); + entry->set_f(f); + VLOG(3) << "Property: set_float " << f; } -void Property::SetFloat(const float& f) { - auto type = proto::ValueProto::FLOAT; - auto entry = property_.add_entrys(); - entry->set_type(type); - entry->set_f(f); - VLOG(3) << "Property: set_float " << f; +void Property::SetFloat(const std::string &name, const float &f) { + auto type = proto::ValueProto::FLOAT; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + entry->set_f(f); + VLOG(3) << "Property: set_float " << f << " name: " << name; } -void Property::SetFloat(const std::string& name, const float& f){ - auto type = proto::ValueProto::FLOAT; - auto entry = property_.add_entrys(); - entry->set_name(name); - entry->set_type(type); - entry->set_f(f); - VLOG(3) << "Property: set_float " << f << " name: " << name; -} - -√ Property::GetFloat(const std::string& name) const{ - for (size_t i = 0; i < Size(); i++){ - auto e = property_.entrys(i); - if (e.has_name() && e.name() == name ){ - return e.f(); - } +float Property::GetFloat(const std::string &name) const { + for (int i = 0; i < Size(); i++) { + auto e = property_.entrys(i); + if (e.has_name() && e.name() == name) { + return e.f(); } - phi::errors::NotFound("name not found"); + } + phi::errors::NotFound("name not found"); + return 0; } -float Property::GetFloat(const int& idx) const{ - if (idx >= Size()){ - phi::errors::OutOfRange("idx out of range"); - } - auto e = property_.entrys(idx); - if (e.has_f()){ - return e.f(); - } - phi::errors::NotFound("not has float"); +float Property::GetFloat(const int &idx) const { + if (idx >= Size()) { + phi::errors::OutOfRange("idx out of range"); + } + auto e = property_.entrys(idx); + if (e.has_f()) { + return e.f(); + } + phi::errors::NotFound("not has float"); + return 0; } -void Property::SetFloats(const std::string& name, const std::vector& v){ - auto type = proto::ValueProto::FLOATS; - auto entry = property_.add_entrys(); - entry->set_name(name); - entry->set_type(type); - for (auto i : v){ - entry->add_floats(i); - } - VLOG(3) << "Property: set_floats " << v[0] << "... name: " << name; - +void Property::SetFloats(const std::string &name, const std::vector &v) { + auto type = proto::ValueProto::FLOATS; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + for (auto i : v) { + entry->add_floats(i); + } + VLOG(3) << "Property: set_floats " << v[0] << "... name: " << name; } -void Property::SetInt64(const std::string& name, const int64_t& i){ - auto type = proto::ValueProto::INT; - auto entry = property_.add_entrys(); - entry->set_name(name); - entry->set_type(type); - entry->set_i(i); - VLOG(3) << "Property: set_int " << i << " name: " << name; +void Property::SetInt64(const std::string &name, const int64_t &i) { + auto type = proto::ValueProto::INT; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + entry->set_i(i); + VLOG(3) << "Property: set_int " << i << " name: " << name; } -void Property::SetInt64s(const std::string& name, const std::vector& v){ - auto type = proto::ValueProto::INTS; - auto entry = property_.add_entrys(); - entry->set_name(name); - entry->set_type(type); - for (auto i : v){ - entry->add_ints(i); - } - VLOG(3) << "Property: set_ints " << v[0] << " name: " << name; - +void Property::SetInt64s(const std::string &name, + const std::vector &v) { + auto type = proto::ValueProto::INTS; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + for (auto i : v) { + entry->add_ints(i); + } + VLOG(3) << "Property: set_ints " << v[0] << " name: " << name; } -void Property::SetString(const std::string& name, const std::string& s){ - auto type = proto::ValueProto::STRING; - auto entry = property_.add_entrys(); - entry->set_name(name); - entry->set_type(type); - entry->set_s(s); - VLOG(3) << "Property: set_string " << s << " name: " << name; +void Property::SetString(const std::string &name, const std::string &s) { + auto type = proto::ValueProto::STRING; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + entry->set_s(s); + VLOG(3) << "Property: set_string " << s << " name: " << name; } -void Property::SetStrings(const std::string& name, const std::vector& v){ - auto type = proto::ValueProto::STRINGS; - auto entry = property_.add_entrys(); - entry->set_name(name); - entry->set_type(type); - for (auto i : v){ - entry->add_strings(i); - } - VLOG(3) << "Property: set_strings " << v[0] << " name: " << name; +void Property::SetStrings(const std::string &name, + const std::vector &v) { + auto type = proto::ValueProto::STRINGS; + auto entry = property_.add_entrys(); + entry->set_name(name); + entry->set_type(type); + for (auto i : v) { + entry->add_strings(i); + } + VLOG(3) << "Property: set_strings " << v[0] << " name: " << name; } - -} // namespace jit -} // namespace paddle +} // namespace jit +} // namespace paddle diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h index 15bffd1c34439..f5db27d615d45 100644 --- a/paddle/fluid/jit/property.h +++ b/paddle/fluid/jit/property.h @@ -31,8 +31,7 @@ class Property { // Explicitly implement the copy constructor for auto parallel Property(const Property &other) - : property_(other.property_), - original_id_(other.original_id_) {} + : property_(other.property_), original_id_(other.original_id_) {} Property &operator=(const Property &other) { property_ = other.property_; @@ -44,32 +43,32 @@ class Property { const proto::PropertyVals *Proto() const { return &property_; } - size_t Size() const; + int Size() const; - void SetFloat(const float& f); - void SetFloat(const std::string& name, const float& f); - void SetFloats(const std::string& name, const std::vector& v); + void SetFloat(const float &f); + void SetFloat(const std::string &name, const float &f); + void SetFloats(const std::string &name, const std::vector &v); - float GetFloat(const std::string& name) const; - float GetFloat(const int& idx) const; + float GetFloat(const std::string &name) const; + float GetFloat(const int &idx) const; - void SetInt64(const std::string& name, const int64_t& f); - void SetInt64s(const std::string& name, const std::vector& v); + void SetInt64(const std::string &name, const int64_t &f); + void SetInt64s(const std::string &name, const std::vector &v); - void SetString(const std::string& name, const std::string& s); - void SetStrings(const std::string& name, const std::vector& v); + void SetString(const std::string &name, const std::string &s); + void SetStrings(const std::string &name, const std::vector &v); // The Id() and OriginalId() are only used for auto parallel. uint64_t Id() const { return id_; } uint64_t OriginalId() const { return original_id_; } void SetOriginalId(uint64_t original_id) { original_id_ = original_id; } -private: + private: void AddEntry(); -private: + private: proto::PropertyVals property_; - + // This thread-safe implementation seems to be redudent since the neural // networks are usually constructed in a single thread. static uint64_t GenerateId() { From b3eadde2e9c37719c591c79477ebc235397017fb Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Tue, 19 Jul 2022 06:30:35 +0000 Subject: [PATCH 04/18] format code --- paddle/fluid/jit/CMakeLists.txt | 3 +-- paddle/fluid/jit/property.proto | 2 +- paddle/fluid/jit/serializer.cc | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/jit/CMakeLists.txt b/paddle/fluid/jit/CMakeLists.txt index d12b037608e64..21af49c68870b 100644 --- a/paddle/fluid/jit/CMakeLists.txt +++ b/paddle/fluid/jit/CMakeLists.txt @@ -54,10 +54,9 @@ if(WITH_TESTING add_dependencies(layer_test jit_download_program) endif() - proto_library(paddle_jit_property_proto SRCS property.proto) cc_library( jit_property SRCS property.cc - DEPS paddle_jit_property_proto) \ No newline at end of file + DEPS paddle_jit_property_proto) diff --git a/paddle/fluid/jit/property.proto b/paddle/fluid/jit/property.proto index 1b95f622300ad..0b83886116a2a 100644 --- a/paddle/fluid/jit/property.proto +++ b/paddle/fluid/jit/property.proto @@ -141,4 +141,4 @@ message ValueProto { message PropertyVals { repeated ValueProto entrys=1; -} \ No newline at end of file +} diff --git a/paddle/fluid/jit/serializer.cc b/paddle/fluid/jit/serializer.cc index 06be974ea2669..a557f9edc6ce3 100644 --- a/paddle/fluid/jit/serializer.cc +++ b/paddle/fluid/jit/serializer.cc @@ -95,7 +95,6 @@ framework::ProgramDesc Deserializer::LoadProgram(const std::string& file_name) { return framework::ProgramDesc(buffer); } - Layer Load(const std::string& file_path, const phi::Place& place) { auto deserializer = Deserializer(); return deserializer(file_path, place); From 0f5d0e0bfd0f9c3305ab9f54dca12ee018d30a77 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Tue, 19 Jul 2022 06:54:57 +0000 Subject: [PATCH 05/18] format code --- paddle/fluid/jit/property.cc | 4 +- paddle/fluid/pybind/protobuf.cc | 106 +++++++++++++++++++------------- paddle/fluid/pybind/protobuf.h | 3 +- 3 files changed, 68 insertions(+), 45 deletions(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 4856e0cd80f23..29fd5838c9c62 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -113,5 +113,5 @@ void Property::SetStrings(const std::string &name, VLOG(3) << "Property: set_strings " << v[0] << " name: " << name; } -} // namespace jit -} // namespace paddle +} // namespace jit +} // namespace paddle diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index a7ba8cee1db1e..dbfa23f4e7ccb 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -25,9 +25,8 @@ limitations under the License. */ #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/framework/var_desc.h" #include "paddle/fluid/framework/version.h" -#include "paddle/fluid/pybind/pybind_boost_headers.h" #include "paddle/fluid/jit/property.h" - +#include "paddle/fluid/pybind/pybind_boost_headers.h" namespace py = pybind11; @@ -53,11 +52,12 @@ static pybind11::bytes SerializeMessage( } template -static void DeserializeMessage(T &self, const std::string& str){ - PADDLE_ENFORCE_EQ(self.Proto()->ParsePartialFromString(str), - true, - platform::errors::InvalidArgument("Failed to parse pb from string")); - return; +static void DeserializeMessage(T *self, const std::string &str) { + PADDLE_ENFORCE_EQ( + self->Proto()->ParsePartialFromString(str), + true, + platform::errors::InvalidArgument("Failed to parse pb from string")); + return; } // Bind Methods @@ -355,44 +355,66 @@ void BindOpDesc(pybind11::module *m) { .def("outputs", &pd::OpDesc::Outputs); } - // Serialize Class Property -void BindPropertyDesc(pybind11::module *m){ +void BindPropertyDesc(pybind11::module *m) { pybind11::class_ property(*m, "Property"); - property.def( - "__init__", - [](jit::Property &self){ new (&self) jit::Property(); }, - pybind11::return_value_policy::reference) - .def("size", - &jit::Property::Size - ) - .def("set_float", - py::overload_cast(&jit::Property::SetFloat), - "set float", - py::arg("val")) - .def("set_float", - py::overload_cast(&jit::Property::SetFloat), - "set float", - py::arg("val"), py::arg("name")) - .def("get_float", - py::overload_cast(&jit::Property::GetFloat, py::const_) - ) - .def("get_float", - py::overload_cast(&jit::Property::GetFloat, py::const_) - ) - .def("set_floats", &jit::Property::SetFloats, "set list of float", py::arg("val"), py::arg("name")) - .def("set_int64", &jit::Property::SetInt64, "set int", py::arg("val"), py::arg("name")) - .def("set_int64s", &jit::Property::SetInt64s, "set list of int", py::arg("val"), py::arg("name")) - .def("set_string", &jit::Property::SetString, "set string", py::arg("val"), py::arg("name")) - .def("set_strings", &jit::Property::SetStrings, "set list of string", py::arg("val"), py::arg("name")) - .def("set_tensor", [](const pd::VarDesc& tensor, const std::string name){ - throw platform::errors::Unimplemented("Not implement set_tensor."); + property + .def( + "__init__", + [](jit::Property &self) { new (&self) jit::Property(); }, + pybind11::return_value_policy::reference) + .def("size", &jit::Property::Size) + .def("set_float", + py::overload_cast(&jit::Property::SetFloat), + "set float", + py::arg("val")) + .def("set_float", + py::overload_cast( + &jit::Property::SetFloat), + "set float", + py::arg("val"), + py::arg("name")) + .def("get_float", + py::overload_cast(&jit::Property::GetFloat, py::const_)) + .def("get_float", + py::overload_cast(&jit::Property::GetFloat, + py::const_)) + .def("set_floats", + &jit::Property::SetFloats, + "set list of float", + py::arg("val"), + py::arg("name")) + .def("set_int64", + &jit::Property::SetInt64, + "set int", + py::arg("val"), + py::arg("name")) + .def("set_int64s", + &jit::Property::SetInt64s, + "set list of int", + py::arg("val"), + py::arg("name")) + .def("set_string", + &jit::Property::SetString, + "set string", + py::arg("val"), + py::arg("name")) + .def("set_strings", + &jit::Property::SetStrings, + "set list of string", + py::arg("val"), + py::arg("name")) + .def("set_tensor", + [](const pd::VarDesc &tensor, const std::string name) { + throw platform::errors::Unimplemented("Not implement set_tensor."); + }) + .def( + "set_tensors", + [](const pybind11::list &tensors, const std::string name) { + throw platform::errors::Unimplemented("Not implement set_tensors."); }) - .def("set_tensors", [](const pybind11::list& tensors, const std::string name){ - throw platform::errors::Unimplemented("Not implement set_tensors."); - }) - .def("serialize_to_string", SerializeMessage) - .def("parse_from_string", DeserializeMessage); + .def("serialize_to_string", SerializeMessage) + .def("parse_from_string", DeserializeMessage); } } // namespace pybind diff --git a/paddle/fluid/pybind/protobuf.h b/paddle/fluid/pybind/protobuf.h index 8ff14bdfa5af8..929721678ed1a 100644 --- a/paddle/fluid/pybind/protobuf.h +++ b/paddle/fluid/pybind/protobuf.h @@ -11,6 +11,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + #pragma once #if defined(_MSC_VER) @@ -34,7 +35,7 @@ void BindBlockDesc(pybind11::module* m); void BindVarDsec(pybind11::module* m); void BindOpDesc(pybind11::module* m); void BindProcessMeshDesc(pybind11::module* m); -void BindPropertyDesc(pybind11::module *m); +void BindPropertyDesc(pybind11::module* m); } // namespace pybind } // namespace paddle From 496256d36853e9b1e1902f533d9671d79012f582 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Tue, 19 Jul 2022 08:28:21 +0000 Subject: [PATCH 06/18] add unittest --- paddle/fluid/jit/property.cc | 15 ++-- paddle/fluid/pybind/protobuf.cc | 4 +- .../dygraph_to_static/test_property_save.py | 71 +++++++++++++++++++ 3 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 29fd5838c9c62..2ce882f016d5a 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/jit/property.h" +#include "paddle/phi/core/enforce.h" #include "paddle/phi/core/errors.h" namespace paddle { @@ -44,20 +45,20 @@ float Property::GetFloat(const std::string &name) const { return e.f(); } } - phi::errors::NotFound("name not found"); - return 0; + + PADDLE_ENFORCE_EQ(false, true, phi::errors::NotFound("name not found")); } float Property::GetFloat(const int &idx) const { - if (idx >= Size()) { - phi::errors::OutOfRange("idx out of range"); - } + PADDLE_ENFORCE_EQ( + idx < Size(), true, phi::errors::OutOfRange("idx out of range")); + auto e = property_.entrys(idx); if (e.has_f()) { return e.f(); } - phi::errors::NotFound("not has float"); - return 0; + + PADDLE_ENFORCE_EQ(false, true, phi::errors::NotFound("idx is not a float")); } void Property::SetFloats(const std::string &name, const std::vector &v) { diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index dbfa23f4e7ccb..a12e672c22439 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -384,12 +384,12 @@ void BindPropertyDesc(pybind11::module *m) { "set list of float", py::arg("val"), py::arg("name")) - .def("set_int64", + .def("set_int", &jit::Property::SetInt64, "set int", py::arg("val"), py::arg("name")) - .def("set_int64s", + .def("set_ints", &jit::Property::SetInt64s, "set list of int", py::arg("val"), diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py new file mode 100644 index 0000000000000..04572960837a3 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py @@ -0,0 +1,71 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy +import unittest +import paddle + + +class TestPropertySave(unittest.TestCase): + """test jit property save + """ + + def setUp(self): + a = paddle.framework.core.Property() + a.set_float('a', 1.0) + a.set_floats('b', [1.02, 2.3, 4.23]) + + b = paddle.framework.core.Property() + b.parse_from_string(a.serialize_to_string()) + + self.a = a + self.b = b + + def test_property_save(self): + self.assertEqual(self.a.get_float('a'), self.b.get_float('a')) + self.assertEqual(self.a.get_float(0), 1.0) + + def test_size(self): + self.assertEqual(self.b.size(), 2) + self.assertEqual(self.a.size(), 2) + + def test_load_float(self): + with self.assertRaises(RuntimeError): + self.a.get_float(1) + + def test_set_float_wo_name(self): + """test save without name + """ + a = paddle.framework.core.Property() + a.set_float(10.0) + self.assertEqual(a.get_float(0), 10.0) + + def test_set(self): + """test propety set. + """ + try: + a = paddle.framework.core.Property() + a.set_float(10.0) + a.set_float('float', 10.0) + a.set_floats('floats', [5.0, 4.0, 3.0]) + a.set_int('int', 5) + a.set_ints('ints', [1, 2, 3]) + a.set_string("str", "hello") + a.set_strings('strs', ["1", "2", "3"]) + except Exception as e: + self.assertEqual(False) + + +if __name__ == '__main__': + unittest.main() From eed7a8c2e1a8bd4de2d40ca65619d8693bb0511a Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Tue, 19 Jul 2022 09:11:54 +0000 Subject: [PATCH 07/18] add more set method --- paddle/fluid/jit/property.cc | 50 ++++++++++++++++++- paddle/fluid/jit/property.h | 21 ++++---- paddle/fluid/pybind/protobuf.cc | 39 +++++++++++++-- .../dygraph_to_static/test_property_save.py | 5 ++ 4 files changed, 100 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 2ce882f016d5a..bddd880393831 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -47,6 +47,7 @@ float Property::GetFloat(const std::string &name) const { } PADDLE_ENFORCE_EQ(false, true, phi::errors::NotFound("name not found")); + return 0; } float Property::GetFloat(const int &idx) const { @@ -59,6 +60,17 @@ float Property::GetFloat(const int &idx) const { } PADDLE_ENFORCE_EQ(false, true, phi::errors::NotFound("idx is not a float")); + return 0; +} + +void Property::SetFloats(const std::vector &v) { + auto type = proto::ValueProto::FLOATS; + auto entry = property_.add_entrys(); + entry->set_type(type); + for (auto i : v) { + entry->add_floats(i); + } + VLOG(3) << "Property: set_floats " << v.size(); } void Property::SetFloats(const std::string &name, const std::vector &v) { @@ -69,7 +81,15 @@ void Property::SetFloats(const std::string &name, const std::vector &v) { for (auto i : v) { entry->add_floats(i); } - VLOG(3) << "Property: set_floats " << v[0] << "... name: " << name; + VLOG(3) << "Property: set_floats " << v.size() << "... name: " << name; +} + +void Property::SetInt64(const int64_t &i) { + auto type = proto::ValueProto::INT; + auto entry = property_.add_entrys(); + entry->set_type(type); + entry->set_i(i); + VLOG(3) << "Property: set_int " << i; } void Property::SetInt64(const std::string &name, const int64_t &i) { @@ -81,6 +101,16 @@ void Property::SetInt64(const std::string &name, const int64_t &i) { VLOG(3) << "Property: set_int " << i << " name: " << name; } +void Property::SetInt64s(const std::vector &v) { + auto type = proto::ValueProto::INTS; + auto entry = property_.add_entrys(); + entry->set_type(type); + for (auto e : v) { + entry->add_ints(e); + } + VLOG(3) << "Property: set_ints " << v.size(); +} + void Property::SetInt64s(const std::string &name, const std::vector &v) { auto type = proto::ValueProto::INTS; @@ -93,6 +123,14 @@ void Property::SetInt64s(const std::string &name, VLOG(3) << "Property: set_ints " << v[0] << " name: " << name; } +void Property::SetString(const std::string &s) { + auto type = proto::ValueProto::STRING; + auto entry = property_.add_entrys(); + entry->set_type(type); + entry->set_s(s); + VLOG(3) << "Property: set_string " << s; +} + void Property::SetString(const std::string &name, const std::string &s) { auto type = proto::ValueProto::STRING; auto entry = property_.add_entrys(); @@ -102,6 +140,16 @@ void Property::SetString(const std::string &name, const std::string &s) { VLOG(3) << "Property: set_string " << s << " name: " << name; } +void Property::SetStrings(const std::vector &v) { + auto type = proto::ValueProto::STRINGS; + auto entry = property_.add_entrys(); + entry->set_type(type); + for (auto i : v) { + entry->add_strings(i); + } + VLOG(3) << "Property: set_strings " << v.size(); +} + void Property::SetStrings(const std::string &name, const std::vector &v) { auto type = proto::ValueProto::STRINGS; diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h index f5db27d615d45..1d98fb90da411 100644 --- a/paddle/fluid/jit/property.h +++ b/paddle/fluid/jit/property.h @@ -27,13 +27,13 @@ namespace jit { class Property { public: - explicit Property() {} + Property() {} // Explicitly implement the copy constructor for auto parallel - Property(const Property &other) + explicit Property(const Property &other) : property_(other.property_), original_id_(other.original_id_) {} - Property &operator=(const Property &other) { + explicit Property &operator=(const Property &other) { property_ = other.property_; original_id_ = other.original_id_; return *this; @@ -47,15 +47,23 @@ class Property { void SetFloat(const float &f); void SetFloat(const std::string &name, const float &f); + + void SetFloats(const std::vector &v); void SetFloats(const std::string &name, const std::vector &v); float GetFloat(const std::string &name) const; float GetFloat(const int &idx) const; - void SetInt64(const std::string &name, const int64_t &f); + void SetInt64(const int64_t &i); + void SetInt64(const std::string &name, const int64_t &i); + + void SetInt64s(const std::vector &v); void SetInt64s(const std::string &name, const std::vector &v); + void SetString(const std::string &s); void SetString(const std::string &name, const std::string &s); + + void SetStrings(const std::vector &v); void SetStrings(const std::string &name, const std::vector &v); // The Id() and OriginalId() are only used for auto parallel. @@ -63,9 +71,6 @@ class Property { uint64_t OriginalId() const { return original_id_; } void SetOriginalId(uint64_t original_id) { original_id_ = original_id; } - private: - void AddEntry(); - private: proto::PropertyVals property_; @@ -85,7 +90,5 @@ class Property { uint64_t original_id_ = id_; }; -bool operator==(const Property &left, const Property &right); - } // namespace jit } // namespace paddle diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index a12e672c22439..4cc7440a962ce 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -380,27 +380,56 @@ void BindPropertyDesc(pybind11::module *m) { py::overload_cast(&jit::Property::GetFloat, py::const_)) .def("set_floats", - &jit::Property::SetFloats, + py::overload_cast &>( + &jit::Property::SetFloats), + "set list of float", + py::arg("vals")) + .def("set_floats", + py::overload_cast &>( + &jit::Property::SetFloats), "set list of float", py::arg("val"), py::arg("name")) .def("set_int", - &jit::Property::SetInt64, + py::overload_cast(&jit::Property::SetInt64), + "set int", + py::arg("val")) + .def("set_int", + py::overload_cast( + &jit::Property::SetInt64), "set int", py::arg("val"), py::arg("name")) .def("set_ints", - &jit::Property::SetInt64s, + py::overload_cast &>( + &jit::Property::SetInt64s), + "set list of int", + py::arg("vals")) + .def("set_ints", + py::overload_cast &>( + &jit::Property::SetInt64s), "set list of int", py::arg("val"), py::arg("name")) .def("set_string", - &jit::Property::SetString, + py::overload_cast(&jit::Property::SetString), + "set string", + py::arg("val")) + .def("set_string", + py::overload_cast( + &jit::Property::SetString), "set string", py::arg("val"), py::arg("name")) .def("set_strings", - &jit::Property::SetStrings, + py::overload_cast &>( + &jit::Property::SetStrings), + "set list of string", + py::arg("vals")) + .def("set_strings", + py::overload_cast &>( + &jit::Property::SetStrings), "set list of string", py::arg("val"), py::arg("name")) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py index 04572960837a3..16b81c1ebfc37 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py @@ -58,10 +58,15 @@ def test_set(self): a = paddle.framework.core.Property() a.set_float(10.0) a.set_float('float', 10.0) + a.set_floats([5.0, 4.0, 3.0]) a.set_floats('floats', [5.0, 4.0, 3.0]) + a.set_int(5) a.set_int('int', 5) + a.set_ints([1, 2, 3]) a.set_ints('ints', [1, 2, 3]) + a.set_string("hello") a.set_string("str", "hello") + a.set_strings(["1", "2", "3"]) a.set_strings('strs', ["1", "2", "3"]) except Exception as e: self.assertEqual(False) From 528ab77e630faf15c01e0cd09f865a158a646c39 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 02:28:57 +0000 Subject: [PATCH 08/18] fix grammar error --- paddle/fluid/jit/property.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h index 1d98fb90da411..1e25bf9668d11 100644 --- a/paddle/fluid/jit/property.h +++ b/paddle/fluid/jit/property.h @@ -30,10 +30,10 @@ class Property { Property() {} // Explicitly implement the copy constructor for auto parallel - explicit Property(const Property &other) + Property(const Property &other) : property_(other.property_), original_id_(other.original_id_) {} - explicit Property &operator=(const Property &other) { + Property &operator=(const Property &other) { property_ = other.property_; original_id_ = other.original_id_; return *this; From 97f2601ea422092a1b3fc7c3cfd97d9dae837cda Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 11:59:44 +0800 Subject: [PATCH 09/18] Update paddle/fluid/jit/property.h Co-authored-by: Aurelius84 --- paddle/fluid/jit/property.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h index 1e25bf9668d11..3b96d0d0680c9 100644 --- a/paddle/fluid/jit/property.h +++ b/paddle/fluid/jit/property.h @@ -30,7 +30,7 @@ class Property { Property() {} // Explicitly implement the copy constructor for auto parallel - Property(const Property &other) + explicit Property(const Property &other) : property_(other.property_), original_id_(other.original_id_) {} Property &operator=(const Property &other) { From 6bd27ea7efdd57674815303d5062d6e51cf60f64 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 12:00:04 +0800 Subject: [PATCH 10/18] Update paddle/fluid/jit/property.cc Co-authored-by: Aurelius84 --- paddle/fluid/jit/property.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index bddd880393831..a8d5b2e25740b 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -81,7 +81,7 @@ void Property::SetFloats(const std::string &name, const std::vector &v) { for (auto i : v) { entry->add_floats(i); } - VLOG(3) << "Property: set_floats " << v.size() << "... name: " << name; + VLOG(3) << "Property: set_floats with length " << v.size() << " for name: " << name; } void Property::SetInt64(const int64_t &i) { From 7c860fa056e1ec84f18e0955b906a462c6980e15 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 12:00:12 +0800 Subject: [PATCH 11/18] Update paddle/fluid/jit/property.cc Co-authored-by: Aurelius84 --- paddle/fluid/jit/property.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index a8d5b2e25740b..09b034449e711 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -128,7 +128,7 @@ void Property::SetString(const std::string &s) { auto entry = property_.add_entrys(); entry->set_type(type); entry->set_s(s); - VLOG(3) << "Property: set_string " << s; + VLOG(3) << "Property: set_string with value : " << s; } void Property::SetString(const std::string &name, const std::string &s) { From 3d7f2c1d307e7db29fcd50603b8aaef3dcbd1f02 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 12:00:19 +0800 Subject: [PATCH 12/18] Update paddle/fluid/jit/property.cc Co-authored-by: Aurelius84 --- paddle/fluid/jit/property.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 09b034449e711..d3bac0642361c 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -70,7 +70,7 @@ void Property::SetFloats(const std::vector &v) { for (auto i : v) { entry->add_floats(i); } - VLOG(3) << "Property: set_floats " << v.size(); + VLOG(3) << "Property: set_floats with length: " << v.size(); } void Property::SetFloats(const std::string &name, const std::vector &v) { From 2765add08c541adab72e39c32571a2fea7aaa898 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 07:47:45 +0000 Subject: [PATCH 13/18] fix comment --- paddle/fluid/jit/property.cc | 12 +++++++----- paddle/fluid/pybind/protobuf.cc | 26 +++++++++++++------------- paddle/fluid/pybind/protobuf.h | 2 +- paddle/fluid/pybind/pybind.cc | 2 +- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index d3bac0642361c..caf7e10a99a23 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -46,20 +46,21 @@ float Property::GetFloat(const std::string &name) const { } } - PADDLE_ENFORCE_EQ(false, true, phi::errors::NotFound("name not found")); + PADDLE_THROW(phi::errors::NotFound("name not found")); return 0; } float Property::GetFloat(const int &idx) const { - PADDLE_ENFORCE_EQ( - idx < Size(), true, phi::errors::OutOfRange("idx out of range")); + PADDLE_ENFORCE_EQ(idx < Size() && idx >= 0, + true, + phi::errors::OutOfRange("idx out of range")); auto e = property_.entrys(idx); if (e.has_f()) { return e.f(); } - PADDLE_ENFORCE_EQ(false, true, phi::errors::NotFound("idx is not a float")); + PADDLE_THROW(phi::errors::NotFound("idx is not a float")); return 0; } @@ -81,7 +82,8 @@ void Property::SetFloats(const std::string &name, const std::vector &v) { for (auto i : v) { entry->add_floats(i); } - VLOG(3) << "Property: set_floats with length " << v.size() << " for name: " << name; + VLOG(3) << "Property: set_floats with length " << v.size() + << " for name: " << name; } void Property::SetInt64(const int64_t &i) { diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index 4cc7440a962ce..b30f5f4257d5c 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -356,7 +356,7 @@ void BindOpDesc(pybind11::module *m) { } // Serialize Class Property -void BindPropertyDesc(pybind11::module *m) { +void BindJitProperty(pybind11::module *m) { pybind11::class_ property(*m, "Property"); property .def( @@ -372,8 +372,8 @@ void BindPropertyDesc(pybind11::module *m) { py::overload_cast( &jit::Property::SetFloat), "set float", - py::arg("val"), - py::arg("name")) + py::arg("name"), + py::arg("var")) .def("get_float", py::overload_cast(&jit::Property::GetFloat, py::const_)) .def("get_float", @@ -388,8 +388,8 @@ void BindPropertyDesc(pybind11::module *m) { py::overload_cast &>( &jit::Property::SetFloats), "set list of float", - py::arg("val"), - py::arg("name")) + py::arg("name"), + py::arg("val")) .def("set_int", py::overload_cast(&jit::Property::SetInt64), "set int", @@ -398,8 +398,8 @@ void BindPropertyDesc(pybind11::module *m) { py::overload_cast( &jit::Property::SetInt64), "set int", - py::arg("val"), - py::arg("name")) + py::arg("name"), + py::arg("val")) .def("set_ints", py::overload_cast &>( &jit::Property::SetInt64s), @@ -409,8 +409,8 @@ void BindPropertyDesc(pybind11::module *m) { py::overload_cast &>( &jit::Property::SetInt64s), "set list of int", - py::arg("val"), - py::arg("name")) + py::arg("name"), + py::arg("val")) .def("set_string", py::overload_cast(&jit::Property::SetString), "set string", @@ -419,8 +419,8 @@ void BindPropertyDesc(pybind11::module *m) { py::overload_cast( &jit::Property::SetString), "set string", - py::arg("val"), - py::arg("name")) + py::arg("name"), + py::arg("val")) .def("set_strings", py::overload_cast &>( &jit::Property::SetStrings), @@ -431,8 +431,8 @@ void BindPropertyDesc(pybind11::module *m) { const std::vector &>( &jit::Property::SetStrings), "set list of string", - py::arg("val"), - py::arg("name")) + py::arg("name"), + py::arg("val")) .def("set_tensor", [](const pd::VarDesc &tensor, const std::string name) { throw platform::errors::Unimplemented("Not implement set_tensor."); diff --git a/paddle/fluid/pybind/protobuf.h b/paddle/fluid/pybind/protobuf.h index 929721678ed1a..9980d0187b5b1 100644 --- a/paddle/fluid/pybind/protobuf.h +++ b/paddle/fluid/pybind/protobuf.h @@ -35,7 +35,7 @@ void BindBlockDesc(pybind11::module* m); void BindVarDsec(pybind11::module* m); void BindOpDesc(pybind11::module* m); void BindProcessMeshDesc(pybind11::module* m); -void BindPropertyDesc(pybind11::module* m); +void BindJitProperty(pybind11::module* m); } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 093a132a09098..4e590646f2477 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -3220,7 +3220,7 @@ All parameter, weight, gradient are variables in Paddle. BindProcessMeshDesc(&m); BindFleetExecutor(&m); BindTCPStore(&m); - BindPropertyDesc(&m); + BindJitProperty(&m); py::class_(m, "LodRankTable") .def("items", [](framework::LoDRankTable &table) { From 37ad0a871fbec570302be46c97a91dca766a8243 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 08:04:18 +0000 Subject: [PATCH 14/18] fix error throw --- paddle/fluid/jit/property.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index caf7e10a99a23..56f22fcf4f532 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -46,7 +46,7 @@ float Property::GetFloat(const std::string &name) const { } } - PADDLE_THROW(phi::errors::NotFound("name not found")); + PADDLE_THROW(phi::errors::NotFound("name: %s not found", name)); return 0; } @@ -60,7 +60,8 @@ float Property::GetFloat(const int &idx) const { return e.f(); } - PADDLE_THROW(phi::errors::NotFound("idx is not a float")); + PADDLE_THROW( + phi::errors::InvalidArgument("get_float: idx (%d) is not a float.", idx)); return 0; } From febae4e5bc27b0cb886cccd3ba6a33f3a9571496 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 09:01:12 +0000 Subject: [PATCH 15/18] fix property save unit test --- .../tests/unittests/dygraph_to_static/test_property_save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py index 16b81c1ebfc37..ea4a7f517a22b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py @@ -41,7 +41,7 @@ def test_size(self): self.assertEqual(self.a.size(), 2) def test_load_float(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(ValueError): self.a.get_float(1) def test_set_float_wo_name(self): From d0864f587eb79c0d3c4d7aca05fcd63874c2fb14 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 20 Jul 2022 11:09:50 +0000 Subject: [PATCH 16/18] fix error info --- paddle/fluid/jit/property.cc | 15 +++++++++------ .../dygraph_to_static/test_property_save.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index 56f22fcf4f532..ca240ef02ea87 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -46,22 +46,25 @@ float Property::GetFloat(const std::string &name) const { } } - PADDLE_THROW(phi::errors::NotFound("name: %s not found", name)); + PADDLE_THROW(phi::errors::NotFound( + "JIT::Property GetFloat: name: %s not found", name)); return 0; } float Property::GetFloat(const int &idx) const { - PADDLE_ENFORCE_EQ(idx < Size() && idx >= 0, - true, - phi::errors::OutOfRange("idx out of range")); + PADDLE_ENFORCE_EQ( + idx < Size() && idx >= 0, + true, + phi::errors::OutOfRange( + "JIT::Property GetFloat: idx=%d out of range %d", idx, Size())); auto e = property_.entrys(idx); if (e.has_f()) { return e.f(); } - PADDLE_THROW( - phi::errors::InvalidArgument("get_float: idx (%d) is not a float.", idx)); + PADDLE_THROW(phi::errors::InvalidArgument( + "JIT::Property GetFloat: input idx (%d) element is not a float.", idx)); return 0; } diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py index ea4a7f517a22b..59c67bfcb49a6 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_property_save.py @@ -69,7 +69,7 @@ def test_set(self): a.set_strings(["1", "2", "3"]) a.set_strings('strs', ["1", "2", "3"]) except Exception as e: - self.assertEqual(False) + self.assertEqual(False, True) if __name__ == '__main__': From 2ed5590a9bc130becdcf083c69a86f581a0c90e9 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Thu, 21 Jul 2022 06:23:23 +0000 Subject: [PATCH 17/18] fix copyright and header import --- paddle/fluid/jit/property.cc | 3 ++- paddle/fluid/jit/property.h | 1 - paddle/fluid/jit/property.proto | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/jit/property.cc b/paddle/fluid/jit/property.cc index ca240ef02ea87..ea24757ff66c9 100644 --- a/paddle/fluid/jit/property.cc +++ b/paddle/fluid/jit/property.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/jit/property.h" +#include "glog/logging.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/errors.h" diff --git a/paddle/fluid/jit/property.h b/paddle/fluid/jit/property.h index 3b96d0d0680c9..10b021ba5bea2 100644 --- a/paddle/fluid/jit/property.h +++ b/paddle/fluid/jit/property.h @@ -19,7 +19,6 @@ #include #include -#include "glog/logging.h" #include "paddle/fluid/jit/property.pb.h" namespace paddle { diff --git a/paddle/fluid/jit/property.proto b/paddle/fluid/jit/property.proto index 0b83886116a2a..9fed1c4d66a8f 100644 --- a/paddle/fluid/jit/property.proto +++ b/paddle/fluid/jit/property.proto @@ -1,8 +1,11 @@ -/*Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + +http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,6 +43,8 @@ message TensorProto { // floating-point number truncated to 16 bits. // This format has 1 sign bit, 8 exponent bits, and 7 mantissa bits. BFLOAT16 = 16; + + // Future extensions go here. } optional bool stop_gradient = 1; From 27c8d0baeb57498598a204a6257b4364e45305be Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Thu, 21 Jul 2022 07:21:42 +0000 Subject: [PATCH 18/18] reorder jit property tensor datatype --- paddle/fluid/jit/property.proto | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/paddle/fluid/jit/property.proto b/paddle/fluid/jit/property.proto index 9fed1c4d66a8f..5f89e1da90b91 100644 --- a/paddle/fluid/jit/property.proto +++ b/paddle/fluid/jit/property.proto @@ -18,26 +18,28 @@ package paddle.jit.proto; message TensorProto { enum DataType { UNDEFINED = 0; - // Basic types. - FLOAT = 1; // float + BOOL = 1; // bool + UINT8 = 2; // uint8_t INT8 = 3; // int8_t UINT16 = 4; // uint16_t INT16 = 5; // int16_t - INT32 = 6; // int32_t - INT64 = 7; // int64_t - STRING = 8; // string - BOOL = 9; // bool + UINT32 = 6; // uint32_t + INT32 = 7; // int32_t + UINT64 = 8; // uint64_t + INT64 = 9; // int64_t + + FLOAT = 10; // float + DOUBLE = 11; + + COMPLEX64 = 12; // complex with float32 real and imaginary components + COMPLEX128 = 13; // complex with float64 real and imaginary components + + STRING = 14; // string // IEEE754 half-precision floating-point format (16 bits wide). // This format has 1 sign bit, 5 exponent bits, and 10 mantissa bits. - FLOAT16 = 10; - - DOUBLE = 11; - UINT32 = 12; - UINT64 = 13; - COMPLEX64 = 14; // complex with float32 real and imaginary components - COMPLEX128 = 15; // complex with float64 real and imaginary components + FLOAT16 = 15; // Non-IEEE floating-point format based on IEEE754 single-precision // floating-point number truncated to 16 bits.