From a4b7bb9a5dd2c021edcd3d68d326255c734d0ef0 Mon Sep 17 00:00:00 2001 From: David Mansfield Date: Tue, 28 Jun 2016 12:59:10 -0400 Subject: [PATCH] Compatibility with tensorflow master (#204) * TF defines the same external repositories but with different commit and BUILD * Fix a bunch of bazel namespace issues @tf changed to @org_tensorflow Tensorflow stopped using protobuf as submodule, now it's external, so use @protobuf * Sync to master of tensorflow submodule: - local version of is_cuda in syntaxnet.bzl, - using unique_ptr in file api. --- syntaxnet/WORKSPACE | 14 ++++----- syntaxnet/syntaxnet/BUILD | 22 +++++++------- syntaxnet/syntaxnet/proto_io.h | 32 +++++++++++---------- syntaxnet/syntaxnet/reader_ops.cc | 7 ++--- syntaxnet/syntaxnet/sentence_features.cc | 4 +-- syntaxnet/syntaxnet/syntaxnet.bzl | 35 ++++++++++------------- syntaxnet/syntaxnet/term_frequency_map.cc | 14 ++++----- syntaxnet/tensorflow | 2 +- syntaxnet/util/utf8/BUILD | 2 +- 9 files changed, 62 insertions(+), 70 deletions(-) diff --git a/syntaxnet/WORKSPACE b/syntaxnet/WORKSPACE index 49643efe6cb..ce49168d57a 100644 --- a/syntaxnet/WORKSPACE +++ b/syntaxnet/WORKSPACE @@ -1,24 +1,24 @@ local_repository( - name = "tf", + name = "org_tensorflow", path = __workspace_dir__ + "/tensorflow", ) load('//tensorflow/tensorflow:workspace.bzl', 'tf_workspace') -tf_workspace("tensorflow/", "@tf") +tf_workspace("tensorflow/", "@org_tensorflow") # Specify the minimum required Bazel version. -load("@tf//tensorflow:tensorflow.bzl", "check_version") +load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") check_version("0.2.0") # ===== gRPC dependencies ===== bind( name = "libssl", - actual = "@boringssl_git//:ssl", + actual = "@ts_boringssl_git//:ssl", ) git_repository( - name = "boringssl_git", + name = "ts_boringssl_git", commit = "436432d849b83ab90f18773e4ae1c7a8f148f48d", init_submodules = True, remote = "https://github.com/mdsteele/boringssl-bazel.git", @@ -26,11 +26,11 @@ git_repository( bind( name = "zlib", - actual = "@zlib_archive//:zlib", + actual = "@ts_zlib_archive//:zlib", ) new_http_archive( - name = "zlib_archive", + name = "ts_zlib_archive", build_file = "zlib.BUILD", sha256 = "879d73d8cd4d155f31c1f04838ecd567d34bebda780156f0e82a20721b3973d5", strip_prefix = "zlib-1.2.8", diff --git a/syntaxnet/syntaxnet/BUILD b/syntaxnet/syntaxnet/BUILD index c4a9c235129..29bbb85cf56 100644 --- a/syntaxnet/syntaxnet/BUILD +++ b/syntaxnet/syntaxnet/BUILD @@ -77,15 +77,15 @@ cc_library( visibility = ["//visibility:public"], deps = [ "@re2//:re2", - "@tf//google/protobuf", - "@tf//third_party/eigen3", + "@protobuf//:protobuf", + "@org_tensorflow//third_party/eigen3", ] + select({ "//conditions:default": [ - "@tf//tensorflow/core:framework", - "@tf//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", ], - "@tf//tensorflow:darwin": [ - "@tf//tensorflow/core:framework_headers_lib", + "@org_tensorflow//tensorflow:darwin": [ + "@org_tensorflow//tensorflow/core:framework_headers_lib", ], }), ) @@ -108,8 +108,8 @@ cc_library( srcs = ["test_main.cc"], linkopts = ["-lm"], deps = [ - "@tf//tensorflow/core:lib", - "@tf//tensorflow/core:testlib", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:testlib", "//external:gtest", ], ) @@ -409,7 +409,7 @@ cc_binary( name = "parser_ops.so", linkopts = select({ "//conditions:default": ["-lm"], - "@tf//tensorflow:darwin": [], + "@org_tensorflow//tensorflow:darwin": [], }), linkshared = 1, linkstatic = 1, @@ -519,8 +519,8 @@ py_library( name = "graph_builder", srcs = ["graph_builder.py"], deps = [ - "@tf//tensorflow:tensorflow_py", - "@tf//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/core:protos_all_py", ":load_parser_ops_py", ":parser_ops", ], diff --git a/syntaxnet/syntaxnet/proto_io.h b/syntaxnet/syntaxnet/proto_io.h index e668acad8ea..9d5bb441832 100644 --- a/syntaxnet/syntaxnet/proto_io.h +++ b/syntaxnet/syntaxnet/proto_io.h @@ -43,18 +43,19 @@ namespace syntaxnet { // A convenience wrapper to read protos with a RecordReader. class ProtoRecordReader { public: - explicit ProtoRecordReader(tensorflow::RandomAccessFile *file) - : file_(file), reader_(new tensorflow::io::RecordReader(file_)) {} + explicit ProtoRecordReader(tensorflow::RandomAccessFile *file) { + file_.reset(file); + reader_.reset(new tensorflow::io::RecordReader(file_.get())); + } explicit ProtoRecordReader(const string &filename) { TF_CHECK_OK( tensorflow::Env::Default()->NewRandomAccessFile(filename, &file_)); - reader_.reset(new tensorflow::io::RecordReader(file_)); + reader_.reset(new tensorflow::io::RecordReader(file_.get())); } ~ProtoRecordReader() { reader_.reset(); - delete file_; } template @@ -70,9 +71,9 @@ class ProtoRecordReader { } private: - tensorflow::RandomAccessFile *file_ = nullptr; uint64 offset_ = 0; std::unique_ptr reader_; + std::unique_ptr file_; }; // A convenience wrapper to write protos with a RecordReader. @@ -80,12 +81,12 @@ class ProtoRecordWriter { public: explicit ProtoRecordWriter(const string &filename) { TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file_)); - writer_.reset(new tensorflow::io::RecordWriter(file_)); + writer_.reset(new tensorflow::io::RecordWriter(file_.get())); } ~ProtoRecordWriter() { writer_.reset(); - delete file_; + file_.reset(); } template @@ -94,8 +95,8 @@ class ProtoRecordWriter { } private: - tensorflow::WritableFile *file_ = nullptr; std::unique_ptr writer_; + std::unique_ptr file_; }; // A file implementation to read from stdin. @@ -176,22 +177,24 @@ class TextReader { void Reset() { sentence_count_ = 0; - tensorflow::RandomAccessFile *file; if (filename_ == "-") { static const int kInputBufferSize = 8 * 1024; /* bytes */ - file = new StdIn(); - buffer_.reset(new tensorflow::io::InputBuffer(file, kInputBufferSize)); + file_.reset(new StdIn()); + buffer_.reset( + new tensorflow::io::InputBuffer(file_.get(), kInputBufferSize)); } else { static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */ TF_CHECK_OK( - tensorflow::Env::Default()->NewRandomAccessFile(filename_, &file)); - buffer_.reset(new tensorflow::io::InputBuffer(file, kInputBufferSize)); + tensorflow::Env::Default()->NewRandomAccessFile(filename_, &file_)); + buffer_.reset( + new tensorflow::io::InputBuffer(file_.get(), kInputBufferSize)); } } private: string filename_; int sentence_count_ = 0; + std::unique_ptr file_; std::unique_ptr buffer_; std::unique_ptr format_; }; @@ -217,7 +220,6 @@ class TextWriter { ~TextWriter() { if (file_) { file_->Close(); - delete file_; } } @@ -234,7 +236,7 @@ class TextWriter { private: string filename_; std::unique_ptr format_; - tensorflow::WritableFile *file_ = nullptr; + std::unique_ptr file_; }; } // namespace syntaxnet diff --git a/syntaxnet/syntaxnet/reader_ops.cc b/syntaxnet/syntaxnet/reader_ops.cc index fd2d55a0fd2..100e9d0cef7 100644 --- a/syntaxnet/syntaxnet/reader_ops.cc +++ b/syntaxnet/syntaxnet/reader_ops.cc @@ -514,19 +514,16 @@ class WordEmbeddingInitializer : public OpKernel { static tensorflow::Status CopyToTmpPath(const string &source_path, string *tmp_path) { // Opens source file. - tensorflow::RandomAccessFile *source_file; + std::unique_ptr source_file; TF_RETURN_IF_ERROR(tensorflow::Env::Default()->NewRandomAccessFile( source_path, &source_file)); - std::unique_ptr source_file_deleter( - source_file); // Creates destination file. - tensorflow::WritableFile *target_file; + std::unique_ptr target_file; *tmp_path = tensorflow::strings::Printf( "/tmp/%d.%lld", getpid(), tensorflow::Env::Default()->NowMicros()); TF_RETURN_IF_ERROR( tensorflow::Env::Default()->NewWritableFile(*tmp_path, &target_file)); - std::unique_ptr target_file_deleter(target_file); // Performs copy. tensorflow::Status s; diff --git a/syntaxnet/syntaxnet/sentence_features.cc b/syntaxnet/syntaxnet/sentence_features.cc index 7bb7e7d660e..fa874d53ade 100644 --- a/syntaxnet/syntaxnet/sentence_features.cc +++ b/syntaxnet/syntaxnet/sentence_features.cc @@ -120,9 +120,9 @@ string AffixTableFeature::WorkspaceName() const { static AffixTable *CreateAffixTable(const string &filename, AffixTable::Type type) { AffixTable *affix_table = new AffixTable(type, 1); - tensorflow::RandomAccessFile *file; + std::unique_ptr file; TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file)); - ProtoRecordReader reader(file); + ProtoRecordReader reader(file.release()); affix_table->Read(&reader); return affix_table; } diff --git a/syntaxnet/syntaxnet/syntaxnet.bzl b/syntaxnet/syntaxnet/syntaxnet.bzl index 7f0a395299d..95fb6aa749c 100644 --- a/syntaxnet/syntaxnet/syntaxnet.bzl +++ b/syntaxnet/syntaxnet/syntaxnet.bzl @@ -13,26 +13,21 @@ # limitations under the License. # ============================================================================== -load("@tf//google/protobuf:protobuf.bzl", "cc_proto_library") -load("@tf//google/protobuf:protobuf.bzl", "py_proto_library") +load("@protobuf//:protobuf.bzl", "cc_proto_library") +load("@protobuf//:protobuf.bzl", "py_proto_library") def if_cuda(if_true, if_false = []): - """Shorthand for select()'ing on whether we're building with CUDA. - - Returns a select statement which evaluates to if_true if we're building - with CUDA enabled. Otherwise, the select statement evaluates to if_false. - - """ + """Shorthand for select()'ing on whether we're building with CUDA.""" return select({ - "@tf//third_party/gpus/cuda:using_nvcc": if_true, - "@tf//third_party/gpus/cuda:using_gcudacc": if_true, + "@org_tensorflow//third_party/gpus/cuda:using_nvcc": if_true, + "@org_tensorflow//third_party/gpus/cuda:using_gcudacc": if_true, "//conditions:default": if_false }) def tf_copts(): return (["-fno-exceptions", "-DEIGEN_AVOID_STL_ARRAY",] + if_cuda(["-DGOOGLE_CUDA=1"]) + - select({"@tf//tensorflow:darwin": [], + select({"@org_tensorflow//tensorflow:darwin": [], "//conditions:default": ["-pthread"]})) def tf_proto_library(name, srcs=[], has_services=False, @@ -47,9 +42,9 @@ def tf_proto_library(name, srcs=[], has_services=False, cc_proto_library(name=name, srcs=srcs, deps=deps, - cc_libs = ["@tf//google/protobuf:protobuf"], - protoc="@tf//google/protobuf:protoc", - default_runtime="@tf//google/protobuf:protobuf", + cc_libs = ["@protobuf//:protobuf"], + protoc="@protobuf//:protoc", + default_runtime="@protobuf//:protobuf", testonly=testonly, visibility=visibility,) @@ -58,8 +53,8 @@ def tf_proto_library_py(name, srcs=[], deps=[], visibility=None, testonly=0): srcs=srcs, srcs_version = "PY2AND3", deps=deps, - default_runtime="@tf//google/protobuf:protobuf_python", - protoc="@tf//google/protobuf:protoc", + default_runtime="@protobuf//:protobuf_python", + protoc="@protobuf//:protoc", visibility=visibility, testonly=testonly,) @@ -72,7 +67,7 @@ def tf_gen_op_libs(op_lib_names): native.cc_library(name=n + "_op_lib", copts=tf_copts(), srcs=["ops/" + n + ".cc"], - deps=(["@tf//tensorflow/core:framework"]), + deps=(["@org_tensorflow//tensorflow/core:framework"]), visibility=["//visibility:public"], alwayslink=1, linkstatic=1,) @@ -89,8 +84,8 @@ def tf_gen_op_wrapper_py(name, out=None, hidden=[], visibility=None, deps=[], linkopts = ["-lm"], copts = tf_copts(), linkstatic = 1, # Faster to link this one-time-use binary dynamically - deps = (["@tf//tensorflow/core:framework", - "@tf//tensorflow/python:python_op_gen_main"] + deps), + deps = (["@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/python:python_op_gen_main"] + deps), ) # Invoke the previous cc_binary to generate a python file. @@ -110,5 +105,5 @@ def tf_gen_op_wrapper_py(name, out=None, hidden=[], visibility=None, deps=[], srcs_version="PY2AND3", visibility=visibility, deps=[ - "@tf//tensorflow/python:framework_for_generated_wrappers", + "@org_tensorflow//tensorflow/python:framework_for_generated_wrappers", ],) diff --git a/syntaxnet/syntaxnet/term_frequency_map.cc b/syntaxnet/syntaxnet/term_frequency_map.cc index 21267c53cbc..30aa38afd34 100644 --- a/syntaxnet/syntaxnet/term_frequency_map.cc +++ b/syntaxnet/syntaxnet/term_frequency_map.cc @@ -58,10 +58,10 @@ void TermFrequencyMap::Load(const string &filename, int min_frequency, if (max_num_terms <= 0) max_num_terms = std::numeric_limits::max(); // Read the first line (total # of terms in the mapping). - tensorflow::RandomAccessFile *file; + std::unique_ptr file; TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file)); static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */ - tensorflow::io::InputBuffer input(file, kInputBufferSize); + tensorflow::io::InputBuffer input(file.get(), kInputBufferSize); string line; TF_CHECK_OK(input.ReadLine(&line)); int32 total = -1; @@ -119,7 +119,7 @@ void TermFrequencyMap::Save(const string &filename) const { std::sort(sorted_data.begin(), sorted_data.end(), SortByFrequencyThenTerm()); // Write the number of terms. - tensorflow::WritableFile *file; + std::unique_ptr file; TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file)); CHECK_LE(term_index_.size(), std::numeric_limits::max()); // overflow const int32 num_terms = term_index_.size(); @@ -136,15 +136,14 @@ void TermFrequencyMap::Save(const string &filename) const { TF_CHECK_OK(file->Close()) << "for file " << filename; LOG(INFO) << "Saved " << term_index_.size() << " terms to " << filename << "."; - delete file; } TagToCategoryMap::TagToCategoryMap(const string &filename) { // Load the mapping. - tensorflow::RandomAccessFile *file; + std::unique_ptr file; TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file)); static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */ - tensorflow::io::InputBuffer input(file, kInputBufferSize); + tensorflow::io::InputBuffer input(file.get(), kInputBufferSize); string line; while (input.ReadLine(&line) == tensorflow::Status::OK()) { vector pair = utils::Split(line, '\t'); @@ -174,7 +173,7 @@ void TagToCategoryMap::SetCategory(const string &tag, const string &category) { void TagToCategoryMap::Save(const string &filename) const { // Write tag and category on each line. - tensorflow::WritableFile *file; + std::unique_ptr file; TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file)); for (const auto &pair : tag_to_category_) { const string line = @@ -182,7 +181,6 @@ void TagToCategoryMap::Save(const string &filename) const { TF_CHECK_OK(file->Append(line)); } TF_CHECK_OK(file->Close()) << "for file " << filename; - delete file; } } // namespace syntaxnet diff --git a/syntaxnet/tensorflow b/syntaxnet/tensorflow index 712e41cf8b3..861644c0bca 160000 --- a/syntaxnet/tensorflow +++ b/syntaxnet/tensorflow @@ -1 +1 @@ -Subproject commit 712e41cf8b316ef2c33c6dd7fd6ade2b4e93ddc0 +Subproject commit 861644c0bcae5d56f7b3f439696eefa6df8580ec diff --git a/syntaxnet/util/utf8/BUILD b/syntaxnet/util/utf8/BUILD index 829c21d285d..46bbe7a6bcc 100644 --- a/syntaxnet/util/utf8/BUILD +++ b/syntaxnet/util/utf8/BUILD @@ -27,7 +27,7 @@ cc_test( "unicodetext_unittest.cc", ], deps = [ - "@tf//tensorflow/core:testlib", + "@org_tensorflow//tensorflow/core:testlib", ":unicodetext", ], )