Compatibility with tensorflow master (#204)

* TF defines the same external repositories but with different commit and BUILD

* Fix a bunch of bazel namespace issues

@tf changed to @org_tensorflow
Tensorflow stopped using protobuf as submodule, now it's external, so use @protobuf

* Sync to master of tensorflow submodule:
  - local version of is_cuda in syntaxnet.bzl,
  - using unique_ptr in file api.

syntaxnet/WORKSPACE

@@ -1,36 +1,36 @@
 local_repository(
- name = "tf",
+ name = "org_tensorflow",
  path = __workspace_dir__ + "/tensorflow",
 )
 
 load('//tensorflow/tensorflow:workspace.bzl', 'tf_workspace')
-tf_workspace("tensorflow/", "@tf")
+tf_workspace("tensorflow/", "@org_tensorflow")
 
 # Specify the minimum required Bazel version.
-load("@tf//tensorflow:tensorflow.bzl", "check_version")
+load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version")
 check_version("0.2.0")
 
 # ===== gRPC dependencies =====
 
 bind(
  name = "libssl",
- actual = "@boringssl_git//:ssl",
+ actual = "@ts_boringssl_git//:ssl",
 )
 
 git_repository(
- name = "boringssl_git",
+ name = "ts_boringssl_git",
  commit = "436432d849b83ab90f18773e4ae1c7a8f148f48d",
  init_submodules = True,
  remote = "https://github.com/mdsteele/boringssl-bazel.git",
 )
 
 bind(
  name = "zlib",
- actual = "@zlib_archive//:zlib",
+ actual = "@ts_zlib_archive//:zlib",
 )
 
 new_http_archive(
- name = "zlib_archive",
+ name = "ts_zlib_archive",
  build_file = "zlib.BUILD",
  sha256 = "879d73d8cd4d155f31c1f04838ecd567d34bebda780156f0e82a20721b3973d5",
  strip_prefix = "zlib-1.2.8",

syntaxnet/syntaxnet/BUILD

@@ -77,15 +77,15 @@ cc_library(
  visibility = ["//visibility:public"],
  deps = [
  "@re2//:re2",
- "@tf//google/protobuf",
- "@tf//third_party/eigen3",
+ "@protobuf//:protobuf",
+ "@org_tensorflow//third_party/eigen3",
  ] + select({
  "//conditions:default": [
- "@tf//tensorflow/core:framework",
- "@tf//tensorflow/core:lib",
+ "@org_tensorflow//tensorflow/core:framework",
+ "@org_tensorflow//tensorflow/core:lib",
  ],
- "@tf//tensorflow:darwin": [
- "@tf//tensorflow/core:framework_headers_lib",
+ "@org_tensorflow//tensorflow:darwin": [
+ "@org_tensorflow//tensorflow/core:framework_headers_lib",
  ],
  }),
 )
@@ -108,8 +108,8 @@ cc_library(
  srcs = ["test_main.cc"],
  linkopts = ["-lm"],
  deps = [
- "@tf//tensorflow/core:lib",
- "@tf//tensorflow/core:testlib",
+ "@org_tensorflow//tensorflow/core:lib",
+ "@org_tensorflow//tensorflow/core:testlib",
  "//external:gtest",
  ],
 )
@@ -409,7 +409,7 @@ cc_binary(
  name = "parser_ops.so",
  linkopts = select({
  "//conditions:default": ["-lm"],
- "@tf//tensorflow:darwin": [],
+ "@org_tensorflow//tensorflow:darwin": [],
  }),
  linkshared = 1,
  linkstatic = 1,
@@ -519,8 +519,8 @@ py_library(
  name = "graph_builder",
  srcs = ["graph_builder.py"],
  deps = [
- "@tf//tensorflow:tensorflow_py",
- "@tf//tensorflow/core:protos_all_py",
+ "@org_tensorflow//tensorflow:tensorflow_py",
+ "@org_tensorflow//tensorflow/core:protos_all_py",
  ":load_parser_ops_py",
  ":parser_ops",
  ],

syntaxnet/syntaxnet/proto_io.h

@@ -43,18 +43,19 @@ namespace syntaxnet {
 // A convenience wrapper to read protos with a RecordReader.
 class ProtoRecordReader {
  public:
- explicit ProtoRecordReader(tensorflow::RandomAccessFile *file)
- : file_(file), reader_(new tensorflow::io::RecordReader(file_)) {}
+ explicit ProtoRecordReader(tensorflow::RandomAccessFile *file) {
+ file_.reset(file);
+ reader_.reset(new tensorflow::io::RecordReader(file_.get()));
+ }
 
  explicit ProtoRecordReader(const string &filename) {
  TF_CHECK_OK(
  tensorflow::Env::Default()->NewRandomAccessFile(filename, &file_));
- reader_.reset(new tensorflow::io::RecordReader(file_));
+ reader_.reset(new tensorflow::io::RecordReader(file_.get()));
  }
 
  ~ProtoRecordReader() {
  reader_.reset();
- delete file_;
  }
 
  template <typename T>
@@ -70,22 +71,22 @@ class ProtoRecordReader {
  }
 
  private:
- tensorflow::RandomAccessFile *file_ = nullptr;
  uint64 offset_ = 0;
  std::unique_ptr<tensorflow::io::RecordReader> reader_;
+ std::unique_ptr<tensorflow::RandomAccessFile> file_;
 };
 
 // A convenience wrapper to write protos with a RecordReader.
 class ProtoRecordWriter {
  public:
  explicit ProtoRecordWriter(const string &filename) {
  TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file_));
- writer_.reset(new tensorflow::io::RecordWriter(file_));
+ writer_.reset(new tensorflow::io::RecordWriter(file_.get()));
  }
 
  ~ProtoRecordWriter() {
  writer_.reset();
- delete file_;
+ file_.reset();
  }
 
  template <typename T>
@@ -94,8 +95,8 @@ class ProtoRecordWriter {
  }
 
  private:
- tensorflow::WritableFile *file_ = nullptr;
  std::unique_ptr<tensorflow::io::RecordWriter> writer_;
+ std::unique_ptr<tensorflow::WritableFile> file_;
 };
 
 // A file implementation to read from stdin.
@@ -176,22 +177,24 @@ class TextReader {
 
  void Reset() {
  sentence_count_ = 0;
- tensorflow::RandomAccessFile *file;
  if (filename_ == "-") {
  static const int kInputBufferSize = 8 * 1024; /* bytes */
- file = new StdIn();
- buffer_.reset(new tensorflow::io::InputBuffer(file, kInputBufferSize));
+ file_.reset(new StdIn());
+ buffer_.reset(
+ new tensorflow::io::InputBuffer(file_.get(), kInputBufferSize));
  } else {
  static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */
  TF_CHECK_OK(
- tensorflow::Env::Default()->NewRandomAccessFile(filename_, &file));
- buffer_.reset(new tensorflow::io::InputBuffer(file, kInputBufferSize));
+ tensorflow::Env::Default()->NewRandomAccessFile(filename_, &file_));
+ buffer_.reset(
+ new tensorflow::io::InputBuffer(file_.get(), kInputBufferSize));
  }
  }
 
  private:
  string filename_;
  int sentence_count_ = 0;
+ std::unique_ptr<tensorflow::RandomAccessFile> file_;
  std::unique_ptr<tensorflow::io::InputBuffer> buffer_;
  std::unique_ptr<DocumentFormat> format_;
 };
@@ -217,7 +220,6 @@ class TextWriter {
  ~TextWriter() {
  if (file_) {
  file_->Close();
- delete file_;
  }
  }
 
@@ -234,7 +236,7 @@ class TextWriter {
  private:
  string filename_;
  std::unique_ptr<DocumentFormat> format_;
- tensorflow::WritableFile *file_ = nullptr;
+ std::unique_ptr<tensorflow::WritableFile> file_;
 };
 
 } // namespace syntaxnet

syntaxnet/syntaxnet/reader_ops.cc

@@ -514,19 +514,16 @@ class WordEmbeddingInitializer : public OpKernel {
  static tensorflow::Status CopyToTmpPath(const string &source_path,
  string *tmp_path) {
  // Opens source file.
- tensorflow::RandomAccessFile *source_file;
+ std::unique_ptr<tensorflow::RandomAccessFile> source_file;
  TF_RETURN_IF_ERROR(tensorflow::Env::Default()->NewRandomAccessFile(
  source_path, &source_file));
- std::unique_ptr<tensorflow::RandomAccessFile> source_file_deleter(
- source_file);
 
  // Creates destination file.
- tensorflow::WritableFile *target_file;
+ std::unique_ptr<tensorflow::WritableFile> target_file;
  *tmp_path = tensorflow::strings::Printf(
  "/tmp/%d.%lld", getpid(), tensorflow::Env::Default()->NowMicros());
  TF_RETURN_IF_ERROR(
  tensorflow::Env::Default()->NewWritableFile(*tmp_path, &target_file));
- std::unique_ptr<tensorflow::WritableFile> target_file_deleter(target_file);
 
  // Performs copy.
  tensorflow::Status s;

syntaxnet/syntaxnet/sentence_features.cc

@@ -120,9 +120,9 @@ string AffixTableFeature::WorkspaceName() const {
 static AffixTable *CreateAffixTable(const string &filename,
  AffixTable::Type type) {
  AffixTable *affix_table = new AffixTable(type, 1);
- tensorflow::RandomAccessFile *file;
+ std::unique_ptr<tensorflow::RandomAccessFile> file;
  TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file));
- ProtoRecordReader reader(file);
+ ProtoRecordReader reader(file.release());
  affix_table->Read(&reader);
  return affix_table;
 }

syntaxnet/syntaxnet/syntaxnet.bzl

@@ -13,26 +13,21 @@
 # limitations under the License.
 # ==============================================================================
 
-load("@tf//google/protobuf:protobuf.bzl", "cc_proto_library")
-load("@tf//google/protobuf:protobuf.bzl", "py_proto_library")
+load("@protobuf//:protobuf.bzl", "cc_proto_library")
+load("@protobuf//:protobuf.bzl", "py_proto_library")
 
 def if_cuda(if_true, if_false = []):
- """Shorthand for select()'ing on whether we're building with CUDA.
-
- Returns a select statement which evaluates to if_true if we're building
- with CUDA enabled. Otherwise, the select statement evaluates to if_false.
-
- """
+ """Shorthand for select()'ing on whether we're building with CUDA."""
  return select({
- "@tf//third_party/gpus/cuda:using_nvcc": if_true,
- "@tf//third_party/gpus/cuda:using_gcudacc": if_true,
+ "@org_tensorflow//third_party/gpus/cuda:using_nvcc": if_true,
+ "@org_tensorflow//third_party/gpus/cuda:using_gcudacc": if_true,
  "//conditions:default": if_false
  })
 
 def tf_copts():
  return (["-fno-exceptions", "-DEIGEN_AVOID_STL_ARRAY",] +
  if_cuda(["-DGOOGLE_CUDA=1"]) +
- select({"@tf//tensorflow:darwin": [],
+ select({"@org_tensorflow//tensorflow:darwin": [],
  "//conditions:default": ["-pthread"]}))
 
 def tf_proto_library(name, srcs=[], has_services=False,
@@ -47,9 +42,9 @@ def tf_proto_library(name, srcs=[], has_services=False,
  cc_proto_library(name=name,
  srcs=srcs,
  deps=deps,
- cc_libs = ["@tf//google/protobuf:protobuf"],
- protoc="@tf//google/protobuf:protoc",
- default_runtime="@tf//google/protobuf:protobuf",
+ cc_libs = ["@protobuf//:protobuf"],
+ protoc="@protobuf//:protoc",
+ default_runtime="@protobuf//:protobuf",
  testonly=testonly,
  visibility=visibility,)
 
@@ -58,8 +53,8 @@ def tf_proto_library_py(name, srcs=[], deps=[], visibility=None, testonly=0):
  srcs=srcs,
  srcs_version = "PY2AND3",
  deps=deps,
- default_runtime="@tf//google/protobuf:protobuf_python",
- protoc="@tf//google/protobuf:protoc",
+ default_runtime="@protobuf//:protobuf_python",
+ protoc="@protobuf//:protoc",
  visibility=visibility,
  testonly=testonly,)
 
@@ -72,7 +67,7 @@ def tf_gen_op_libs(op_lib_names):
  native.cc_library(name=n + "_op_lib",
  copts=tf_copts(),
  srcs=["ops/" + n + ".cc"],
- deps=(["@tf//tensorflow/core:framework"]),
+ deps=(["@org_tensorflow//tensorflow/core:framework"]),
  visibility=["//visibility:public"],
  alwayslink=1,
  linkstatic=1,)
@@ -89,8 +84,8 @@ def tf_gen_op_wrapper_py(name, out=None, hidden=[], visibility=None, deps=[],
  linkopts = ["-lm"],
  copts = tf_copts(),
  linkstatic = 1, # Faster to link this one-time-use binary dynamically
- deps = (["@tf//tensorflow/core:framework",
- "@tf//tensorflow/python:python_op_gen_main"] + deps),
+ deps = (["@org_tensorflow//tensorflow/core:framework",
+ "@org_tensorflow//tensorflow/python:python_op_gen_main"] + deps),
  )
 
  # Invoke the previous cc_binary to generate a python file.
@@ -110,5 +105,5 @@ def tf_gen_op_wrapper_py(name, out=None, hidden=[], visibility=None, deps=[],
  srcs_version="PY2AND3",
  visibility=visibility,
  deps=[
- "@tf//tensorflow/python:framework_for_generated_wrappers",
+ "@org_tensorflow//tensorflow/python:framework_for_generated_wrappers",
  ],)

syntaxnet/syntaxnet/term_frequency_map.cc

@@ -58,10 +58,10 @@ void TermFrequencyMap::Load(const string &filename, int min_frequency,
  if (max_num_terms <= 0) max_num_terms = std::numeric_limits<int>::max();
 
  // Read the first line (total # of terms in the mapping).
- tensorflow::RandomAccessFile *file;
+ std::unique_ptr<tensorflow::RandomAccessFile> file;
  TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file));
  static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */
- tensorflow::io::InputBuffer input(file, kInputBufferSize);
+ tensorflow::io::InputBuffer input(file.get(), kInputBufferSize);
  string line;
  TF_CHECK_OK(input.ReadLine(&line));
  int32 total = -1;
@@ -119,7 +119,7 @@ void TermFrequencyMap::Save(const string &filename) const {
  std::sort(sorted_data.begin(), sorted_data.end(), SortByFrequencyThenTerm());
 
  // Write the number of terms.
- tensorflow::WritableFile *file;
+ std::unique_ptr<tensorflow::WritableFile> file;
  TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file));
  CHECK_LE(term_index_.size(), std::numeric_limits<int32>::max()); // overflow
  const int32 num_terms = term_index_.size();
@@ -136,15 +136,14 @@ void TermFrequencyMap::Save(const string &filename) const {
  TF_CHECK_OK(file->Close()) << "for file " << filename;
  LOG(INFO) << "Saved " << term_index_.size() << " terms to " << filename
  << ".";
- delete file;
 }
 
 TagToCategoryMap::TagToCategoryMap(const string &filename) {
  // Load the mapping.
- tensorflow::RandomAccessFile *file;
+ std::unique_ptr<tensorflow::RandomAccessFile> file;
  TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file));
  static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */
- tensorflow::io::InputBuffer input(file, kInputBufferSize);
+ tensorflow::io::InputBuffer input(file.get(), kInputBufferSize);
  string line;
  while (input.ReadLine(&line) == tensorflow::Status::OK()) {
  vector<string> pair = utils::Split(line, '\t');
@@ -174,15 +173,14 @@ void TagToCategoryMap::SetCategory(const string &tag, const string &category) {
 
 void TagToCategoryMap::Save(const string &filename) const {
  // Write tag and category on each line.
- tensorflow::WritableFile *file;
+ std::unique_ptr<tensorflow::WritableFile> file;
  TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file));
  for (const auto &pair : tag_to_category_) {
  const string line =
  tensorflow::strings::StrCat(pair.first, "\t", pair.second, "\n");
  TF_CHECK_OK(file->Append(line));
  }
  TF_CHECK_OK(file->Close()) << "for file " << filename;
- delete file;
 }
 
 } // namespace syntaxnet

syntaxnet/util/utf8/BUILD

@@ -27,7 +27,7 @@ cc_test(
  "unicodetext_unittest.cc",
  ],
  deps = [
- "@tf//tensorflow/core:testlib",
+ "@org_tensorflow//tensorflow/core:testlib",
  ":unicodetext",
  ],
 )