Merge branch 'master' into master

CMakeLists.txt

@@ -22,6 +22,7 @@ cmake_minimum_required(VERSION 3.2)
 # There is no C per se in WDT but if you use CXX only here many checks fail
 # Version is Major.Minor.YYMMDDX for up to 10 releases per day (X from 0 to 9)
 # Minor currently is also the protocol version - has to match with Protocol.cpp
+
 project("WDT" LANGUAGES C CXX VERSION 1.31.1704270)
 
 # On MacOS this requires the latest (master) CMake (and/or CMake 3.1.1/3.2)

Reporting.h

@@ -30,6 +30,7 @@ namespace wdt {
 const double kMbToB = 1024 * 1024;
 const double kMicroToMilli = 1000;
 const double kMicroToSec = 1000 * 1000;
+const double kMilliToSec = 1000;
 
 typedef std::chrono::high_resolution_clock Clock;
 

SenderThread.h

@@ -7,6 +7,7 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 #pragma once
+#include <folly/Conv.h>
 #include <wdt/Sender.h>
 #include <wdt/WdtThread.h>
 #include <wdt/util/ClientSocket.h>

TARGETS

@@ -164,7 +164,7 @@ cpp_binary(
   deps = [
       ":wdtlib",
   ],
-  external_deps = [ ("gtest", None) ],
+  external_deps = [ ("googletest", None, "gtest") ],
   compiler_flags = wdt_compiler_flags,
 )
 
@@ -177,7 +177,7 @@ cpp_binary(
   deps = [
       ":wdtlib_short_flags",
   ],
-  external_deps = [ ("gtest", None) ],
+  external_deps = [ ("googletest", None, "gtest") ],
   output_subdir = 'short_flags',
   preprocessor_flags = [
     "-DSTANDALONE_APP"
@@ -256,6 +256,7 @@ cpp_unittest(
   deps = [
       ":wdtlib",
       ":wdtlib4tests",
+      "@/folly:conv",
   ],
   compiler_flags = wdt_compiler_flags,
 )
@@ -356,7 +357,7 @@ custom_unittest(
       ':wdt',
   ],
   env = wdt_env,
-  tags = ['disabled'],
+  tags = ['disabled', 'extended'],
 )
 
 custom_unittest(
@@ -370,7 +371,7 @@ custom_unittest(
       ':wdt',
   ],
   env = wdt_env,
-  tags = ['disabled'],
+  tags = ['disabled', 'extended'],
 )
 
 custom_unittest(
@@ -413,7 +414,7 @@ custom_unittest(
       ':wdt',
   ],
   env = wdt_env,
-  tags = ['disabled'],
+  tags = ['disabled', 'extended'],
 )
 
 custom_unittest(
@@ -427,7 +428,7 @@ custom_unittest(
       ':wdt',
   ],
   env = wdt_env,
-  tags = ['disabled'],
+  tags = ['disabled', 'extended'],
 )
 
 custom_unittest(
@@ -483,7 +484,7 @@ custom_unittest(
       ':wdt',
   ],
   env = wdt_env,
-  tags = ['disabled'],
+  tags = ['disabled', 'extended'],
 )
 
 custom_unittest(
@@ -497,7 +498,7 @@ custom_unittest(
       ':wdt',
   ],
   env = wdt_env,
-  tags = ['disabled'],
+  tags = ['disabled', 'extended'],
 )
 
 custom_unittest(
@@ -623,7 +624,7 @@ cpp_library (
     srcs = [
         "test/TestCommon.cpp"
         ],
-  external_deps = [ ("gtest", None) ],
+  external_deps = [ ("googletest", None, "gtest") ],
   compiler_flags = wdt_compiler_flags,
 )
 
@@ -662,7 +663,7 @@ cpp_binary(
     deps = [
       ":wdtlib4tests",
     ],
-    external_deps = [ ("gtest", None) ],
+    external_deps = [ ("googletest", None, "gtest") ],
     output_subdir = 'test',
     compiler_flags = wdt_compiler_flags,
 )

WdtOptions.cpp

@@ -42,7 +42,6 @@ void WdtOptions::modifyOptions(
                             msg)
     CHANGE_IF_NOT_SPECIFIED(resume_using_dir_tree, userSpecifiedOptions, true,
                             msg)
-    CHANGE_IF_NOT_SPECIFIED(skip_fadvise, userSpecifiedOptions, true, msg)
     return;
   }
   if (optionType != FLASH_OPTION_TYPE) {

build/folly-config.h.in

@@ -18,3 +18,5 @@
 #cmakedefine FOLLY_HAVE_WEAK_SYMBOLS 1
 #cmakedefine FOLLY_HAVE_BITS_FUNCTEXCEPT_H 1
 #cmakedefine FOLLY_HAVE_MEMRCHR 1
+
+#define FOLLY_HAVE_PTHREAD 1

build/wdt_cont_build_run.tcl

@@ -113,7 +113,7 @@ if {$os == "Darwin"} {
      time wdt/test/wdt_max_send_test.sh ./buck-out/gen/wdt/wdt |& tail -50 &&\
      EXTRA_WDT_OPTIONS=\"-scuba_dataset=wdt_transfer_report_test\" time wdt/test/wdt_max_send_test.sh ./buck-out/gen/wdt/fbonly/wdt_fb |& tail -50 &&\
      time buck build @mode/dbg-asan wdt/... &&\
-     time $timeoutCmd $maxTestDuration buck test @mode/dbg-asan wdt/... -- --extended-tests --run-disabled --record-results --return-nonzero-on-timeouts --print-long-results &&\
+     time $timeoutCmd $maxTestDuration buck test @mode/dbg-asan wdt/... -- --run-disabled --record-results --return-nonzero-on-timeouts --print-long-results &&\
      sudo tc qdisc add dev lo root netem delay 20ms 10ms \
      duplicate 1% corrupt 0.1% &&\
      echo rerunning tests with tc delays &&\

build/wdt_cont_build_setup.sh

@@ -6,7 +6,7 @@ echo "if sending email later fails, you might need:"
 echo "sudo sed -ie 's#^\(mailhub\).*$#\1=localhost#' /etc/ssmtp/ssmtp.conf"
 echo "but that shouldn't be necessary anymore"
 set -x
-sudo yum install iproute-tc devtoolset-4-gcc-c++ automake autoconf boost-devel
+sudo yum install iproute-tc devtoolset-4-gcc-c++ automake autoconf boost-devel libtool
 export PATH=/opt/rh/devtoolset-4/root/bin:$PATH
 while sh -c "g++ --version | fgrep 4.8." ; do
   smcc add-services -e hhvm.oss $HOSTNAME:22
@@ -50,7 +50,7 @@ git clone https://github.com/schuhschuh/gflags.git
 (mkdir gflags/build; cd gflags/build; cmake -DCMAKE_INSTALL_PREFIX=$CDIR -DGFLAGS_NAMESPACE=google -DBUILD_SHARED_LIBS=on .. && make -j 16 && make install)
 git clone https://github.com/google/glog.git
 echo "if this fails with aclocal error; run autoreconf"
-( cd glog && ./configure --with-gflags=$CDIR --prefix=$CDIR && make -j 16 && make install )
+( cd glog && ./configure --with-gflags=$CDIR --prefix=$CDIR && autoreconf -vfi && make -j 16 && make install )
 OPENSSL_VERSION=openssl-1.0.1u
 wget https://www.openssl.org/source/$OPENSSL_VERSION.tar.gz
 tar xfz $OPENSSL_VERSION.tar.gz

test/FdTest.cpp

@@ -10,6 +10,7 @@
 
 #include <wdt/Wdt.h>
 
+#include <folly/Conv.h>
 #include <folly/Range.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>

test/wdt_download_resumption_test.sh

@@ -125,7 +125,7 @@ SENDER_ID="123456"
 RECEIVER_ID="123456"
 
 WDTBIN_OPTS="-ipv6 -num_ports=$threads -full_reporting \
--max_mbytes_per_sec=90 -run_as_daemon=false \
+-max_mbytes_per_sec=25 -run_as_daemon=false \
 -full_reporting -read_timeout_millis=500 -write_timeout_millis=500 \
 -enable_download_resumption -treat_fewer_port_as_error \
 -resume_using_dir_tree=$RESUME_USING_DIR_TREE -enable_perf_stat_collection \
@@ -145,7 +145,7 @@ echo "Testing in $DIR"
 
 SRC_DIR=$DIR/src
 
-generateRandomFiles "$SRC_DIR" 16 # 16 Mb for smallest file
+generateRandomFiles "$SRC_DIR" 4 # 4 Mb for smallest file
 
 TEST_COUNT=0
 

test/wdt_network_test.sh

@@ -78,7 +78,7 @@ DIR=`mktemp -d --tmpdir=$BASEDIR`
 echo "Testing in $DIR"
 
 WDTBIN_OPTS="-enable_perf_stat_collection -ipv6 -start_port=$STARTING_PORT \
--avg_mbytes_per_sec=60 -max_mbytes_per_sec=65 -run_as_daemon=false \
+-avg_mbytes_per_sec=15 -max_mbytes_per_sec=20 -run_as_daemon=false \
 -full_reporting -read_timeout_millis=495 -write_timeout_millis=495 \
 -progress_report_interval_millis=-1 -abort_check_interval_millis=100 \
 -treat_fewer_port_as_error -exit_on_bad_flags=false -skip_fadvise \
@@ -92,7 +92,7 @@ WDTBIN_CLIENT="$WDT_SENDER $WDTBIN_OPTS -destination $HOSTNAME \
 
 mkdir -p $DIR/src/dir1
 cp -R folly $DIR/src/dir1
-for ((i = 2; i <= 200; i++))
+for ((i = 2; i <= 50; i++))
 do
   mkdir $DIR/src/dir${i}
   cp -R $DIR/src/dir1 $DIR/src/dir${i}

util/SerializationUtil.cpp

@@ -117,6 +117,11 @@ bool decodeIntFixedLength(folly::ByteRange &br, T &res) {
   return true;
 }
 
+bool decodeInt16FixedLength(folly::ByteRange &br, int16_t &res) {
+  bool success = decodeIntFixedLength<int16_t>(br, res);
+  return success;
+}
+
 bool decodeInt32FixedLength(folly::ByteRange &br, int32_t &res) {
   return decodeIntFixedLength<int32_t>(br, res);
 }
@@ -138,6 +143,10 @@ bool encodeIntFixedLength(char *dest, int64_t sz, int64_t &off, const T val) {
   return true;
 }
 
+bool encodeInt16FixedLength(char *dest, int64_t sz, int64_t &off, int16_t val) {
+  return encodeIntFixedLength<int16_t>(dest, sz, off, val);
+}
+
 bool encodeInt32FixedLength(char *dest, int64_t sz, int64_t &off, int32_t val) {
   return encodeIntFixedLength<int32_t>(dest, sz, off, val);
 }

util/SerializationUtil.h

@@ -36,10 +36,14 @@ bool decodeUInt64(folly::ByteRange &br, uint64_t &res);
 bool decodeInt64C(folly::ByteRange &br, int64_t &res);
 /// Encoded with encodeVarI64C (aka as unsigned) and fits in 32 bits
 bool decodeInt32C(folly::ByteRange &br, int32_t &result);
+/// Decodes fixed length int16
+bool decodeInt16FixedLength(folly::ByteRange &br, int16_t &res);
 /// Decodes fixed length int32
 bool decodeInt32FixedLength(folly::ByteRange &br, int32_t &res);
 /// Decodes fixed length int64
 bool decodeInt64FixedLength(folly::ByteRange &br, int64_t &res);
+/// Encodes fixed length int16
+bool encodeInt16FixedLength(char *dest, int64_t sz, int64_t &off, int16_t val);
 /// Encodes fixed length int32
 bool encodeInt32FixedLength(char *dest, int64_t sz, int64_t &off, int32_t val);
 /// Encodes fixed length int64

util/WdtSocket.h

@@ -86,6 +86,14 @@ class WdtSocket {
   ///           fails to get unacked bytes for this socket
   int getUnackedBytes() const;
 
+  int64_t getNumRead() const {
+    return totalRead_;
+  }
+
+  int64_t getNumWritten() const {
+    return totalWritten_;
+  }
+
   void disableIvChange() {
     WLOG(INFO) << "Disabling periodic encryption iv change";
     ivChangeInterval_ = 0;