Merge pull request #10 from kobanium/develop

Develop

Makefile

@@ -2,20 +2,21 @@ CXXFLAGS += -std=c++11 -Wextra -O2 -march=native -mtune=native
 CPPFLAGS += -MMD -MP -Isrc/common -DNDEBUG -DUSE_SSE4
 LDFLAGS += -llzma -lpthread -lOpenCL
 
-TARGETS := bin/aobaz bin/autousi bin/server bin/gencode bin/playshogi bin/crc64 bin/extract bin/ocldevs
+TARGETS := bin/aobaz bin/autousi bin/server bin/gencode bin/playshogi bin/crc64 bin/extract bin/ocldevs bin/net-test
 AUTOUSI_OBJS := src/autousi/autousi.o src/autousi/client.o src/autousi/pipe.o src/common/iobase.o src/common/option.o src/common/jqueue.o src/common/xzi.o src/common/err.o src/common/shogibase.o src/common/osi.o
 SERVER_OBJS := src/server/server.o src/server/listen.o src/server/datakeep.o src/common/iobase.o src/common/xzi.o src/common/jqueue.o src/common/err.o src/common/option.o src/server/logging.o src/common/shogibase.o src/common/osi.o
 GENCODE_OBJS := src/gencode/gencode.o
 PLAYSHOGI_OBJS := src/playshogi/playshogi.o src/common/option.o src/common/err.o src/common/iobase.o src/common/xzi.o src/common/shogibase.o src/common/osi.o
 CRC64_OBJS := src/crc64/crc64.o src/common/xzi.o src/common/err.o src/common/iobase.o src/common/osi.o
 EXTRACT_OBJS := src/extract/extract.o src/common/xzi.o src/common/err.o src/common/iobase.o src/common/osi.o
 OCLDEVS_OBJS := src/ocldevs/ocldevs.o src/common/err.o
-OBJS := $(AUTOUSI_OBJS) $(SERVER_OBJS) $(GENCODE_OBJS) $(PLAYSHOGI_OBJS) $(CRC64_OBJS) $(EXTRACT_OBJS) $(OCLDEVS_OBJS)
+NET_TEST_OBJS := src/net-test/net-test.o src/common/err.o src/common/shogibase.o
+OBJS := $(AUTOUSI_OBJS) $(SERVER_OBJS) $(GENCODE_OBJS) $(PLAYSHOGI_OBJS) $(CRC64_OBJS) $(EXTRACT_OBJS) $(OCLDEVS_OBJS) $(NET_TEST_OBJS)
 INC_OUT := src/common/tbl_zkey.inc src/common/tbl_board.inc src/common/tbl_sq.inc src/common/tbl_bmap.inc
 
 all: $(TARGETS)
 
-bin/aobaz: src/usi_engine/aobaz
+bin/aobaz: src/usi-engine/aobaz
  cp $^ $@
 
 bin/autousi: $(AUTOUSI_OBJS)
@@ -40,19 +41,23 @@ bin/extract: $(EXTRACT_OBJS)
 bin/ocldevs: $(OCLDEVS_OBJS)
  $(CXX) -o $@ $^ $(LDFLAGS)
 
+bin/net-test: $(NET_TEST_OBJS)
+ $(CXX) -o $@ $^ $(LDFLAGS)
+
 .cpp.o:
  $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $<
 
 clean:
  -$(RM) $(TARGETS) $(OBJS) $(OBJS:.o=.d) $(INC_OUT) Makefile~ build_vs.bat~
- cd src/usi_engine; $(MAKE) clean
+ cd src/usi-engine; $(MAKE) clean
 
-src/usi_engine/aobaz: FORCE
- cd src/usi_engine; $(MAKE)
+src/usi-engine/aobaz: FORCE
+ cd src/usi-engine; $(MAKE)
 
 src/autousi/pipe.cpp: bin/gencode
 src/server/datakeep.cpp: bin/gencode
 src/common/shogibase.cpp: bin/gencode
+src/net-test/net-test.cpp : bin/gencode
 src/playshogi/playshogi.cpp: bin/gencode
 
 -include $(OBJS:.o=.d)

README.md

@@ -8,11 +8,6 @@ The text written in English is [here](README_en.md).
 GPUがあれば、より高速に棋譜を生成できます。
 CPUだと10倍から100倍遅くなりますが、将棋をプレイして楽しむことは可能です。
 
-# 重要なニュース
-
-release 1.2 では非常に重要な不具合修正がbin/aobazになされました。アップデートをお願い致します。
-
-
 # AobaZero
 
 AobaZeroは、AlphaZeroの将棋の実験の追試を行うことを最終目的とした将棋AIプロジェクトです。
@@ -32,17 +27,17 @@ Silverら（2017a, 2018）は囲碁やチェスでの実験結果も報告して
 
 CPUだけのマシンは
 ```
-aobazero-1.0-w64-cpu-only.zip
+aobazero-1.3-w64-cpu-only.zip
 ```
 GPUがついたマシンは
 ```
-aobazero-1.0-w64-opencl.zip
+aobazero-1.3-w64-opencl.zip
 ```
 をダウンロード、展開して、中のclick_me.batを実行してください。
 
 Linuxの方は
 ```
-aobazero-1.0.tar.gz
+aobazero-1.3.tar.gz
 ```
 を展開してmakeしてから
 ```
@@ -51,13 +46,13 @@ aobazero-1.0.tar.gz
 を実行してください。詳しくは同梱のreadme.txtをご覧ください。
 
 # 将棋所で遊んでみたい
-CPU版をダウンロードして、click_me.batを実行します。しばらくすると最新のネットワークの重みファイルをダウンロードして「PI」が表示されて棋譜の生成を開始します。すかさずCtrl + Cで停止させます。(signal 1 caught)が表示されて、しばらく待つと止まります。 
+CPU版をダウンロードして、click_me.batを実行します。しばらくすると最新のネットワークの重みファイルをダウンロードして「self-play start」が表示されて棋譜の生成を開始します。すかさずCtrl + Cで停止させます。(signal 1 caught)が表示されて、しばらく待つと止まります。 
 weight_save/の下にw000000000468.txt という230MBほどのファイルが作られます。
 (468、の数値は異なります)
 
-aobazero-1.0-w64-cpu-only.zipに同梱されているaobaz.batを編集します。最後の1行が以下のようになっています。
+aobazero-1.3-w64-cpu-only.zipに同梱されているaobaz.batを編集します。最後の1行が以下のようになっています。
 ```
-bin/aobaz -q -p 30 -w weight_save\w000000000467.txt
+bin/aobaz -q -i -p 30 -w weight_save\w000000000467.txt
 ```
 この467の部分を実際にダウンロードしてきたファイル名に合わせて書き直し、保存します。 
 将棋所にaobaz.batをエンジンとして登録します。 

README_en.md

@@ -26,17 +26,17 @@ and Chess [LCZero](https://lczero.org). Both projects have achieved super human
 
 For machine without GPU
 ```
-aobazero-1.0-w64-cpu-only.zip
+aobazero-1.3-w64-cpu-only.zip
 ```
 For machine with GPU
 ```
-aobazero-1.0-w64-opencl.zip
+aobazero-1.3-w64-opencl.zip
 ```
 Download it, unzip, and run click_me.bat.
 
 For Linux,
 ```
-aobazero-1.0.tar.gz
+aobazero-1.3.tar.gz
 ```
 Unzip it, make, then run
 ```
@@ -46,15 +46,15 @@ Please see readme.txt for details.
 
 # I'd like to play with ShogiDokoro.
 Download CPU version and run click_me.bat.
-After a while, it downloads the latest network weight file, and "PI" is displayed, and self-play starts. Input "Ctrl + C" immediately. (signal 1 caught) is displayed and it will stop after a while.
+After a while, it downloads the latest network weight file, and "self-play start" is displayed, and self-play starts. Input "Ctrl + C" immediately. (signal 1 caught) is displayed and it will stop after a while.
 
 weight_save/w0000000000468.txt will be created. Its size is about 230MB.
 (the numbers "468" will be different.)
 
-Edit aobaz.bat that is in aobazero-1.0-w64-cpu-only.zip.
+Edit aobaz.bat that is in aobazero-1.3-w64-cpu-only.zip.
 The last line is like this,
 ```
-bin/aobaz -q -p 30 -w weight_save\w000000000467.txt
+bin/aobaz -q -i -p 30 -w weight_save\w000000000467.txt
 ```
 Rewrite this "467" according to the file name actually downloaded, and save.
 Register aobaz.bat as a engine in ShogiDokoro.

autousi.cfg

@@ -1,11 +1,11 @@
 # USI engine
 DirLog ./log
 CmdPath bin/aobaz
-# Device 0 1 2  # use three GPUs of device no. 0, 1, and 2
-Device -1 # use a default devide
+# Device 0 1 2 # use three OpenCL devices (ID 0, 1, and 2)
+Device -1  # use a default device
 
 # socket communication
-WeightSave ./weight_save
+WeightSave ./weight-save
 SizeSendQueue 64
 RecvTO 30 # in second
 SendTO 30 # in second
@@ -18,6 +18,7 @@ Port 20000
 # output
 DirCSA ./csa
 MaxCSA 10000
-PrintSpeed 1 # 0:off 1:on
-PrintCSA 4 # 0:off 1-:specify the number of moves per a line
-KeepWeight 1 # 0:off 1:on
+PrintStatus 5 # 0:off 1-:specify status-output time interval in sec
+PrintCSA 4 # 0:off 1-:specify the number of moves per a line
+VerboseEngine 1 # 0:off 1:on
+KeepWeight 1 # 0:off 1:on

readme.txt

@@ -29,13 +29,14 @@ Ubuntu での手順を記します。
 
 CentOS での手順を記します。
 
-- liblzma をインストール
+- liblzma などをインストール
 
-> sudo yum install xz-devel
+> sudo yum install xz-devel zlib-devel
 
 - GCC 5.3.1 をインストール
 
-> sudo yum -y install centos-release-scl devtoolset-4
+> sudo yum -y install centos-release-scl
+> sudo yum -y install devtoolset-4
 > scl enable devtoolset-4 bash
 
 GCC 5.3.1 が利用可能な環境で bash が起動します。
@@ -46,7 +47,7 @@ GCC 5.3.1 が利用可能な環境で bash が起動します。
 > gzip -dc boost_1_58_0.tar.gz | tar xvf -
 > cd boost_1_58_0
 > ./bootstrap.sh
-> ./b2 install -j8 --prefix=/[a path with write permission]/inst-dts4
+> ./b2 install -j[number of threads] --prefix=/[a path with write permission]/inst-dts4
 
 - 環境変数の設定
 

src/autousi/autousi.cpp

@@ -5,37 +5,50 @@
 #include "client.hpp"
 #include "pipe.hpp"
 #include "option.hpp"
+#include <algorithm>
 #include <atomic>
+#include <chrono>
 #include <exception>
 #include <iostream>
+#include <iomanip>
 #include <map>
 #include <random>
 #include <string>
 #include <fstream>
 #include <vector>
+#include <cinttypes>
+#include <cstdio>
 using std::atomic;
 using std::cerr;
 using std::cout;
 using std::current_exception;
 using std::endl;
 using std::exception;
 using std::exception_ptr;
+using std::fill_n;
 using std::ifstream;
 using std::map;
+using std::min;
 using std::random_device;
 using std::rethrow_exception;
 using std::string;
 using std::set_terminate;
 using std::vector;
-using std::chrono::milliseconds;
+using std::chrono::duration_cast;
+using std::chrono::time_point;
+using std::chrono::system_clock;
+using std::chrono::seconds;
 using std::this_thread::sleep_for;
 using namespace IOAux;
 using ErrAux::die;
 using uint = unsigned int;
 using ushort = unsigned short int;
 
-constexpr uint max_sleep = 3000U; // in msec
-volatile atomic<int> flag_signal(0);
+constexpr uint max_sleep = 3U; // in sec
+atomic<int> flag_signal(0);
+static uint print_status, print_csa;
+static vector<int> devices;
+static time_point<system_clock> time_start;
 
 static bool is_posi(uint u) { return 0 < u; }
 
@@ -66,8 +79,9 @@ static void init() noexcept {
  {"SendBufSiz", "8192"},
  {"MaxRetry", "7"},
  {"MaxCSA", "0"},
- {"PrintSpeed",  "0"},
+ {"PrintStatus", "0"},
  {"PrintCSA", "0"},
+ {"VerboseEngine", "0"},
  {"KeepWeight", "0"},
  {"Addr", "127.0.0.1"},
  {"Port", "20000"}};
@@ -77,34 +91,116 @@ static void init() noexcept {
  const char *cstr_dlog = Config::get_cstr(m, "DirLog", maxlen_path);
  const char *cstr_csa = Config::get_cstr(m, "DirCSA", maxlen_path);
  const char *cstr_addr = Config::get_cstr(m, "Addr", 64);
- uint size_queue  = Config::get<uint> (m, "SizeSendQueue", is_posi);
- uint recvTO  = Config::get<uint> (m, "RecvTO", is_posi);
- uint sendTO  = Config::get<uint> (m, "SendTO", is_posi);
- uint recv_bufsiz  = Config::get<uint> (m, "RecvBufSiz", is_posi);
- uint send_bufsiz  = Config::get<uint> (m, "SendBufSiz", is_posi);
- uint max_retry  = Config::get<uint> (m, "MaxRetry", is_posi);
- uint max_csa  = Config::get<uint> (m, "MaxCSA");
- uint print_speed = Config::get<uint> (m, "PrintSpeed");
- uint print_csa = Config::get<uint> (m, "PrintCSA");
- uint keep_wght = Config::get<uint> (m, "KeepWeight");
- uint port  = Config::get<ushort>(m, "Port");
- vector<int> devices = Config::getv<int> (m, "Device");
-
+ uint size_queue = Config::get<uint> (m, "SizeSendQueue", is_posi);
+ uint recvTO = Config::get<uint> (m, "RecvTO", is_posi);
+ uint sendTO = Config::get<uint> (m, "SendTO", is_posi);
+ uint recv_bufsiz = Config::get<uint> (m, "RecvBufSiz", is_posi);
+ uint send_bufsiz = Config::get<uint> (m, "SendBufSiz", is_posi);
+ uint max_retry = Config::get<uint> (m, "MaxRetry", is_posi);
+ uint max_csa = Config::get<uint> (m, "MaxCSA");
+ uint keep_wght = Config::get<uint> (m, "KeepWeight");
+ uint verbose_eng = Config::get<uint> (m, "VerboseEngine");
+ uint port  = Config::get<ushort>(m, "Port");
+ devices  = Config::getv<int> (m, "Device");
+ print_status = Config::get<uint> (m, "PrintStatus");
+ print_csa = Config::get<uint> (m, "PrintCSA");
  Client::get().start(cstr_dwght, cstr_addr, port, recvTO, recv_bufsiz, sendTO,
  send_bufsiz, max_retry, size_queue, keep_wght);
  OSI::handle_signal(on_signal);
  Pipe::get().start(cstr_cname, cstr_dlog, devices, cstr_csa, max_csa,
- print_csa, print_speed);
+ verbose_eng);
+ time_start = system_clock::now();
  cout << "self-play start" << endl; }
 
+static void output() noexcept {
+ static bool first = true;
+ static bool print_csa_do_nl = false;
+ static uint print_csa_num = 0;
+ static time_point<system_clock> time_last = system_clock::now();
+
+ // print moves of child id #0
+ string s;
+ while (Pipe::get().get_moves_id0(s)) {
+ if (print_csa == 0) continue;
+ if (print_csa_do_nl) { print_csa_do_nl = false; puts(""); }
+ else if (!first) fputs(", ", stdout);
+ first = false;
+ fputs(s.c_str(), stdout);
+ if (s.at(0) == '%' || (++print_csa_num % print_csa) == 0) {
+ if (s.at(0) == '%') puts("");
+ print_csa_num = 0;
+ print_csa_do_nl = true; }
+ fflush(stdout); }
+
+ // print status
+ if (print_status == 0) return;
+
+ time_point<system_clock> time_now = system_clock::now();
+ if (time_now < time_last + seconds(print_status)) return;
+
+ static uint prev_ntot = 0;
+ static uint prev_nsend = 0;
+ static uint prev_ndiscard = 0;
+ uint ntot = Pipe::get().get_ngen_records();
+ uint nsend = Client::get().get_nsend();
+ uint ndiscard = Client::get().get_ndiscard();
+ if ( prev_ntot == ntot && prev_nsend == nsend && prev_ndiscard == ndiscard ) return;
+ prev_ntot = ntot;
+ prev_nsend = nsend;
+ prev_ndiscard = ndiscard;
+
+ first = true;
+ print_csa_do_nl = false;
+ print_csa_num = 0;
+ time_last = time_now;
+ puts("");
+ puts("+------+-----+--------+---< Aobaz Status >------------------------+");
+ puts("| PID | Dev | Average| Moves |");
+ puts("+------+-----+--------+-------------------------------------------+");
+ for (uint u = 0; u < devices.size(); ++u) {
+ const int BUF_SIZE = 64;
+ char spid[BUF_SIZE] = " N/A ";
+ if ( ! Pipe::get().is_closed(u) ) snprintf(spid,BUF_SIZE,"%6d",Pipe::get().get_pid(u));
+ char buf[BUF_SIZE];
+ fill_n(buf, sizeof(buf), '#');
+ uint len = std::min(Pipe::get().get_nmove(u) / 5,
+ static_cast<uint>(sizeof(buf)) - 1U);
+ buf[len] = '\0';
+ printf("|%s|%4d |%6.0fms|%3d:%-39s|\n",
+ spid, devices[u],
+ Pipe::get().get_speed_average(u),
+ Pipe::get().get_nmove(u), buf); }
+ puts("+------+-----+--------+-------------------------------------------+");
+
+ printf("- Send Status: Sent %d, Lost %d, Waiting %d\n",
+ nsend, ndiscard, ntot - nsend - ndiscard);
+
+ int64_t wght_id = Client::get().get_wght_id();
+ bool is_downloading = Client::get().is_downloading();
+ const char *buf_time = Client::get().get_buf_wght_time();
+ printf("- Recv Status: Weights' ID %" PRIi64 ", ", wght_id);
+
+ if (is_downloading) puts("NOW DOWNLOADING NEW WEIGHTS\n");
+ else printf("Last Check %s\n", buf_time);
+ auto t = system_clock::to_time_t(time_now) - system_clock::to_time_t(time_start);
+ if ( t==0 ) t = 1;
+ double hour = (double)t/3600.0;
+ double day = (double)t/(3600.0*24);
+ printf("- %.1f sent/hour, %.1f sent/day, Running for %.1f hours(%.1f days).\n\n",
+ nsend / hour, nsend / day, hour, day);
+}
+
 int main() {
  OSI::prevent_multirun(FName("/tmp/autousi.jBQoNA7kEd.lock"));
- sleep_for(milliseconds(random_device()() % max_sleep));
+ sleep_for(seconds(random_device()() % max_sleep));
  set_terminate(on_terminate);
+
  init();
- while (!flag_signal) Pipe::get().wait();
+ while (!flag_signal) {
+ output();
+ Pipe::get().wait(); }
 
- cout << "signal " << flag_signal << " caught" << endl;
+ cout << "\nsignal " << flag_signal << " caught" << endl;
  Client::get().end();
  Pipe::get().end();
  return 0;