Merge pull request bitcoin#494 from kyuupichan/univalue

[port] bring src/univalue up to core/master

doc/release-notes.md

@@ -42,6 +42,11 @@ Example item
 Example text.
 
 
+- Full UTF-8 support in the RPC API. Non-ASCII characters in, for example,
+  wallet labels have always been malformed because they weren't taken into account
+  properly in JSON RPC processing. This is no longer the case. This also affects
+  the GUI debug console.
+
 C++11
 -----
 
@@ -65,10 +70,10 @@ git merge commit are mentioned.
 Asm script outputs replacements for OP_NOP2 and OP_NOP3
 -------------------------------------------------------
 
-OP_NOP2 has been renamed to OP_CHECKLOCKTIMEVERIFY by [BIP 
+OP_NOP2 has been renamed to OP_CHECKLOCKTIMEVERIFY by [BIP
 65](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki)
 
-OP_NOP3 has been renamed to OP_CHECKSEQUENCEVERIFY by [BIP 
+OP_NOP3 has been renamed to OP_CHECKSEQUENCEVERIFY by [BIP
 112](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki)
 
 The following outputs are affected by this change:

qa/rpc-tests/test_framework/authproxy.py

@@ -69,10 +69,12 @@ def EncodeDecimal(o):
 class AuthServiceProxy(object):
     __id_count = 0
 
-    def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None):
+    # ensure_ascii: escape unicode as \uXXXX, passed to json.dumps
+    def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True):
         self.__timeout = timeout
         self.__service_url = service_url
         self._service_name = service_name
+        self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests
         self.__url = urlparse.urlparse(service_url)
         if self.__url.port is None:
             port = 80
@@ -153,12 +155,12 @@ def __call__(self, *args):
         AuthServiceProxy.__id_count += 1
 
         log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self._service_name,
-                                 json.dumps(args, default=EncodeDecimal)))
+                                 json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
         postdata = json.dumps({'version': '1.1',
                                'method': self._service_name,
                                'params': args,
-                               'id': AuthServiceProxy.__id_count}, default=EncodeDecimal)
-        response = self._request('POST', self.__url.path, postdata)
+                               'id': AuthServiceProxy.__id_count}, default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
+        response = self._request('POST', self.__url.path, postdata.encode('utf-8'))
         if response['error'] is not None:
             raise JSONRPCException(response['error'])
         elif 'result' not in response:
@@ -168,9 +170,9 @@ def __call__(self, *args):
             return response['result']
 
     def _batch(self, rpc_call_list):
-        postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal)
+        postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
         log.debug("--> "+postdata)
-        return self._request('POST', self.__url.path, postdata)
+        return self._request('POST', self.__url.path, postdata.encode('utf-8'))
 
     def _get_response(self):
         http_response = self.__conn.getresponse()
@@ -186,7 +188,7 @@ def _get_response(self):
         responsedata = http_response.read().decode('utf8')
         response = json.loads(responsedata, parse_float=decimal.Decimal)
         if "error" in response and response["error"] is None:
-            log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal)))
+            log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
         else:
             log.debug("<-- "+responsedata)
         return response

qa/rpc-tests/wallet.py

@@ -318,6 +318,20 @@ def run_test (self):
         balance_nodes = [self.nodes[i].getbalance() for i in range(3)]
         block_count = self.nodes[0].getblockcount()
 
+        # Check modes:
+        #   - True: unicode escaped as \u....
+        #   - False: unicode directly as UTF-8
+        for mode in [True, False]:
+            self.nodes[0].ensure_ascii = mode
+            # unicode check: Basic Multilingual Plane, Supplementary Plane respectively
+            for s in [u'рыба', u'𝅘𝅥𝅯']:
+                addr = self.nodes[0].getaccountaddress(s)
+                label = self.nodes[0].getaccount(addr)
+                assert_equal(label, s)
+                assert(s in self.nodes[0].listaccounts().keys())
+        self.nodes[0].ensure_ascii = True # restore to default
+
+        # maintenance tests
         maintenance = [
             '-rescan',
             '-reindex',

src/univalue/.travis.yml

@@ -1,4 +1,3 @@
-
 language: cpp
 
 compiler:
@@ -26,6 +25,7 @@ addons:
       - pkg-config
 
 before_script:
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew uninstall libtool; brew install libtool; fi
   - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi
   - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
 

src/univalue/Makefile.am

@@ -3,7 +3,7 @@ ACLOCAL_AMFLAGS = -I build-aux/m4
 .INTERMEDIATE: $(GENBIN)
 
 include_HEADERS = include/univalue.h
-noinst_HEADERS = lib/univalue_escapes.h
+noinst_HEADERS = lib/univalue_escapes.h lib/univalue_utffilter.h
 
 lib_LTLIBRARIES = libunivalue.la
 
@@ -73,6 +73,10 @@ TEST_FILES = \
 	$(TEST_DATA_DIR)/fail35.json \
 	$(TEST_DATA_DIR)/fail36.json \
 	$(TEST_DATA_DIR)/fail37.json \
+	$(TEST_DATA_DIR)/fail38.json \
+	$(TEST_DATA_DIR)/fail39.json \
+	$(TEST_DATA_DIR)/fail40.json \
+	$(TEST_DATA_DIR)/fail41.json \
 	$(TEST_DATA_DIR)/fail3.json \
 	$(TEST_DATA_DIR)/fail4.json \
 	$(TEST_DATA_DIR)/fail5.json \
@@ -83,6 +87,7 @@ TEST_FILES = \
 	$(TEST_DATA_DIR)/pass1.json \
 	$(TEST_DATA_DIR)/pass2.json \
 	$(TEST_DATA_DIR)/pass3.json \
-	$(TEST_DATA_DIR)/round1.json
+	$(TEST_DATA_DIR)/round1.json \
+	$(TEST_DATA_DIR)/round2.json
 
 EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS)

src/univalue/configure.ac

@@ -1,7 +1,7 @@
 m4_define([libunivalue_major_version], [1])
 m4_define([libunivalue_minor_version], [1])
-m4_define([libunivalue_micro_version], [1])
-m4_define([libunivalue_interface_age], [1])
+m4_define([libunivalue_micro_version], [2])
+m4_define([libunivalue_interface_age], [2])
 # If you need a modifier for the version number. 
 # Normally empty, but can be used to make "fixup" releases.
 m4_define([libunivalue_extraversion], [])
@@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter
 m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()])
 
 
-AC_INIT([univalue], [1.0.1],
+AC_INIT([univalue], [1.0.2],
         [http://github.com/jgarzik/univalue/])
 
 dnl make the compilation flags quiet unless V=1 is used

src/univalue/include/univalue.h

@@ -59,7 +59,7 @@ class UniValue {
     bool setNumStr(const std::string& val);
     bool setInt(uint64_t val);
     bool setInt(int64_t val);
-    bool setInt(int val) { return setInt((int64_t)val); }
+    bool setInt(int val_) { return setInt((int64_t)val_); }
     bool setFloat(double val);
     bool setStr(const std::string& val);
     bool setArray();
@@ -98,28 +98,28 @@ class UniValue {
     bool push_backV(const std::vector<UniValue>& vec);
 
     bool pushKV(const std::string& key, const UniValue& val);
-    bool pushKV(const std::string& key, const std::string& val) {
-        UniValue tmpVal(VSTR, val);
+    bool pushKV(const std::string& key, const std::string& val_) {
+        UniValue tmpVal(VSTR, val_);
         return pushKV(key, tmpVal);
     }
     bool pushKV(const std::string& key, const char *val_) {
-        std::string val(val_);
-        return pushKV(key, val);
+        std::string _val(val_);
+        return pushKV(key, _val);
     }
-    bool pushKV(const std::string& key, int64_t val) {
-        UniValue tmpVal(val);
+    bool pushKV(const std::string& key, int64_t val_) {
+        UniValue tmpVal(val_);
         return pushKV(key, tmpVal);
     }
-    bool pushKV(const std::string& key, uint64_t val) {
-        UniValue tmpVal(val);
+    bool pushKV(const std::string& key, uint64_t val_) {
+        UniValue tmpVal(val_);
         return pushKV(key, tmpVal);
     }
-    bool pushKV(const std::string& key, int val) {
-        UniValue tmpVal((int64_t)val);
+    bool pushKV(const std::string& key, int val_) {
+        UniValue tmpVal((int64_t)val_);
         return pushKV(key, tmpVal);
     }
-    bool pushKV(const std::string& key, double val) {
-        UniValue tmpVal(val);
+    bool pushKV(const std::string& key, double val_) {
+        UniValue tmpVal(val_);
         return pushKV(key, tmpVal);
     }
     bool pushKVs(const UniValue& obj);
@@ -145,10 +145,10 @@ class UniValue {
 public:
     // Strict type-specific getters, these throw std::runtime_error if the
     // value is of unexpected type
-    std::vector<std::string> getKeys() const;
-    std::vector<UniValue> getValues() const;
+    const std::vector<std::string>& getKeys() const;
+    const std::vector<UniValue>& getValues() const;
     bool get_bool() const;
-    std::string get_str() const;
+    const std::string& get_str() const;
     int get_int() const;
     int64_t get_int64() const;
     double get_real() const;

src/univalue/lib/univalue.cpp

@@ -119,32 +119,29 @@ bool UniValue::setNumStr(const string& val_)
     return true;
 }
 
-bool UniValue::setInt(uint64_t val)
+bool UniValue::setInt(uint64_t val_)
 {
-    string s;
     ostringstream oss;
 
-    oss << val;
+    oss << val_;
 
     return setNumStr(oss.str());
 }
 
-bool UniValue::setInt(int64_t val)
+bool UniValue::setInt(int64_t val_)
 {
-    string s;
     ostringstream oss;
 
-    oss << val;
+    oss << val_;
 
     return setNumStr(oss.str());
 }
 
-bool UniValue::setFloat(double val)
+bool UniValue::setFloat(double val_)
 {
-    string s;
     ostringstream oss;
 
-    oss << std::setprecision(16) << val;
+    oss << std::setprecision(16) << val_;
 
     bool ret = setNumStr(oss.str());
     typ = VNUM;
@@ -173,12 +170,12 @@ bool UniValue::setObject()
     return true;
 }
 
-bool UniValue::push_back(const UniValue& val)
+bool UniValue::push_back(const UniValue& val_)
 {
     if (typ != VARR)
         return false;
 
-    values.push_back(val);
+    values.push_back(val_);
     return true;
 }
 
@@ -192,13 +189,13 @@ bool UniValue::push_backV(const std::vector<UniValue>& vec)
     return true;
 }
 
-bool UniValue::pushKV(const std::string& key, const UniValue& val)
+bool UniValue::pushKV(const std::string& key, const UniValue& val_)
 {
     if (typ != VOBJ)
         return false;
 
     keys.push_back(key);
-    values.push_back(val);
+    values.push_back(val_);
     return true;
 }
 
@@ -228,7 +225,7 @@ int UniValue::findKey(const std::string& key) const
 bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t)
 {
     for (std::map<std::string,UniValue::VType>::const_iterator it = t.begin();
-         it != t.end(); it++) {
+         it != t.end(); ++it) {
         int idx = findKey(it->first);
         if (idx < 0)
             return false;
@@ -286,14 +283,14 @@ const UniValue& find_value(const UniValue& obj, const std::string& name)
     return NullUniValue;
 }
 
-std::vector<std::string> UniValue::getKeys() const
+const std::vector<std::string>& UniValue::getKeys() const
 {
     if (typ != VOBJ)
         throw std::runtime_error("JSON value is not an object as expected");
     return keys;
 }
 
-std::vector<UniValue> UniValue::getValues() const
+const std::vector<UniValue>& UniValue::getValues() const
 {
     if (typ != VOBJ && typ != VARR)
         throw std::runtime_error("JSON value is not an object or array as expected");
@@ -307,7 +304,7 @@ bool UniValue::get_bool() const
     return getBool();
 }
 
-std::string UniValue::get_str() const
+const std::string& UniValue::get_str() const
 {
     if (typ != VSTR)
         throw std::runtime_error("JSON value is not a string as expected");