Merge pull request #103 from citp/v0.5

Update BlockSci to v0.5.0

Version 0.5.0 focuses mainly on improvements and cleanups in the python interface. The largest new feature is the introduction of vectorized operations which return NumPy arrays, enabling much more rapid usage of BlockSci's python interface. You can read more details about the release in the [release notes](https://citp.github.io/BlockSci/changelog.html#version-0-5-0).

Fixes #72, fixes #76, fixes #84, and fixes #98

Author: hkalodner

.gitignore

@@ -0,0 +1,12 @@
+release
+xcode
+debug
+build
+*.DS_Store
+Notebooks/blocksci
+*.so
+*.dylib
+*.pyc
+include/blocksci/external
+python/blocksci/pybind11
+*.egg-info

.gitmodules

@@ -1,24 +1,24 @@
 [submodule "libs/pybind11"]
-	path = libs/pybind11
+	path = blockscipy/blocksci/pybind11
 	url = https://github.com/pybind/pybind11/
 [submodule "libs/bitcoin-api-cpp"]
-	path = libs/bitcoin-api-cpp
+	path = external/bitcoin-api-cpp
 	url = https://github.com/hkalodner/bitcoin-api-cpp.git
 [submodule "libs/dset"]
-	path = libs/dset
+	path = external/dset
 	url = https://github.com/wjakob/dset.git
 [submodule "libs/variant"]
-	path = libs/variant
+	path = external/variant
 	url = https://github.com/mpark/variant.git
 [submodule "libs/range-v3"]
-	path = libs/range-v3
+	path = external/range-v3
 	url = https://github.com/ericniebler/range-v3.git
-[submodule "libs/type_safe"]
-	path = libs/type_safe
-	url = https://github.com/foonathan/type_safe
 [submodule "Notebooks/blocksci/Blockchain-Known-Pools"]
-	path = Notebooks/blocksci/Blockchain-Known-Pools
+	path = blockscipy/blocksci/Blockchain-Known-Pools
 	url = https://github.com/blockchain/Blockchain-Known-Pools
 [submodule "libs/clipp"]
-	path = libs/clipp
+	path = external/clipp
 	url = https://github.com/muellan/clipp
+[submodule "libs/rocksdb"]
+	path = external/rocksdb
+	url = https://github.com/facebook/rocksdb

CHANGELOG.rst

@@ -4,6 +4,59 @@
 Release notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+Version 0.5.0
+========================
+
+Feature Enhancements
+---------------------
+
+- Expanded iterator and range functionality to return NumPy arrays.
+
+  Many methods and properties of BlockSci objects return range or iterator objects such as :class:`blocksci.TxRange`. These objects allow vectorized operations over sequences of BlockSci objects. Their API matches up with the API of their member objects, and thus :class:`blocksci.TxRange` has almost the same set of methods as :class:`blocksci.Tx`. These methods will effiently call the given method over all items in the range or iterator. Depending on the return type of the method, the result will either be another range, a NumPy array, or a python list. For further information, look for these classes in the reference.
+
+- Add custom BlockSci pickler to enable sending and recieving serialized BlockSci objects. This means that returning BlockSci objects from the multiprocessing interface now works correctly.
+
+- Enhance the change address heuristics interface
+
+  Change address heuristics are now composible in order to form new customized heuristics using the :class:`blocksci.heuristics.change.ChangeHeuristic` interface. These can be used in combination with the new clustering interface described below.
+
+- Incorporate clustering module into main BlockSci library
+
+  The formerly external clustering module is now avaiable as :mod:`blocksci.cluster`. Further, it is now possible to generate new clusterings through the python interface using the :func:`~blocksci.cluster.ClusterManager.create_clustering` method. Users can select their choice of change address heuristic in order to experiment with different clustering strategies.
+
+- Simplified build system
+
+  BlockSci's install process no longer requires the compilation of any external dependencies to compile on Ubuntu 16.04.
+
+  The BlockSci library no longer has any public dependencies so compiling against it will not require linking against anything else.
+
+  The CMake build script has now been updated to install a Config file which allows you to use `find_package(blocksci)` to import BlockSci's target's into your build script. This makes it much easier to build libraries that use BlockSci as a dependency.
+
+  The BlockSci python module has been moved into a separate module to allow for a simple SetupTools or pip based install process: `pip install -e pyblocksci`. The main BlockSci library must be installed first for this to work.
+
+- Updated mempool recorder and integrated it into BlockSci interface.
+
+  For instructions on running the mempool recorder and using the data it produces, see the setup_ section.
+
+- Improve and clean up auto generated API reference. 
+
+  All method signatures display correct types and all properties display the type of the returned value. Further, all types link to their definition in the documentation.
+
+
+Bug Fixes
+----------
+
+- Removed hardcoded paths (`Issue #72`_)
+- Fixed :func:`~blocksci.Block.miner` (`Issue #76`_)
+- Added pickling support (`Issue #84`_)
+- Added :func:`~blocksci.Blockchain.reload` (`Issue #98`_)
+
+.. _setup: https://citp.github.io/BlockSci/setup.html
+.. _Issue #72: https://github.com/citp/BlockSci/issues/72
+.. _Issue #76: https://github.com/citp/BlockSci/issues/76
+.. _Issue #84: https://github.com/citp/BlockSci/issues/84
+.. _Issue #98: https://github.com/citp/BlockSci/issues/98
+
 Version 0.4.5
 ========================
 

CMakeLists.txt

@@ -1,67 +1,15 @@
-cmake_minimum_required(VERSION 3.9)
+cmake_minimum_required(VERSION 3.5)
 project(blocksci)
 
-set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_MACOSX_RPATH 1)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic")
+set(BlockSci_VERSION 0.5.0)
 
-#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-old-style-cast -Wno-documentation-unknown-command -Wno-documentation -Wno-shadow -Wno-switch-enum -Wno-missing-prototypes -Wno-weak-vtables -Wno-exit-time-destructors -Wno-unused-macros -Wno-padded -Wno-undefined-func-template")
+include(GNUInstallDirs)
 
-#set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_GLIBCXX_DEBUG")
+add_subdirectory(external)
 
-# use, i.e. don't skip the full RPATH for the build tree
-SET(CMAKE_SKIP_BUILD_RPATH  FALSE)
-
-# when building, don't use the install RPATH already
-# (but later on when installing)
-SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) 
-
-SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-
-# add the automatically determined parts of the RPATH
-# which point to directories outside the build tree to the install RPATH
-SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-
-# the RPATH to be used when installing, but only if it's not a system directory
-LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
-IF("${isSystemDir}" STREQUAL "-1")
-   SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-ENDIF("${isSystemDir}" STREQUAL "-1")
-
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
-
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads REQUIRED)
-
-find_package( Boost 1.58 COMPONENTS system filesystem thread iostreams serialization REQUIRED )
-add_library(boost INTERFACE)
-target_link_libraries(boost INTERFACE ${Boost_LIBRARIES})
-target_include_directories(boost INTERFACE SYSTEM ${Boost_INCLUDE_DIR})
-
-find_package(OpenSSL REQUIRED)
-add_library(openssl INTERFACE)
-target_link_libraries(openssl INTERFACE ${OPENSSL_LIBRARIES})
-target_include_directories(openssl INTERFACE SYSTEM ${OPENSSL_INCLUDE_DIR})
-
-add_library(Ranges INTERFACE)
-target_include_directories(Ranges INTERFACE SYSTEM libs/range-v3/include)
-
-add_library(clipp INTERFACE)
-target_include_directories(clipp INTERFACE SYSTEM libs/clipp/include)
-
-link_directories(/usr/local/lib)
-include_directories("src")
-
-set(PYBIND11_INSTALL true)
-add_subdirectory(libs/pybind11)
-
-add_subdirectory(libs/variant)
-
-add_subdirectory(libs/type_safe)
-
-add_subdirectory(src/blocksci)
-add_subdirectory(src/parser)
-add_subdirectory(src/mempool_recorder)
-add_subdirectory(src/python-interface)
-add_subdirectory(src/example)
+add_subdirectory(src)
+add_subdirectory(tools)
+add_subdirectory(benchmark)
+add_subdirectory(example)

Notebooks/BlockSci Demo.ipynb

@@ -106,7 +106,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%time txes = chain.address_type_txes(0, len(chain), blocksci.address_type.nulldata)"
+    "%time txes = chain.blocks.txes.including_output_of_type(blocksci.address_type.nulldata).all"
    ]
   },
   {
@@ -156,8 +156,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fees = [sum(block.txes.fee) / len(block) for block in chain.range('2017')]\n",
-    "times = [block.time for block in chain.range('2017')]"
+    "# BlockSci vectorized interface\n",
+    "blocks = chain.range('2017')\n",
+    "fees = blocks.fee / blocks.tx_count\n",
+    "times = blocks.time"
    ]
   },
   {
@@ -233,10 +235,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import blocksci.cluster_python\n",
     "# cluster_data_directory should be set to the directory containing the\n",
     "# output of the `clusterer` program\n",
-    "cm = blocksci.cluster_python.ClusterManager(cluster_data_directory, chain)"
+    "cm = blocksci.cluster.ClusterManager(cluster_data_directory, chain)"
    ]
   },
   {