Skip to content

Commit 53ce267

Browse files
hkalodnerhkalodner
committed
Switch to using sized ranges to fix __getitem__ performance issue
Fixing this required forking ranges-v3 in order to fix ericniebler/range-v3#824.
1 parent 7a29cb0 commit 53ce267

13 files changed

+53
-38
lines changed

‎.gitmodules

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
url = https://github.com/mpark/variant.git
1313
[submodule "libs/range-v3"]
1414
path = external/range-v3
15-
url = https://github.com/ericniebler/range-v3.git
15+
url = https://github.com/hkalodner/range-v3.git
1616
[submodule "Notebooks/blocksci/Blockchain-Known-Pools"]
1717
path = blockscipy/blocksci/Blockchain-Known-Pools
1818
url = https://github.com/blockchain/Blockchain-Known-Pools

‎blockscipy/src/blocksci_range.hpp

+4-4
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@
1818
template <typename T>
1919
struct RangeClasses {
2020
pybind11::class_<ranges::any_view<T>> iterator;
21-
pybind11::class_<ranges::any_view<T, ranges::category::random_access>> range;
21+
pybind11::class_<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>> range;
2222
pybind11::class_<ranges::any_view<ranges::optional<T>>> optionalIterator;
23-
pybind11::class_<ranges::any_view<ranges::optional<T>, ranges::category::random_access>> optionalRange;
23+
pybind11::class_<ranges::any_view<ranges::optional<T>, ranges::category::random_access | ranges::category::sized>> optionalRange;
2424

2525
RangeClasses(pybind11::module &m) :
2626
iterator(m, strdup(PythonTypeName<ranges::any_view<T>>::name().c_str())),
27-
range(m, strdup(PythonTypeName<ranges::any_view<T, ranges::category::random_access>>::name().c_str())),
27+
range(m, strdup(PythonTypeName<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>>::name().c_str())),
2828
optionalIterator(m, strdup(PythonTypeName<ranges::any_view<ranges::optional<T>>>::name().c_str())),
29-
optionalRange(m, strdup(PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access>>::name().c_str())) {}
29+
optionalRange(m, strdup(PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access | ranges::category::sized>>::name().c_str())) {}
3030
};
3131

3232
#endif /* blocksci_blocksci_range_h */

‎blockscipy/src/chain/block/block_py.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ struct AddBlockMethods {
2828
using namespace blocksci;
2929
namespace py = pybind11;
3030

31-
func(property_tag, "txes", [](const Block &block) -> ranges::any_view<Transaction, ranges::category::random_access> {
31+
func(property_tag, "txes", [](const Block &block) -> ranges::any_view<Transaction, ranges::category::random_access | ranges::category::sized> {
3232
return block;
3333
}, "A range of all of the txes in the block");
3434
func(property_tag, "inputs", [](const Block &block) -> ranges::any_view<Input> {

‎blockscipy/src/chain/blockchain_py.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ void init_blockchain(py::class_<Blockchain> &cl) {
3030
}
3131
return chain[posIndex];
3232
}, py::arg("index"))
33-
.def("__getitem__", [](Blockchain &chain, pybind11::slice slice) -> ranges::any_view<decltype(chain[0]), ranges::category::random_access> {
33+
.def("__getitem__", [](Blockchain &chain, pybind11::slice slice) -> ranges::any_view<decltype(chain[0]), ranges::category::random_access | ranges::category::sized> {
3434
size_t start, stop, step, slicelength;
3535
if (!slice.compute(chain.size(), &start, &stop, &step, &slicelength))
3636
throw pybind11::error_already_set();

‎blockscipy/src/cluster/cluster/cluster_py.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ void init_cluster_manager(pybind11::module &s) {
6060
.def("cluster_with_address", [](const ClusterManager &cm, const Address &address) -> Cluster {
6161
return cm.getCluster(address);
6262
}, py::arg("address"), "Return the cluster containing the given address")
63-
.def("clusters", [](ClusterManager &cm) -> ranges::any_view<Cluster, ranges::category::random_access> {
63+
.def("clusters", [](ClusterManager &cm) -> ranges::any_view<Cluster, ranges::category::random_access | ranges::category::sized> {
6464
return cm.getClusters();
6565
}, "Get a list of all clusters (The list is lazy so there is no cost to calling this method)")
6666
.def("tagged_clusters", [](ClusterManager &cm, const std::unordered_map<blocksci::Address, std::string> &tags) -> ranges::any_view<TaggedCluster> {

‎blockscipy/src/method_types.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ struct PythonTypeName<ranges::optional<T>> {
223223
};
224224

225225
template <typename T>
226-
struct PythonTypeName<ranges::any_view<T, ranges::category::random_access>> {
226+
struct PythonTypeName<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>> {
227227
static std::string name() {
228228
return PythonTypeName<T>::name() + "Range";
229229
}
@@ -237,7 +237,7 @@ struct PythonTypeName<ranges::any_view<T>> {
237237
};
238238

239239
template <typename T>
240-
struct PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access>> {
240+
struct PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access | ranges::category::sized>> {
241241
static std::string name() {
242242
return PythonTypeName<T>::name() + "OptionalRange";
243243
}

‎blockscipy/src/range_conversion.cpp

+5-5
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ typename NumpyConverter<blocksci::uint160>::type NumpyConverter<blocksci::uint16
5858

5959
template <typename T>
6060
pybind11::list convertRangeToPythonPy(T && t) {
61-
if constexpr (ranges::RandomAccessRange<T>()) {
62-
auto rangeSize = static_cast<size_t>(ranges::distance(t));
61+
if constexpr (ranges::SizedRange<T>()) {
62+
auto rangeSize = static_cast<size_t>(ranges::size(t));
6363
pybind11::list list{rangeSize};
6464
RANGES_FOR(auto && a, t) {
6565
list.append(std::forward<decltype(a)>(a));
@@ -80,8 +80,8 @@ pybind11::array_t<typename NumpyConverter<ranges::range_value_type_t<T>>::type>
8080
using numpy_value_type = typename numpy_converter::type;
8181

8282
auto numpy_converted = std::forward<T>(t) | ranges::view::transform(numpy_converter{});
83-
if constexpr (ranges::RandomAccessRange<T>()) {
84-
auto rangeSize = static_cast<size_t>(ranges::distance(numpy_converted));
83+
if constexpr (ranges::SizedRange<T>()) {
84+
auto rangeSize = static_cast<size_t>(ranges::size(numpy_converted));
8585
pybind11::array_t<numpy_value_type> ret{rangeSize};
8686
auto retPtr = ret.mutable_data();
8787
ranges::copy(numpy_converted, retPtr);
@@ -110,7 +110,7 @@ converted_range_t<T> convertAnyRangeToPython(T && t) {
110110
}
111111

112112
template <typename T>
113-
using random_range = ranges::any_view<T, ranges::category::random_access>;
113+
using random_range = ranges::any_view<T, ranges::category::random_access | ranges::category::sized>;
114114

115115
#define createConvertedRangeTag(functype) \
116116
template converted_range_t<functype> convertAnyRangeToPython<functype>(functype &&);

‎blockscipy/src/range_conversion.hpp

+6-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313

1414
#include <pybind11/numpy.h>
1515

16-
#include <range/v3/distance.hpp>
1716
#include <range/v3/range_for.hpp>
1817
#include <range/v3/range_traits.hpp>
1918
#include <range/v3/view/any_view.hpp>
@@ -183,9 +182,12 @@ struct make_optional<ranges::optional<T>> { using type = ranges::optional<T>; };
183182
template <typename T>
184183
using make_optional_t = typename make_optional<T>::type;
185184

185+
template<class T> struct dependent_false : std::false_type {};
186+
186187
constexpr ranges::category getBlockSciCategory(ranges::category cat) {
187-
if ((cat & ranges::category::random_access) == ranges::category::random_access) {
188-
return ranges::category::random_access;
188+
constexpr auto randomSized = ranges::category::random_access | ranges::category::sized;
189+
if ((cat & randomSized) == randomSized) {
190+
return randomSized;
189191
} else {
190192
return ranges::category::input;
191193
}
@@ -277,7 +279,7 @@ template <typename T>
277279
struct is_any_view : std::false_type {};
278280

279281
template <typename T>
280-
struct is_any_view<ranges::any_view<T, ranges::category::random_access>> : std::true_type {};
282+
struct is_any_view<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>> : std::true_type {};
281283

282284
template <typename T>
283285
struct is_any_view<ranges::any_view<T>> : std::true_type {};

‎blockscipy/src/range_filter_apply_py.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ auto applyRangeMethodsToRange(Class &cl, Applier applier) {
114114
template <template<typename> class Applier, typename T>
115115
void applyRangeFiltersToRange(RangeClasses<T> &cls) {
116116
applyRangeMethodsToRange(cls.iterator, Applier<ranges::any_view<T>>{});
117-
applyRangeMethodsToRange(cls.range, Applier<ranges::any_view<T, ranges::category::random_access>>{});
117+
applyRangeMethodsToRange(cls.range, Applier<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>>{});
118118
}
119119

120120
#endif /* range_filter_apply_py_h */

‎blockscipy/src/ranges_py.cpp

+7-6
Original file line numberDiff line numberDiff line change
@@ -14,52 +14,53 @@ namespace py = pybind11;
1414
using namespace blocksci;
1515

1616
void init_ranges(py::module &m) {
17+
constexpr auto rangeCat = ranges::category::random_access | ranges::category::sized;
1718
{
1819
py::class_<ranges::any_view<ranges::optional<int64_t>>> cl(m, "IntOptionalIterator");
1920
addRangeMethods(cl);
2021
}
2122
{
22-
py::class_<ranges::any_view<ranges::optional<int64_t>, ranges::category::random_access>> cl(m, "IntOptionalRange");
23+
py::class_<ranges::any_view<ranges::optional<int64_t>, rangeCat>> cl(m, "IntOptionalRange");
2324
addRangeMethods(cl);
2425
}
2526
{
2627
py::class_<ranges::any_view<ranges::optional<std::chrono::system_clock::time_point>>> cl(m, "DateOptionalIterator");
2728
addRangeMethods(cl);
2829
}
2930
{
30-
py::class_<ranges::any_view<ranges::optional<std::chrono::system_clock::time_point>, ranges::category::random_access>> cl(m, "DateOptionalRange");
31+
py::class_<ranges::any_view<ranges::optional<std::chrono::system_clock::time_point>, rangeCat>> cl(m, "DateOptionalRange");
3132
addRangeMethods(cl);
3233
}
3334
{
3435
py::class_<ranges::any_view<ranges::optional<uint256>>> cl(m, "UInt256OptionalIterator");
3536
addRangeMethods(cl);
3637
}
3738
{
38-
py::class_<ranges::any_view<ranges::optional<uint256>, ranges::category::random_access>> cl(m, "UInt256OptionalRange");
39+
py::class_<ranges::any_view<ranges::optional<uint256>, rangeCat>> cl(m, "UInt256OptionalRange");
3940
addRangeMethods(cl);
4041
}
4142
{
4243
py::class_<ranges::any_view<ranges::optional<bool>>> cl(m, "BoolOptionalIterator");
4344
addRangeMethods(cl);
4445
}
4546
{
46-
py::class_<ranges::any_view<ranges::optional<bool>, ranges::category::random_access>> cl(m, "BoolOptionalRange");
47+
py::class_<ranges::any_view<ranges::optional<bool>, rangeCat>> cl(m, "BoolOptionalRange");
4748
addRangeMethods(cl);
4849
}
4950
{
5051
py::class_<ranges::any_view<ranges::optional<AddressType::Enum>>> cl(m, "AddressTypeOptionalIterator");
5152
addRangeMethods(cl);
5253
}
5354
{
54-
py::class_<ranges::any_view<ranges::optional<AddressType::Enum>, ranges::category::random_access>> cl(m, "AddressTypeOptionalRange");
55+
py::class_<ranges::any_view<ranges::optional<AddressType::Enum>, rangeCat>> cl(m, "AddressTypeOptionalRange");
5556
addRangeMethods(cl);
5657
}
5758
{
5859
py::class_<ranges::any_view<ranges::optional<py::bytes>>> cl(m, "BytesOptionalIterator");
5960
addRangeMethods(cl);
6061
}
6162
{
62-
py::class_<ranges::any_view<ranges::optional<py::bytes>, ranges::category::random_access>> cl(m, "BytesOptionalRange");
63+
py::class_<ranges::any_view<ranges::optional<py::bytes>, rangeCat>> cl(m, "BytesOptionalRange");
6364
addRangeMethods(cl);
6465
}
6566
}

‎blockscipy/src/ranges_py.hpp

+22-10
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <pybind11/functional.h>
1616

1717
#include <range/v3/range_for.hpp>
18+
#include <range/v3/size.hpp>
1819
#include <range/v3/view/any_view.hpp>
1920
#include <range/v3/view/filter.hpp>
2021
#include <range/v3/view/slice.hpp>
@@ -58,9 +59,14 @@ auto addRangeMethods(Class &cl) {
5859
if constexpr (ranges::ForwardRange<Range>()) {
5960
cl
6061
.def("__bool__", [](Range &range) { return !ranges::empty(range); })
61-
.def("__len__", [](Range &range) { return ranges::distance(range); })
62+
;
63+
}
64+
65+
if constexpr (ranges::BidirectionalRange<Range>()) {
66+
cl
67+
.def("__len__", [](Range &range) { return range.size(); })
6268
.def("__getitem__", [](Range &range, int64_t posIndex) {
63-
auto chainSize = static_cast<int64_t>(ranges::distance(range));
69+
auto chainSize = static_cast<int64_t>(range.size());
6470
if (posIndex < 0) {
6571
posIndex += chainSize;
6672
}
@@ -69,21 +75,27 @@ auto addRangeMethods(Class &cl) {
6975
}
7076
return range[posIndex];
7177
}, pybind11::arg("index"))
72-
;
73-
}
7478

75-
if constexpr (ranges::BidirectionalRange<Range>()) {
76-
cl.def("__getitem__", [](Range &range, pybind11::slice slice) {
79+
.def("__getitem__", [](Range &range, pybind11::slice slice) {
7780
size_t start, stop, step, slicelength;
78-
auto chainSize = ranges::distance(range);
81+
const auto &constRange = range;
82+
auto chainSize = ranges::size(constRange);
83+
// static_assert(std::is_integral<decltype(chainSize)>::value, "Size is not integral");
84+
// static_assert(!ranges::disable_sized_range<Range>::value, "Size is disabled");
85+
// static_assert(ranges::SizedRange<Range>(), "range is not sized");
86+
// static_assert(ranges::SizedRange<const Range>(), "const range is not sized");
7987
if (!slice.compute(chainSize, &start, &stop, &step, &slicelength))
8088
throw pybind11::error_already_set();
8189

8290
auto subset = ranges::view::slice(range,
8391
static_cast<ranges::range_difference_type_t<Range>>(start),
8492
static_cast<ranges::range_difference_type_t<Range>>(stop));
85-
return convertRangeToPython(subset | ranges::view::stride(step));
86-
}, pybind11::arg("slice"));
93+
// static_assert(ranges::SizedRange<const decltype(subset)>(), "slice is not sized");
94+
auto strided = subset | ranges::view::stride(step);
95+
// static_assert(ranges::SizedRange<const decltype(strided)>(), "strided is not sized");
96+
return convertRangeToPython(strided);
97+
}, pybind11::arg("slice"))
98+
;
8799
}
88100

89101

@@ -95,7 +107,7 @@ auto addRangeMethods(Class &cl) {
95107
ss << "\n\n:type: :class:`" << PythonTypeName<ranges::any_view<WrappedType, getBlockSciCategory(ranges::get_categories<Range>())>>::name() << "`";
96108

97109
std::stringstream ss2;
98-
ss2 << "\n\n:type: :class:` numpy.ndarray[bool]`";
110+
ss2 << "\n\n:type: :class:`numpy.ndarray[bool]`";
99111
cl
100112
.def_property_readonly("has_value", [](Range &range) {
101113
return convertRangeToPython(range | ranges::view::transform([](auto && val) { return val.has_value(); }));

‎external/range-v3

Submodule range-v3 updated 57 files

‎include/blocksci/chain/blockchain.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ namespace blocksci {
3434
class DataAccess;
3535

3636
template <AddressType::Enum type>
37-
using ScriptRange = ranges::any_view<ScriptAddress<type>, ranges::category::random_access>;
37+
using ScriptRange = ranges::any_view<ScriptAddress<type>, ranges::category::random_access | ranges::category::sized>;
3838
using ScriptRangeVariant = to_variadic_t<to_address_tuple_t<ScriptRange>, mpark::variant>;
3939

4040
namespace internal {

0 commit comments

Comments
 (0)