Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AttributeStore memory tweaks #583

Merged
merged 34 commits into from
Nov 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
697b1cf
make AttributeStore::get const
cldellow Nov 12, 2023
066b817
use a string pool for AttributeSet keys
cldellow Nov 12, 2023
22e60b9
lock-free reads for keys, vector for pairs
cldellow Nov 12, 2023
a80dd09
Have a "hot" shard for popular pairs
cldellow Nov 13, 2023
66adb1d
treat 0 as a sentinel
cldellow Nov 13, 2023
859e415
de-dupe all AttributePairs
cldellow Nov 13, 2023
73b8769
store pointers in pairMaps, optimize debug spew
cldellow Nov 13, 2023
1f08a81
use boost::container::flat_map over std::map
cldellow Nov 13, 2023
39f1c43
don't memoize hash function
cldellow Nov 13, 2023
2641612
output_object: avoid Tile_Value temporaries
cldellow Nov 13, 2023
2e66005
defer creating Tile_Value
cldellow Nov 15, 2023
60c36f3
adjust headers, remove unneeded rng
cldellow Nov 15, 2023
f06c2ac
any integer 0 <= 25 is eligible for hot pool
cldellow Nov 16, 2023
f8060f5
Use a small vector optimization for pair indexes
cldellow Nov 16, 2023
13f72ae
simplify AttributeKeyStore
cldellow Nov 16, 2023
1f43dbc
use camelCase
cldellow Nov 17, 2023
23b6a90
re-write to avoid static lifetime
cldellow Nov 17, 2023
a4b55d3
reduce lock contention
cldellow Nov 17, 2023
7570737
Improve TileCoordinates hash function
cldellow Nov 17, 2023
8ddf6de
d'oh, avoid looking up the key name needlessly
cldellow Nov 17, 2023
f9a1931
change AttributeXyz(...) to be last-written wins
cldellow Nov 17, 2023
f9ccf3c
remove misleading comment
cldellow Nov 17, 2023
6ba6db2
include deque
cldellow Nov 17, 2023
2f61bc9
include map
cldellow Nov 17, 2023
3e6df15
return vector, not set
cldellow Nov 17, 2023
9436324
avoid GNU-specific initializer
cldellow Nov 17, 2023
4e5d2ef
Revert "Improve TileCoordinates hash function"
cldellow Nov 17, 2023
063bc3c
remove dead code
cldellow Nov 18, 2023
6f31fc3
avoid copying AttributePairs
cldellow Nov 18, 2023
557a077
OutputObjects - greatly reduce need for locks
cldellow Nov 18, 2023
75ff493
AttributeKeyStore: use a TLS cache
cldellow Nov 18, 2023
de9e4d6
AttributePairStore: reduce lock contention
cldellow Nov 18, 2023
a285c93
ensure atomics are initialized
cldellow Nov 18, 2023
008ea72
don't store duplicate way geometries
cldellow Nov 18, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
425 changes: 353 additions & 72 deletions include/attribute_store.h

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions include/osm_lua_processing.h
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ class OsmLuaProcessing {
multiPolygonInited = false;
relationAccepted = false;
relationSubscript = -1;
lastStoredGeometryId = 0;
}

const inline Point getPoint() {
Expand Down Expand Up @@ -243,6 +244,9 @@ class OsmLuaProcessing {
MultiPolygon multiPolygonCache;
bool multiPolygonInited;

NodeID lastStoredGeometryId;
OutputGeometryType lastStoredGeometryType;

const class Config &config;
class LayerDefinition &layers;

Expand Down
49 changes: 2 additions & 47 deletions include/output_object.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,13 @@ std::ostream& operator<<(std::ostream& os, OutputGeometryType geomType);
#pragma pack(push, 4)
class OutputObject {

protected:
public:
OutputObject(OutputGeometryType type, uint_least8_t l, NodeID id, AttributeIndex attributes, uint mz)
: objectID(id), geomType(type), layer(l), z_order(0),
minZoom(mz), attributes(attributes)
{ }


public:
NodeID objectID : 36; // id of point/linestring/polygon
unsigned minZoom : 4; // minimum zoom level in which object is written
AttributeIndex attributes : 30; // index in attribute storage
Expand Down Expand Up @@ -79,54 +78,10 @@ class OutputObject {
* (we can't easily use find() because of the different value-type encoding -
* should be possible to improve this though)
*/
int findValue(std::vector<vector_tile::Tile_Value> *valueList, vector_tile::Tile_Value const &value) const;
int findValue(const std::vector<vector_tile::Tile_Value>* valueList, const AttributePair& value) const;
};
#pragma pack(pop)

/**
* \brief An OutputObject derived class that contains data originally from OsmMemTiles
*/
class OutputObjectPoint : public OutputObject
{
public:
OutputObjectPoint(OutputGeometryType type, uint_least8_t l, NodeID id, AttributeIndex attributes, uint minzoom)
: OutputObject(type, l, id, attributes, minzoom)
{
assert(type == POINT_);
}
};

class OutputObjectLinestring : public OutputObject
{
public:
OutputObjectLinestring(OutputGeometryType type, uint_least8_t l, NodeID id, AttributeIndex attributes, uint minzoom)
: OutputObject(type, l, id, attributes, minzoom)
{
assert(type == LINESTRING_);
}
};

class OutputObjectMultiLinestring : public OutputObject
{
public:
OutputObjectMultiLinestring(OutputGeometryType type, uint_least8_t l, NodeID id, AttributeIndex attributes, uint minzoom)
: OutputObject(type, l, id, attributes, minzoom)
{
assert(type == MULTILINESTRING_);
}
};


class OutputObjectMultiPolygon : public OutputObject
{
public:
OutputObjectMultiPolygon(OutputGeometryType type, uint_least8_t l, NodeID id, AttributeIndex attributes, uint minzoom)
: OutputObject(type, l, id, attributes, minzoom)
{
assert(type == POLYGON_);
}
};

class OutputObjectRef
{
OutputObject *oo;
Expand Down
18 changes: 10 additions & 8 deletions include/tile_data.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class TileDataSource {
protected:
std::mutex mutex;
TileIndex tileIndex;
std::deque<OutputObject> objects;
std::deque<std::deque<OutputObject>> objects;

// rtree index of large objects
using oo_rtree_param_type = boost::geometry::index::quadratic<128>;
Expand Down Expand Up @@ -55,7 +55,7 @@ class TileDataSource {

public:
TileDataSource(unsigned int baseZoom)
: baseZoom(baseZoom)
: baseZoom(baseZoom)
{ }

///This must be thread safe!
Expand All @@ -70,12 +70,7 @@ class TileDataSource {
MergeSingleTileDataAtZoom(dstIndex, zoom, baseZoom, tileIndex, dstTile);
}

OutputObjectRef CreateObject(OutputObject const &oo) {
std::lock_guard<std::mutex> lock(mutex);
objects.push_back(oo);
return &objects.back();
}

OutputObjectRef CreateObject(OutputObject const &oo);
void AddGeometryToIndex(Linestring const &geom, std::vector<OutputObjectRef> const &outputs);
void AddGeometryToIndex(MultiLinestring const &geom, std::vector<OutputObjectRef> const &outputs);
void AddGeometryToIndex(MultiPolygon const &geom, std::vector<OutputObjectRef> const &outputs);
Expand Down Expand Up @@ -103,6 +98,13 @@ class TileDataSource {
linestring_store = std::make_unique<linestring_store_t>();
multi_polygon_store = std::make_unique<multi_polygon_store_t>();
multi_linestring_store = std::make_unique<multi_linestring_store_t>();

// Put something at index 0 of all stores so that 0 can be used
// as a sentinel.
point_store->push_back(Point(0,0));
linestring_store->push_back(linestring_t());
multi_polygon_store->push_back(multi_polygon_t());
multi_linestring_store->push_back(multi_linestring_t());
}
void reportSize() const;

Expand Down
Loading
Loading