Merge pull request #622 from yakra/gtmb_rebase_5

TMBitsets for subgraph membership, etc

Author: jteresco

siteupdate/cplusplus/classes/Args/Args.cpp

@@ -7,6 +7,8 @@
 /* k */ bool Args::skipgraphs = 0;
 /* v */ bool Args::mtvertices = 0;
 /* C */ bool Args::mtcsvfiles = 0;
+/* E */ bool Args::edgecounts = 0;
+/* b */ bool Args::bitsetlogs = 0;
 /* w */ std::string Args::datapath = "../../HighwayData";
 /* s */ std::string Args::systemsfile = "systems.csv";
 /* u */ std::string Args::userlistfilepath = "../../UserData/list_files";
@@ -33,6 +35,8 @@ bool Args::init(int argc, char *argv[])
 		else if ARG(0, "-k", "--skipgraphs")		 skipgraphs = 1;
 		else if ARG(0, "-v", "--mt-vertices")		 mtvertices = 1;
 		else if ARG(0, "-C", "--mt-csvs")		 mtcsvfiles = 1;
+		else if ARG(0, "-E", "--edge-counts")		 edgecounts = 1;
+		else if ARG(0, "-b", "--bitset-logs")		 bitsetlogs = 1;
 		else if ARG(0, "-h", "--help")			{show_help(); return 1;}
 		else if ARG(1, "-w", "--datapath")		{datapath	  = argv[++n];}
 		else if ARG(1, "-s", "--systemsfile")		{systemsfile      = argv[++n];}
@@ -81,7 +85,7 @@ void Args::show_help()
 	std::cout  <<  indent << "        [-c CSVSTATFILEPATH] [-g GRAPHFILEPATH] [-k]\n";
 	std::cout  <<  indent << "        [-n NMPMERGEPATH] [-p SPLITREGIONPATH SPLITREGION]\n";
 	std::cout  <<  indent << "        [-U USERLIST [USERLIST ...]] [-t NUMTHREADS] [-e]\n";
-	std::cout  <<  indent << "        [-T TIMEPRECISION] [-v] [-C]\n";
+	std::cout  <<  indent << "        [-T TIMEPRECISION] [-v] [-C] [-E] [-b]\n";
 	std::cout  <<  indent << "        [-L COLOCATIONLIMIT] [-N NMPTHRESHOLD]\n";
 	std::cout  <<  "\n";
 	std::cout  <<  "Create SQL, stats, graphs, and log files from highway and user data for the\n";
@@ -124,6 +128,9 @@ void Args::show_help()
 	std::cout  <<  "		        timestamp readouts\n";
 	std::cout  <<  "  -v, --mt-vertices     Multi-threaded vertex construction\n";
 	std::cout  <<  "  -C, --mt-csvs         Multi-threaded stats csv files\n";
+	std::cout  <<  "  -E, --edge-counts     Report the quantity of each format graph edge\n";
+	std::cout  <<  "  -b, --bitset-logs     Write TMBitset RAM use logs for region & system\n";
+	std::cout  <<  "		        vertices & edges\n";
 	std::cout  <<  "  -L, --colocationlimit COLOCATIONLIMIT\n";
 	std::cout  <<  "		        Threshold to report colocation counts\n";
 	std::cout  <<  "  -N, --nmp-threshold NMPTHRESHOLD\n";

siteupdate/cplusplus/classes/Args/Args.h

@@ -19,6 +19,8 @@ class Args
 	/* T */ static int timeprecision;
 	/* v */ static bool mtvertices;
 	/* C */ static bool mtcsvfiles;
+	/* E */ static bool edgecounts;
+	/* b */ static bool bitsetlogs;
 	/* L */ static int colocationlimit;
 	/* N */ static double nmpthreshold; 
 		static const char* exec;

siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp

@@ -6,11 +6,11 @@
 std::vector<GraphListEntry> GraphListEntry::entries;
 size_t GraphListEntry::num; // iterator for entries
 
-GraphListEntry::GraphListEntry(std::string r, std::string d, char f, char c, std::list<Region*> *rg, std::list<HighwaySystem*> *sys, PlaceRadius *pr):
+GraphListEntry::GraphListEntry(std::string r, std::string d, char f, char c, std::vector<Region*> *rg, std::vector<HighwaySystem*> *sys, PlaceRadius *pr):
 	regions(rg), systems(sys), placeradius(pr),
 	root(r), descr(d), form(f), cat(c) {}
 
-void GraphListEntry::add_group(std::string&& r, std::string&& d, char c, std::list<Region*> *rg, std::list<HighwaySystem*> *sys, PlaceRadius *pr)
+void GraphListEntry::add_group(std::string&& r, std::string&& d, char c, std::vector<Region*> *rg, std::vector<HighwaySystem*> *sys, PlaceRadius *pr)
 {	GraphListEntry::entries.emplace_back(r, d, 's', c, rg, sys, pr);
 	GraphListEntry::entries.emplace_back(r, d, 'c', c, rg, sys, pr);
 	GraphListEntry::entries.emplace_back(r, d, 't', c, rg, sys, pr);

siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.h

@@ -1,7 +1,6 @@
 class HighwaySystem;
 class PlaceRadius;
 class Region;
-#include <list>
 #include <string>
 #include <vector>
 
@@ -12,8 +11,8 @@ class GraphListEntry
 	match column names in the "graphs" DB table. */
 	public:
 	// Info for use in threaded subgraph generation. Not used in DB.
-	std::list<Region*> *regions;
-	std::list<HighwaySystem*> *systems;
+	std::vector<Region*> *regions;
+	std::vector<HighwaySystem*> *systems;
 	PlaceRadius *placeradius;
 
 	// Info for the "graphs" DB table
@@ -29,6 +28,6 @@ class GraphListEntry
 	static size_t num; // iterator for entries
 	std::string tag();
 
-	GraphListEntry(std::string, std::string, char, char, std::list<Region*>*, std::list<HighwaySystem*>*, PlaceRadius*);
-	static void add_group(std::string&&,  std::string&&,  char, std::list<Region*>*, std::list<HighwaySystem*>*, PlaceRadius*);
+	GraphListEntry(std::string, std::string, char, char, std::vector<Region*>*, std::vector<HighwaySystem*>*, PlaceRadius*);
+	static void add_group(std::string&&,  std::string&&,  char, std::vector<Region*>*, std::vector<HighwaySystem*>*, PlaceRadius*);
 };

siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp

@@ -7,51 +7,31 @@
 #include "../Waypoint/Waypoint.h"
 #include "../../templates/contains.cpp"
 
+HGVertex* HGEdge::v_array;
+
 HGEdge::HGEdge(HighwaySegment *s)
 {	// initial construction is based on a HighwaySegment
-	written = new char[Args::numthreads];
-		  // deleted by HGEdge::detach
-	// we only need to initialize the first element, for master graphs.
-	// the rest will get zeroed out for each subgraph set.
-	written[0] = 0;
 	vertex1 = s->waypoint1->hashpoint()->vertex;
 	vertex2 = s->waypoint2->hashpoint()->vertex;
 	format = simple | collapsed | traveled;
 	segment_name = s->segment_name();
-	vertex1->incident_s_edges.push_back(this);
-	vertex2->incident_s_edges.push_back(this);
-	vertex1->incident_c_edges.push_back(this);
-	vertex2->incident_c_edges.push_back(this);
-	vertex1->incident_t_edges.push_back(this);
-	vertex2->incident_t_edges.push_back(this);
+	vertex1->incident_edges.push_back(this);
+	vertex2->incident_edges.push_back(this);
+	vertex1->edge_count++;
+	vertex2->edge_count++;
 	// canonical segment, used to reference region and list of travelers
 	// assumption: each edge/segment lives within a unique region
 	// and a 'multi-edge' would not be able to span regions as there
 	// would be a required visible waypoint at the border
 	segment = s;
 }
 
-HGEdge::HGEdge(HGVertex *vertex, unsigned char fmt_mask)
+HGEdge::HGEdge(HGVertex *vertex, unsigned char fmt_mask, HGEdge *e1, HGEdge *e2)
 {	// build by collapsing two existing edges around a common hidden vertex
-	written = new char[Args::numthreads];
-		  // deleted by HGEdge::detach
-	written[0] = 0;
+	c_idx = vertex - v_array;
 	format = fmt_mask;
-	// we know there are exactly 2 incident edges, as we
-	// checked for that, and we will replace these two
-	// with the single edge we are constructing here...
-	HGEdge *edge1 = 0;
-	HGEdge *edge2 = 0;
-	if (fmt_mask & collapsed)
-	{	// ...in the compressed graph, and/or...
-		edge1 = vertex->incident_c_edges.front();
-		edge2 = vertex->incident_c_edges.back();
-	}
-	if (fmt_mask & traveled)
-	{	// ...in the traveled graph, as appropriate
-		edge1 = vertex->incident_t_edges.front();
-		edge2 = vertex->incident_t_edges.back();
-	}
+	HGEdge *edge1 = e1;
+	HGEdge *edge2 = e2;
 	// segment names should match as routes should not start or end
 	// nor should concurrencies begin or end at a hidden point
 	if (edge1->segment_name != edge2->segment_name)
@@ -103,50 +83,31 @@ HGEdge::HGEdge(HGVertex *vertex, unsigned char fmt_mask)
 			intermediate_point_string() << " to " << *(vertex2->unique_name) << std::endl;
 	//std::cout << "DEBUG: new " << str() << std::endl;
 
+	// clear format bits of old edges
+	edge1->format &= ~fmt_mask;
+	edge2->format &= ~fmt_mask;
 	// replace edge references at our endpoints with ourself
-	edge1->detach(fmt_mask);
-	edge2->detach(fmt_mask);
-	if (fmt_mask & collapsed)
-	{	vertex1->incident_c_edges.push_back(this);
-		vertex2->incident_c_edges.push_back(this);
-	}
-	if (fmt_mask & traveled)
-	{	vertex1->incident_t_edges.push_back(this);
-		vertex2->incident_t_edges.push_back(this);
-	}
+	if (!edge1->format) edge1->detach();
+	if (!edge2->format) edge2->detach();
+	vertex1->incident_edges.push_back(this);
+	vertex2->incident_edges.push_back(this);
 }
 
-void HGEdge::detach() {return detach(format);}
-void HGEdge::detach(unsigned char fmt_mask)
+void HGEdge::detach()
 {	// detach edge from vertices in graph(s) specified in fmt_mask
-	#define DETACH(v) \
-	  for (std::list<HGEdge*>::iterator e = v.begin(); e != v.end(); e++) \
+	auto detach = [&](std::vector<HGEdge*>& v) \
+	{ for (std::vector<HGEdge*>::iterator e = v.begin(); e != v.end(); e++) \
 	    if (*e == this) \
 	    {	v.erase(e); \
 		break; \
 	    }
-	if (fmt_mask & simple)
-	{	DETACH(vertex1->incident_s_edges)
-		DETACH(vertex2->incident_s_edges)
-	}
-	if (fmt_mask & collapsed)
-	{	DETACH(vertex1->incident_c_edges)
-		DETACH(vertex2->incident_c_edges)
-	}
-	if (fmt_mask & traveled)
-	{	DETACH(vertex1->incident_t_edges)
-		DETACH(vertex2->incident_t_edges)
-	}
-	#undef DETACH
-	format &= ~fmt_mask;
-	if (!format)
-	{	delete[] written;
-		delete this;
-	}
+	};
+	detach(vertex1->incident_edges);
+	detach(vertex2->incident_edges);
 }
 
 // write line to tmg collapsed edge file
-void HGEdge::collapsed_tmg_line(std::ofstream& file, char* fstr, unsigned int threadnum, std::list<HighwaySystem*> *systems)
+void HGEdge::collapsed_tmg_line(std::ofstream& file, char* fstr, unsigned int threadnum, std::vector<HighwaySystem*> *systems)
 {	file << vertex1->c_vertex_num[threadnum] << ' ' << vertex2->c_vertex_num[threadnum] << ' ';
 	segment->write_label(file, systems);
 	for (HGVertex *intermediate : intermediate_points)
@@ -157,7 +118,7 @@ void HGEdge::collapsed_tmg_line(std::ofstream& file, char* fstr, unsigned int th
 }
 
 // write line to tmg traveled edge file
-void HGEdge::traveled_tmg_line(std::ofstream& file, char* fstr, unsigned int threadnum, std::list<HighwaySystem*> *systems, bool trav, char* code)
+void HGEdge::traveled_tmg_line(std::ofstream& file, char* fstr, unsigned int threadnum, std::vector<HighwaySystem*> *systems, bool trav, char* code)
 {	file << vertex1->t_vertex_num[threadnum] << ' ' << vertex2->t_vertex_num[threadnum] << ' ';
 	segment->write_label(file, systems);
 	file << ' ' << (trav ? segment->clinchedby_code(code, threadnum) : "0");
@@ -169,7 +130,7 @@ void HGEdge::traveled_tmg_line(std::ofstream& file, char* fstr, unsigned int thr
 }
 
 /* line appropriate for a tmg collapsed edge file, with debug info
-std::string HGEdge::debug_tmg_line(std::list<HighwaySystem*> *systems, unsigned int threadnum)
+std::string HGEdge::debug_tmg_line(std::vector<HighwaySystem*> *systems, unsigned int threadnum)
 {	std::string line = std::to_string(vertex1->c_vertex_num[threadnum]) + " [" + *vertex1->unique_name + "] " \
 			 + std::to_string(vertex2->c_vertex_num[threadnum]) + " [" + *vertex2->unique_name + "] " + label(systems);
 	char fstr[58];

siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h

@@ -3,32 +3,37 @@ class HighwaySegment;
 class HighwaySystem;
 #include <iostream>
 #include <list>
+#include <vector>
 
 class HGEdge
 {   /* This class encapsulates information needed for a highway graph
     edge that can incorporate intermediate points.
     */
 	public:
-	char *written;
 	std::string segment_name;
 	HGVertex *vertex1, *vertex2;
 	std::list<HGVertex*> intermediate_points; // if more than 1, will go from vertex1 to vertex2
 	HighwaySegment *segment;
+	uint32_t c_idx; // index of last vertex collapsed, if applicable
+			// no "real" use, only for diagnostics & logging
 	unsigned char format;
 
 	// constants for more human-readable format masks
 	static constexpr unsigned char simple = 1;
 	static constexpr unsigned char collapsed = 2;
 	static constexpr unsigned char traveled = 4;
 
+	// this avoids adding more arguments to the collapse ctor
+	// and adding more ugly code to the collapse routine in the graph ctor
+	static HGVertex* v_array;	// for calculating c_idx
+
 	HGEdge(HighwaySegment *);
-	HGEdge(HGVertex *, unsigned char);
+	HGEdge(HGVertex *, unsigned char, HGEdge*, HGEdge*);
 
 	void detach();
-	void detach(unsigned char);
-	void collapsed_tmg_line(std::ofstream&, char*, unsigned int, std::list<HighwaySystem*>*);
-	void traveled_tmg_line (std::ofstream&, char*, unsigned int, std::list<HighwaySystem*>*, bool, char*);
-	std::string debug_tmg_line(std::list<HighwaySystem*> *, unsigned int);
+	void collapsed_tmg_line(std::ofstream&, char*, unsigned int, std::vector<HighwaySystem*>*);
+	void traveled_tmg_line (std::ofstream&, char*, unsigned int, std::vector<HighwaySystem*>*, bool, char*);
+	std::string debug_tmg_line(std::vector<HighwaySystem*> *, unsigned int);
 	std::string str();
 	std::string intermediate_point_string();
 };

siteupdate/cplusplus/classes/GraphGeneration/HGVertex.cpp

@@ -7,40 +7,62 @@
 #include "../Route/Route.h"
 #include "../Waypoint/Waypoint.h"
 
+std::atomic_uint HGVertex::num_hidden(0);
+
 void HGVertex::setup(Waypoint *wpt, const std::string *n)
 {	lat = wpt->lat;
 	lng = wpt->lng;
 	wpt->vertex = this;
 	s_vertex_num = new int[Args::numthreads];
 	c_vertex_num = new int[Args::numthreads];
 	t_vertex_num = new int[Args::numthreads];
-		       // deleted by ~HGVertex, called by HighwayGraph::clear
+		       // deleted by ~HGVertex
 	unique_name = n;
+	edge_count = 0;
 	visibility = 0;
 	    // permitted values:
 	    // 0: never visible outside of simple graphs
 	    // 1: visible only in traveled graph; hidden in collapsed graph
 	    // 2: visible in both traveled & collapsed graphs
-	if (!wpt->colocated)
-	{	if (!wpt->is_hidden) visibility = 2;
-		wpt->route->region->add_vertex(this, wpt);
-		wpt->route->system->add_vertex(this);
-		return;
-	}
-	for (Waypoint *w : *(wpt->colocated))
+	if (wpt->colocated)
 	{	// will consider hidden iff all colocated waypoints are hidden
-		if (!w->is_hidden) visibility = 2;
-		w->route->region->add_vertex(this, wpt);// Yes, a region/system can get the same vertex multiple times
-		w->route->system->add_vertex(this);	// from different routes. But HighwayGraph::write_subgraphs_tmg
-	}						// gets rid of any redundancy when making the final set.
+		for (Waypoint *w : *(wpt->colocated))
+		  if (!w->is_hidden)
+		  {	visibility = 2;
+			return;
+		  }
+		num_hidden++;
+	}
+	else if (!wpt->is_hidden)
+		visibility = 2;
+	else	num_hidden++;
 }
 
 HGVertex::~HGVertex()
 {	//std::cout << "deleting vertex at " << first_waypoint->str() << std::endl;
-	while (incident_s_edges.size()) incident_s_edges.front()->detach();
-	while (incident_c_edges.size()) incident_c_edges.front()->detach();
-	while (incident_t_edges.size()) incident_t_edges.front()->detach();
 	delete[] s_vertex_num;
 	delete[] c_vertex_num;
 	delete[] t_vertex_num;
 }
+
+HGEdge* HGVertex::front(unsigned char format)
+{	for (HGEdge* e : incident_edges)
+	  if (e->format & format)
+	    return e;
+	// Using this as intended with single-format bitmasks,
+	// we should never reach this point, because this function
+	// will only be called on vertices with edge_count of 2.
+	// Nonetheless, let's stop the compiler from complaining.
+	throw this;
+}
+
+HGEdge* HGVertex::back(unsigned char format)
+{	for (auto e = incident_edges.rbegin(); e != incident_edges.rend(); ++e)
+	  if ((*e)->format & format)
+	    return *e;
+	// Using this as intended with single-format bitmasks,
+	// we should never reach this point, because this function
+	// will only be called on vertices with edge_count of 2.
+	// Nonetheless, let's stop the compiler from complaining.
+	throw this;
+}

siteupdate/cplusplus/classes/GraphGeneration/HGVertex.h

@@ -2,7 +2,8 @@ class HGEdge;
 class HighwaySystem;
 class Region;
 class Waypoint;
-#include <list>
+#include <atomic>
+#include <vector>
 #include <string>
 
 class HGVertex
@@ -12,14 +13,18 @@ class HGVertex
 	public:
 	double lat, lng;
 	const std::string *unique_name;
-	std::list<HGEdge*> incident_s_edges; // simple
-	std::list<HGEdge*> incident_c_edges; // collapsed
-	std::list<HGEdge*> incident_t_edges; // traveled
+	std::vector<HGEdge*> incident_edges;
 	int *s_vertex_num;
 	int *c_vertex_num;
 	int *t_vertex_num;
+	uint16_t edge_count;
 	char visibility;
 
+	static std::atomic_uint num_hidden;
+
 	void setup(Waypoint*, const std::string*);
 	~HGVertex();
+
+	HGEdge* front(unsigned char);
+	HGEdge* back (unsigned char);
 };