22093: Removes query_count from language and vastly improves performa…

…nce of size opcode, especially with queries, MAJOR (#300)

docs/language.js

@@ -1401,14 +1401,6 @@ var data = [
 		"example" : "(create_entities (list \"TestEntity\" \"Child\")\n  (lambda (null ##TargetLabel 3))\n) \n\n (compute_on_contained_entities \"TestEntity\" (list\n  (query_exists \"TargetLabel\")\n)) \n\n ; For more examples see the individual entries for each query."
 	},
 
-	{
-		"parameter" : "query_count",
-		"output" : "query",
-		"new value" : "new",
-		"description" : "When used as a compute_on_contained_entities argument, counts the number of entities that match the criteria and returns the number.",
-		"example" : "(compute_on_contained_entities \"TestEntity\" (list\n  (query_count)\n))"
-	},
-
 	{
 		"parameter" : "query_select number num_to_select [number start_offset] [number random_seed]",
 		"output" : "query",

src/Amalgam/Opcodes.cpp

@@ -255,7 +255,6 @@ void StringInternPool::InitializeStaticStrings()
 	//entity query
 	EmplaceNodeTypeString(ENT_CONTAINED_ENTITIES, "contained_entities");
 	EmplaceNodeTypeString(ENT_COMPUTE_ON_CONTAINED_ENTITIES, "compute_on_contained_entities");
-	EmplaceNodeTypeString(ENT_QUERY_COUNT, "query_count");
 	EmplaceNodeTypeString(ENT_QUERY_SELECT, "query_select");
 	EmplaceNodeTypeString(ENT_QUERY_SAMPLE, "query_sample");
 	EmplaceNodeTypeString(ENT_QUERY_WEIGHTED_SAMPLE, "query_weighted_sample");

src/Amalgam/Opcodes.h

@@ -241,7 +241,6 @@ enum EvaluableNodeType : uint8_t
 	ENT_QUERY_WEIGHTED_SAMPLE,
 	ENT_QUERY_IN_ENTITY_LIST,
 	ENT_QUERY_NOT_IN_ENTITY_LIST,
-	ENT_QUERY_COUNT,
 	ENT_QUERY_EXISTS,
 	ENT_QUERY_NOT_EXISTS,
 	ENT_QUERY_EQUALS,
@@ -339,7 +338,7 @@ constexpr OrderedChildNodeType GetInstructionOrderedChildNodeType(EvaluableNodeT
 	case ENT_CREATE_ENTITIES:
 	case ENT_CONTAINED_ENTITIES:	case ENT_COMPUTE_ON_CONTAINED_ENTITIES:
 	case ENT_QUERY_SELECT:			case ENT_QUERY_SAMPLE:									case ENT_QUERY_WEIGHTED_SAMPLE:
-	case ENT_QUERY_IN_ENTITY_LIST:	case ENT_QUERY_NOT_IN_ENTITY_LIST:						case ENT_QUERY_COUNT:
+	case ENT_QUERY_IN_ENTITY_LIST:	case ENT_QUERY_NOT_IN_ENTITY_LIST:
 	case ENT_QUERY_EXISTS:			case ENT_QUERY_NOT_EXISTS:
 	case ENT_QUERY_EQUALS:			case ENT_QUERY_NOT_EQUALS:
 	case ENT_QUERY_BETWEEN:			case ENT_QUERY_NOT_BETWEEN:
@@ -482,7 +481,7 @@ constexpr bool DoesEvaluableNodeTypeCreateScope(EvaluableNodeType t)
 //returns true if t is a query
 constexpr bool IsEvaluableNodeTypeQuery(EvaluableNodeType t)
 {
-	return (t == ENT_QUERY_SELECT || t == ENT_QUERY_IN_ENTITY_LIST || t == ENT_QUERY_NOT_IN_ENTITY_LIST || t == ENT_QUERY_COUNT
+	return (t == ENT_QUERY_SELECT || t == ENT_QUERY_IN_ENTITY_LIST || t == ENT_QUERY_NOT_IN_ENTITY_LIST
 		|| t == ENT_QUERY_SAMPLE || t == ENT_QUERY_WEIGHTED_SAMPLE || t == ENT_QUERY_EXISTS || t == ENT_QUERY_NOT_EXISTS
 		|| t == ENT_QUERY_EQUALS || t == ENT_QUERY_NOT_EQUALS
 		|| t == ENT_QUERY_BETWEEN || t == ENT_QUERY_NOT_BETWEEN || t == ENT_QUERY_AMONG || t == ENT_QUERY_NOT_AMONG

src/Amalgam/amlg_code/full_test.amlg

@@ -2513,16 +2513,6 @@
     (query_not_in_entity_list (list "Child6" "Child7"))
   )))
 
-  (print "--query_count--\n")
-  (print (compute_on_contained_entities "TestContainerExec" (list
-  	(query_count)
-  )) "\n")
-
-  (print (compute_on_contained_entities "TestContainerExec" (list
-  	(query_not_exists "q")
-	(query_count)
-  )) "\n")
-
   (print "--query_exists--\n")
   (print (contained_entities "TestContainerExec" (list
   	(query_exists "q")

src/Amalgam/entity/EntityQueries.cpp

@@ -20,7 +20,6 @@ bool EntityQueryCondition::DoesEntityMatchCondition(Entity *e)
 		case ENT_QUERY_SELECT:
 		case ENT_QUERY_SAMPLE:
 		case ENT_QUERY_WEIGHTED_SAMPLE:
-		case ENT_QUERY_COUNT:
 			//it does not fail the condition here - needs to be checked elsewhere
 			return true;
 
@@ -431,15 +430,6 @@ EvaluableNodeReference EntityQueryCondition::GetMatchingEntities(Entity *contain
 		return EvaluableNodeReference::Null();
 	}
 
-	case ENT_QUERY_COUNT:
-	{
-		//not useful unless computing
-		if(enm == nullptr)
-			return EvaluableNodeReference::Null();
-
-		return EvaluableNodeReference(enm->AllocNode(static_cast<double>(matching_entities.size())), true);
-	}
-
 	case ENT_QUERY_EXISTS:
 	{
 		//find those that match

src/Amalgam/entity/EntityQueryCaches.cpp

@@ -917,7 +917,8 @@ void EntityQueryCaches::GetMatchingEntitiesViaSamplingWithReplacement(EntityQuer
 }
 
 EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Entity *container,
-	std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm, bool return_query_value)
+	std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm,
+	bool return_query_value, bool immediate_result)
 {
 	//get the cache associated with this container
 	// use the first condition as an heuristic for building it if it doesn't exist
@@ -948,16 +949,14 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 
 		//if query_none, return results as empty list
 		if(cond.queryType == ENT_NULL)
+		{
+			if(immediate_result)
+				return EvaluableNodeReference(0.0);
 			return EvaluableNodeReference(enm->AllocNode(ENT_LIST), true);
+		}
 
 		switch(cond.queryType)
 		{
-		case ENT_QUERY_COUNT:
-			if(is_first)
-				return EvaluableNodeReference(enm->AllocNode(static_cast<double>(container->GetNumContainedEntities())), true);
-			else
-				return EvaluableNodeReference(enm->AllocNode(static_cast<double>(matching_ents.size())), true);
-
 		case ENT_QUERY_IN_ENTITY_LIST:
 		{
 			if(is_first)
@@ -1041,9 +1040,9 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 			entity_caches->GetMatchingEntities(&cond, matching_ents, compute_results, is_first, !is_last || !return_query_value);
 
 			if(compute_results.size() > 0)
-				return EvaluableNodeReference(enm->AllocNode(static_cast<double>(compute_results[0].distance)), true);
+				return enm->AllocIfNotImmediate(static_cast<double>(compute_results[0].distance), immediate_result);
 			else
-				return EvaluableNodeReference(enm->AllocNode(std::numeric_limits<double>::quiet_NaN()), true);
+				return EvaluableNodeReference::Null();
 		}
 
 		case ENT_QUERY_MODE:
@@ -1053,16 +1052,16 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 				entity_caches->GetMatchingEntities(&cond, matching_ents, compute_results, is_first, !is_last || !return_query_value);
 
 				if(compute_results.size() > 0)
-					return EvaluableNodeReference(enm->AllocNode(static_cast<double>(compute_results[0].distance)), true);
+					return enm->AllocIfNotImmediate(static_cast<double>(compute_results[0].distance), immediate_result);
 				else
-					return EvaluableNodeReference(enm->AllocNode(std::numeric_limits<double>::quiet_NaN()), true);
+					return EvaluableNodeReference::Null();
 			}
 			else if(cond.singleLabelType == ENIVT_STRING_ID)
 			{
 				StringInternPool::StringID mode = string_intern_pool.NOT_A_STRING_ID;
 
 				if(entity_caches->ComputeValueFromMatchingEntities(&cond, matching_ents, mode, is_first))
-					return EvaluableNodeReference(enm->AllocNode(ENT_STRING, mode), true);
+					return enm->AllocIfNotImmediate(mode, immediate_result);
 				else
 					return EvaluableNodeReference::Null();
 			}
@@ -1076,6 +1075,9 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 				FastHashMap<double, double, std::hash<double>, DoubleNanHashComparator> value_weights;
 				entity_caches->ComputeValuesFromMatchingEntities(&cond, matching_ents, value_weights, is_first);
 
+				if(immediate_result)
+					return EvaluableNodeReference(static_cast<double>(value_weights.size()));
+
 				EvaluableNode *assoc = enm->AllocNode(ENT_ASSOC);
 				assoc->ReserveMappedChildNodes(value_weights.size());
 
@@ -1093,6 +1095,9 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 				FastHashMap<StringInternPool::StringID, double> value_weights;
 				entity_caches->ComputeValuesFromMatchingEntities(&cond, matching_ents, value_weights, is_first);
 
+				if(immediate_result)
+					return EvaluableNodeReference(static_cast<double>(value_weights.size()));
+
 				EvaluableNode *assoc = enm->AllocNode(ENT_ASSOC);
 				assoc->ReserveMappedChildNodes(value_weights.size());
 
@@ -1238,7 +1243,12 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 
 	//if last query condition is query sample, return each sampled entity id which may include duplicates
 	if(last_query_type == ENT_QUERY_SAMPLE || last_query_type == ENT_QUERY_WEIGHTED_SAMPLE)
+	{
+		if(immediate_result)
+			return EvaluableNodeReference(static_cast<double>(indices_with_duplicates.size()));
+
 		return CreateListOfStringsIdsFromIteratorAndFunction(indices_with_duplicates, enm, entity_index_to_id);
+	}
 
 	//return data as appropriate
 	if(return_query_value && last_query != nullptr)
@@ -1252,12 +1262,18 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 			|| last_query_type == ENT_COMPUTE_ENTITY_CONVICTIONS
 			|| last_query_type == ENT_COMPUTE_ENTITY_KL_DIVERGENCES)
 		{
+			if(immediate_result)
+				return EvaluableNodeReference(static_cast<double>(compute_results.size()));
+
 			return EntityManipulation::ConvertResultsToEvaluableNodes<size_t>(compute_results,
 				enm, last_query->returnSortedList, last_query->additionalSortedListLabels,
 				[&contained_entities](auto entity_index) { return contained_entities[entity_index]; });
 		}
 		else //if there are no compute results, return an assoc of the requested labels for each entity
 		{
+			if(immediate_result)
+				return EvaluableNodeReference(static_cast<double>(matching_ents.size()));
+
 			//return assoc of distances if requested
 			EvaluableNode *query_return = enm->AllocNode(ENT_ASSOC);
 			query_return->ReserveMappedChildNodes(matching_ents.size());
@@ -1296,11 +1312,14 @@ EvaluableNodeReference EntityQueryCaches::GetMatchingEntitiesFromQueryCaches(Ent
 		}
 	}
 
+	if(immediate_result)
+		return EvaluableNodeReference(static_cast<double>(matching_ents.size()));
 	return CreateListOfStringsIdsFromIteratorAndFunction(matching_ents, enm, entity_index_to_id);
 }
 
 
-EvaluableNodeReference EntityQueryCaches::GetEntitiesMatchingQuery(EntityReadReference &container, std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm, bool return_query_value)
+EvaluableNodeReference EntityQueryCaches::GetEntitiesMatchingQuery(EntityReadReference &container,
+	std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm, bool return_query_value, bool immediate_result)
 {
 	if(_enable_SBF_datastore && CanUseQueryCaches(conditions))
 	{
@@ -1319,7 +1338,7 @@ EvaluableNodeReference EntityQueryCaches::GetEntitiesMatchingQuery(EntityReadRef
 		#endif
 		}
 
-		return GetMatchingEntitiesFromQueryCaches(container, conditions, enm, return_query_value);
+		return GetMatchingEntitiesFromQueryCaches(container, conditions, enm, return_query_value, immediate_result);
 	}
 
 	if(container == nullptr)
@@ -1360,7 +1379,7 @@ EvaluableNodeReference EntityQueryCaches::GetEntitiesMatchingQuery(EntityReadRef
 			#endif
 			}
 
-			return GetMatchingEntitiesFromQueryCaches(container, conditions, enm, return_query_value);	
+			return GetMatchingEntitiesFromQueryCaches(container, conditions, enm, return_query_value, immediate_result);	
 		}
 
 		query_return_value = conditions[cond_index].GetMatchingEntities(container, matching_entities, first_condition, (return_query_value && last_condition) ? enm : nullptr);

src/Amalgam/entity/EntityQueryCaches.h

@@ -124,11 +124,16 @@ class EntityQueryCaches
 	//searches container for contained entities matching query.
 	// if return_query_value is false, then returns a list of all IDs of matching contained entities
 	// if return_query_value is true, then returns whatever the appropriate structure is for the query type for the final query
-	static EvaluableNodeReference GetEntitiesMatchingQuery(EntityReadReference &container, std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm, bool return_query_value);
+	// if immediate_result is true, will return an immediate value as the result
+	static EvaluableNodeReference GetEntitiesMatchingQuery(EntityReadReference &container,
+		std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm,
+		bool return_query_value, bool immediate_result);
 
 	//returns the collection of entities (and optionally associated compute values) that satisfy the specified chain of query conditions
 	// uses efficient querying methods with a query database, one database per container
-	static EvaluableNodeReference GetMatchingEntitiesFromQueryCaches(Entity *container, std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm, bool return_query_value);
+	static EvaluableNodeReference GetMatchingEntitiesFromQueryCaches(Entity *container,
+		std::vector<EntityQueryCondition> &conditions, EvaluableNodeManager *enm,
+		bool return_query_value, bool immediate_result);
 
 	//the container this is a cache for
 	Entity *container;

src/Amalgam/evaluablenode/EvaluableNodeManagement.h

@@ -469,6 +469,15 @@ class EvaluableNodeManager
 		original = EvaluableNodeReference(copy, (copy->GetNumChildNodes() == 0));
 	}
 
+	//returns an EvaluableNodeReference for value, allocating if necessary based on if immediate result is needed
+	template<typename T>
+	inline EvaluableNodeReference AllocIfNotImmediate(T value, bool immediate_result)
+	{
+		if(immediate_result)
+			return EvaluableNodeReference(value);
+		return EvaluableNodeReference(AllocNode(value), true);
+	}
+
 	//attempts to reuse candidate if it is unique and change it into the specified type
 	//if candidate is not unique, then it allocates and returns a new node
 	inline EvaluableNodeReference ReuseOrAllocNode(EvaluableNodeReference candidate, EvaluableNodeType type)

src/Amalgam/evaluablenode/EvaluableNodeTreeManipulation.cpp

@@ -2130,7 +2130,6 @@ CompactHashMap<EvaluableNodeType, double> EvaluableNodeTreeManipulation::evaluab
 	{ENT_QUERY_WEIGHTED_SAMPLE,							0.2},
 	{ENT_QUERY_IN_ENTITY_LIST,							0.2},
 	{ENT_QUERY_NOT_IN_ENTITY_LIST,						0.2},
-	{ENT_QUERY_COUNT,									0.2},
 	{ENT_QUERY_EXISTS,									0.2},
 	{ENT_QUERY_NOT_EXISTS,								0.2},
 	{ENT_QUERY_EQUALS,									0.2},

src/Amalgam/interpreter/Interpreter.cpp

@@ -255,7 +255,6 @@ std::array<Interpreter::OpcodeFunction, ENT_NOT_A_BUILT_IN_TYPE + 1> Interpreter
 	&Interpreter::InterpretNode_ENT_QUERY_and_COMPUTE_opcodes,										// ENT_QUERY_WEIGHTED_SAMPLE
 	&Interpreter::InterpretNode_ENT_QUERY_and_COMPUTE_opcodes,										// ENT_QUERY_IN_ENTITY_LIST
 	&Interpreter::InterpretNode_ENT_QUERY_and_COMPUTE_opcodes,										// ENT_QUERY_NOT_IN_ENTITY_LIST
-	&Interpreter::InterpretNode_ENT_QUERY_and_COMPUTE_opcodes,										// ENT_QUERY_COUNT
 	&Interpreter::InterpretNode_ENT_QUERY_and_COMPUTE_opcodes,										// ENT_QUERY_EXISTS
 	&Interpreter::InterpretNode_ENT_QUERY_and_COMPUTE_opcodes,										// ENT_QUERY_NOT_EXISTS
 	&Interpreter::InterpretNode_ENT_QUERY_and_COMPUTE_opcodes,										// ENT_QUERY_EQUALS

src/Amalgam/interpreter/Interpreter.h

@@ -455,9 +455,7 @@ class Interpreter
 	template<typename T>
 	inline EvaluableNodeReference AllocReturn(T value, bool immediate_result)
 	{
-		if(immediate_result)
-			return EvaluableNodeReference(value);
-		return EvaluableNodeReference(evaluableNodeManager->AllocNode(value), true);
+		return evaluableNodeManager->AllocIfNotImmediate(value, immediate_result);
 	}
 
 	//like AllocReturn, but if immediate_result, then it will attempt to free candidate,

src/Amalgam/interpreter/InterpreterOpcodesEntityAccess.cpp

@@ -99,6 +99,10 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_CONTAINED_ENTITIES_and_COM
 
 		auto &contained_entities = source_entity->GetContainedEntities();
 
+		//if only looking for how many entities are contained, quickly exit
+		if(immediate_result)
+			return EvaluableNodeReference(static_cast<double>(contained_entities.size()));
+
 		//new list containing the contained entity ids to return
 		EvaluableNodeReference result(
 			evaluableNodeManager->AllocListNodeWithOrderedChildNodes(ENT_STRING, contained_entities.size()), true);
@@ -159,7 +163,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_CONTAINED_ENTITIES_and_COM
 	}
 
 	//perform query
-	auto result = EntityQueryCaches::GetEntitiesMatchingQuery(source_entity, conditionsBuffer, evaluableNodeManager, return_query_value);
+	auto result = EntityQueryCaches::GetEntitiesMatchingQuery(source_entity,
+		conditionsBuffer, evaluableNodeManager, return_query_value, immediate_result);
 
 	//free query_params after the query just in case query_params is the only place that a given string id exists,
 	//so the value isn't swapped out

src/Amalgam/interpreter/InterpreterOpcodesListManipulation.cpp

@@ -530,22 +530,36 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SIZE(EvaluableNode *en, bo
 	if(ocn.size() == 0)
 		return EvaluableNodeReference::Null();
 
-	auto cur = InterpretNodeForImmediateUse(ocn[0]);
-	size_t size = 0;
-	if(cur != nullptr)
+	auto n = InterpretNodeForImmediateUse(ocn[0], true);
+
+	double size = 0;
+	if(n.IsImmediateValue())
+	{
+		auto &value = n.GetValue();
+
+		if(value.nodeType == ENIVT_NUMBER)
+			size = value.nodeValue.number;
+		if(value.nodeType == ENIVT_STRING_ID)
+			size = static_cast<double>(StringManipulation::GetNumUTF8Characters(value.nodeValue.stringID->string));
+		else if(value.nodeType == ENIVT_CODE && value.nodeValue.code != nullptr)
+			size = static_cast<double>(value.nodeValue.code->GetNumChildNodes());
+
+		return AllocReturn(size, immediate_result);
+	}
+	else if(n != nullptr)
 	{
-		if(cur->GetType() == ENT_STRING)
+		if(n->GetType() == ENT_STRING)
 		{
-			auto &s = cur->GetStringValue();
-			size = StringManipulation::GetNumUTF8Characters(s);
+			auto &s = n->GetStringValue();
+			size = static_cast<double>(StringManipulation::GetNumUTF8Characters(s));
 		}
 		else
 		{
-			size = cur->GetNumChildNodes();
+			size = static_cast<double>(n->GetNumChildNodes());
 		}
 	}
 
-	return ReuseOrAllocReturn(cur, static_cast<double>(size), immediate_result);
+	return ReuseOrAllocReturn(n, size, immediate_result);
 }
 
 EvaluableNodeReference Interpreter::InterpretNode_ENT_RANGE(EvaluableNode *en, bool immediate_result)