Stabilize single_node.sql and others that report illegal node removal (…

…#6751)

See
https://app.circleci.com/pipelines/github/citusdata/citus/30859/workflows/223d61db-8c1d-4909-9aea-d8e470f0368b/jobs/1009243.

src/backend/distributed/metadata/node_metadata.c

@@ -1918,6 +1918,10 @@ ErrorIfNodeContainsNonRemovablePlacements(WorkerNode *workerNode)
 {
 	int32 groupId = workerNode->groupId;
 	List *shardPlacements = AllShardPlacementsOnNodeGroup(groupId);
+
+	/* sort the list to prevent regression tests getting flaky */
+	shardPlacements = SortList(shardPlacements, CompareGroupShardPlacements);
+
 	GroupShardPlacement *placement = NULL;
 	foreach_ptr(placement, shardPlacements)
 	{

src/backend/distributed/planner/multi_physical_planner.c

@@ -5343,8 +5343,7 @@ ActiveShardPlacementLists(List *taskList)
 
 
 /*
- * CompareShardPlacements compares two shard placements by their tuple oid; this
- * oid reflects the tuple's insertion order into pg_dist_placement.
+ * CompareShardPlacements compares two shard placements by placement id.
  */
 int
 CompareShardPlacements(const void *leftElement, const void *rightElement)
@@ -5370,6 +5369,35 @@ CompareShardPlacements(const void *leftElement, const void *rightElement)
 }
 
 
+/*
+ * CompareGroupShardPlacements compares two group shard placements by placement id.
+ */
+int
+CompareGroupShardPlacements(const void *leftElement, const void *rightElement)
+{
+	const GroupShardPlacement *leftPlacement =
+		*((const GroupShardPlacement **) leftElement);
+	const GroupShardPlacement *rightPlacement =
+		*((const GroupShardPlacement **) rightElement);
+
+	uint64 leftPlacementId = leftPlacement->placementId;
+	uint64 rightPlacementId = rightPlacement->placementId;
+
+	if (leftPlacementId < rightPlacementId)
+	{
+		return -1;
+	}
+	else if (leftPlacementId > rightPlacementId)
+	{
+		return 1;
+	}
+	else
+	{
+		return 0;
+	}
+}
+
+
 /*
  * LeftRotateList returns a copy of the given list that has been cyclically
  * shifted to the left by the given rotation count. For this, the function

src/include/distributed/multi_physical_planner.h

@@ -553,6 +553,7 @@ extern bool BinaryOpExpression(Expr *clause, Node **leftOperand, Node **rightOpe
 /* helper functions */
 extern Var * MakeInt4Column(void);
 extern int CompareShardPlacements(const void *leftElement, const void *rightElement);
+extern int CompareGroupShardPlacements(const void *leftElement, const void *rightElement);
 extern bool ShardIntervalsOverlap(ShardInterval *firstInterval,
 								  ShardInterval *secondInterval);
 extern bool ShardIntervalsOverlapWithParams(Datum firstMin, Datum firstMax,