WIP Add support for recursive aggregate

src/RelationTag.h

@@ -34,6 +34,9 @@ enum class RelationTag {
     SUPPRESSED,    // warnings suppressed
     BRIE,          // use brie data-structure
     BTREE,         // use btree data-structure
+    BTREE_MIN,     // use btree_min data-structure
+    BTREE_MAX,     // use btree_max data-structure
+    BTREE_SUM,     // use btree_sum data-structure
     BTREE_DELETE,  // use btree_delete data-structure
     EQREL,         // use union data-structure
 };
@@ -56,6 +59,9 @@ enum class RelationRepresentation {
     DEFAULT,       // use default data-structure
     BRIE,          // use brie data-structure
     BTREE,         // use btree data-structure
+    BTREE_MIN,     // use btree_min data-structure
+    BTREE_MAX,     // use btree_max data-structure
+    BTREE_SUM,     // use btree_sum data-structure
     BTREE_DELETE,  // use btree_delete data-structure
     EQREL,         // use union data-structure
     PROVENANCE,    // use custom btree data-structure with provenance extras
@@ -69,6 +75,9 @@ inline bool isRelationRepresentationTag(const RelationTag& tag) {
     switch (tag) {
         case RelationTag::BRIE:
         case RelationTag::BTREE:
+        case RelationTag::BTREE_MIN:
+        case RelationTag::BTREE_MAX:
+        case RelationTag::BTREE_SUM:
         case RelationTag::BTREE_DELETE:
         case RelationTag::EQREL: return true;
         default: return false;
@@ -118,6 +127,9 @@ inline RelationRepresentation getRelationRepresentationFromTag(const RelationTag
     switch (tag) {
         case RelationTag::BRIE: return RelationRepresentation::BRIE;
         case RelationTag::BTREE: return RelationRepresentation::BTREE;
+        case RelationTag::BTREE_MIN: return RelationRepresentation::BTREE_MIN;
+        case RelationTag::BTREE_MAX: return RelationRepresentation::BTREE_MAX;
+        case RelationTag::BTREE_SUM: return RelationRepresentation::BTREE_SUM;
         case RelationTag::BTREE_DELETE: return RelationRepresentation::BTREE_DELETE;
         case RelationTag::EQREL: return RelationRepresentation::EQREL;
         default: fatal("invalid relation tag");
@@ -139,6 +151,9 @@ inline std::ostream& operator<<(std::ostream& os, RelationTag qualifier) {
         case RelationTag::SUPPRESSED: return os << "suppressed";
         case RelationTag::BRIE: return os << "brie";
         case RelationTag::BTREE: return os << "btree";
+        case RelationTag::BTREE_MIN: return os << "btree_min";
+        case RelationTag::BTREE_MAX: return os << "btree_max";
+        case RelationTag::BTREE_SUM: return os << "btree_sum";
         case RelationTag::BTREE_DELETE: return os << "btree_delete";
         case RelationTag::EQREL: return os << "eqrel";
     }
@@ -165,6 +180,9 @@ inline std::ostream& operator<<(std::ostream& os, RelationQualifier qualifier) {
 inline std::ostream& operator<<(std::ostream& os, RelationRepresentation representation) {
     switch (representation) {
         case RelationRepresentation::BTREE: return os << "btree";
+        case RelationRepresentation::BTREE_MIN: return os << "btree_min";
+        case RelationRepresentation::BTREE_MAX: return os << "btree_max";
+        case RelationRepresentation::BTREE_SUM: return os << "btree_sum";
         case RelationRepresentation::BTREE_DELETE: return os << "btree_delete";
         case RelationRepresentation::BRIE: return os << "brie";
         case RelationRepresentation::EQREL: return os << "eqrel";

src/ast/transform/MinimiseProgram.cpp

@@ -267,6 +267,12 @@ bool MinimiseProgramTransformer::reduceLocallyEquivalentClauses(TranslationUnit&
         for (auto&& cl : program.getClauses(*rel)) {
             auto* clause = &*cl;
             bool added = false;
+            const auto* head = clause->getHead();
+            const auto* relation = program.getRelation(*head);
+            // keep duplicate clauses for btree_sum
+            if (relation->getRepresentation() == RelationRepresentation::BTREE_SUM) {
+                continue;
+            }
 
             for (std::vector<Clause*>& eqClass : equivalenceClasses) {
                 const auto& normedRep = normalisations.getNormalisation(eqClass[0]);
@@ -353,6 +359,11 @@ bool MinimiseProgramTransformer::removeRedundantClauses(TranslationUnit& transla
             return false;
         }
         const auto* head = clause->getHead();
+        const auto* relation = program.getRelation(*head);
+        // Keep duplicate clauses for btree_sum
+        if (relation->getRepresentation() == RelationRepresentation::BTREE_SUM) {
+            return false;
+        }
         for (const auto* lit : clause->getBodyLiterals()) {
             if (*head == *lit) {
                 return true;

src/ast/transform/SemanticChecker.cpp

@@ -629,6 +629,25 @@ void SemanticCheckerImpl::checkRelation(const Relation& relation) {
                     "Equivalence relation " + toString(relation.getQualifiedName()) + " is not binary",
                     relation.getSrcLoc());
         }
+    } 
+    else if (relation.getRepresentation() == RelationRepresentation::BTREE_MIN
+                || relation.getRepresentation() == RelationRepresentation::BTREE_MAX
+                || relation.getRepresentation() == RelationRepresentation::BTREE_SUM) {
+        if (relation.getArity() < 2) {
+            report.addError(
+                    "Aggregate relation " + toString(relation.getQualifiedName()) + " needs at least 2 attributes",
+                    relation.getSrcLoc());
+        }
+        // TODO: check SUM is only over numbers (unsigned, signed & float)
+        // else if (relation.getRepresentation() == RelationRepresentation::BTREE_SUM) {
+        //     const auto& attributes = relation.getAttributes();
+        //     auto&& attr = attributes[relation.getArity()-1];
+        //     auto&& typeName = attr->getTypeName();
+        //     if (!typeEnv.isPrimitiveType(typeName)) {
+        //         report.addError(
+        //                 "Summing over " + toString(typeName) + " is not supported", relation.getSrcLoc());
+        //     }
+        // }
     }
 
     // check subsumption relations
@@ -647,7 +666,7 @@ void SemanticCheckerImpl::checkRelation(const Relation& relation) {
     if (relation.getRepresentation() == RelationRepresentation::BTREE_DELETE && relation.getArity() == 0) {
         report.addError("Subsumptive relation \"" + toString(relation.getQualifiedName()) +
                                 "\"  must not be a nullary relation",
-                relation.getSrcLoc());
+                        relation.getSrcLoc());
     }
 
     // start with declaration

src/ast2ram/seminaive/ClauseTranslator.cpp

@@ -17,6 +17,7 @@
 #include "ast2ram/seminaive/ClauseTranslator.h"
 #include "Global.h"
 #include "LogStatement.h"
+#include "RelationTag.h"
 #include "ast/Aggregator.h"
 #include "ast/BranchInit.h"
 #include "ast/Clause.h"
@@ -44,6 +45,7 @@
 #include "ram/DebugInfo.h"
 #include "ram/EmptinessCheck.h"
 #include "ram/ExistenceCheck.h"
+#include "ram/AggregateExistenceCheck.h"
 #include "ram/Filter.h"
 #include "ram/FloatConstant.h"
 #include "ram/GuardedInsert.h"
@@ -507,7 +509,7 @@ Own<ram::Operation> ClauseTranslator::addNegatedDeltaAtom(
 }
 
 Own<ram::Operation> ClauseTranslator::addNegatedAtom(
-        Own<ram::Operation> op, const ast::Clause& /* clause */, const ast::Atom* atom) const {
+        Own<ram::Operation> op, const ast::Clause& clause, const ast::Atom* atom) const {
     std::size_t arity = atom->getArity();
     std::string name = getConcreteRelationName(atom->getQualifiedName());
 
@@ -522,8 +524,19 @@ Own<ram::Operation> ClauseTranslator::addNegatedAtom(
     for (std::size_t i = 0; i < arity; i++) {
         values.push_back(context.translateValue(*valueIndex, args[i]));
     }
-    return mk<ram::Filter>(
-            mk<ram::Negation>(mk<ram::ExistenceCheck>(name, std::move(values))), std::move(op));
+
+    const ast::Atom* head = clause.getHead();
+    const auto* relation = context.getProgram()->getRelation(*head);
+    if (
+        relation->getRepresentation() == RelationRepresentation::BTREE_MIN || 
+        relation->getRepresentation() == RelationRepresentation::BTREE_MAX
+    ) {
+        return mk<ram::Filter>(
+                mk<ram::Negation>(mk<ram::AggregateExistenceCheck>(name, std::move(values))), std::move(op));
+    } else {
+        return mk<ram::Filter>(
+                mk<ram::Negation>(mk<ram::ExistenceCheck>(name, std::move(values))), std::move(op));
+    }
 }
 
 Own<ram::Operation> ClauseTranslator::addBodyLiteralConstraints(

src/ast2ram/seminaive/UnitTranslator.cpp

@@ -642,9 +642,16 @@ Own<ram::Statement> UnitTranslator::generateStoreRelation(const ast::Relation* r
 Own<ram::Relation> UnitTranslator::createRamRelation(
         const ast::Relation* baseRelation, std::string ramRelationName) const {
     auto arity = baseRelation->getArity();
+    auto auxArity = 0;
     auto representation = baseRelation->getRepresentation();
     if (representation == RelationRepresentation::BTREE_DELETE && ramRelationName[0] == '@') {
         representation = RelationRepresentation::DEFAULT;
+    } else if (
+        representation == RelationRepresentation::BTREE_MIN 
+        || representation == RelationRepresentation::BTREE_MAX 
+        || representation == RelationRepresentation::BTREE_SUM 
+    ) {
+        auxArity = 1;
     }
 
     std::vector<std::string> attributeNames;
@@ -655,7 +662,7 @@ Own<ram::Relation> UnitTranslator::createRamRelation(
     }
 
     return mk<ram::Relation>(
-            ramRelationName, arity, 0, attributeNames, attributeTypeQualifiers, representation);
+            ramRelationName, arity, auxArity, attributeNames, attributeTypeQualifiers, representation);
 }
 
 VecOwn<ram::Relation> UnitTranslator::createRamRelations(const std::vector<std::size_t>& sccOrdering) const {

src/include/souffle/datastructure/BTree.h

@@ -91,8 +91,8 @@ class btree {
     /* -------------- updater utilities ------------- */
 
     mutable Updater upd;
-    void update(Key& old_k, const Key& new_k) {
-        upd.update(old_k, new_k);
+    bool update(Key& old_k, const Key& new_k) {
+        return upd.update(old_k, new_k);
     }
 
     /* -------------- the node type ----------------- */
@@ -1225,14 +1225,15 @@ class btree {
                     }
 
                     // update provenance information
-                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {
+                    if (typeid(Comparator) != typeid(WeakComparator)) {
                         if (!cur->lock.try_upgrade_to_write(cur_lease)) {
                             // start again
                             return insert(k, hints);
                         }
-                        update(*pos, k);
+                        bool changed = false;
+                        changed = update(*pos, k);
                         cur->lock.end_write();
-                        return true;
+                        return changed;
                     }
 
                     // we found the element => no check of lock necessary
@@ -1280,14 +1281,15 @@ class btree {
                 }
 
                 // update provenance information
-                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {
+                if (typeid(Comparator) != typeid(WeakComparator)) {
                     if (!cur->lock.try_upgrade_to_write(cur_lease)) {
                         // start again
                         return insert(k, hints);
                     }
-                    update(*(pos - 1), k);
+                    bool changed = false;
+                    changed = update(*(pos - 1), k);
                     cur->lock.end_write();
-                    return true;
+                    return changed;
                 }
 
                 // we found the element => done
@@ -1432,9 +1434,8 @@ class btree {
                 // early exit for sets
                 if (isSet && pos != b && weak_equal(*pos, k)) {
                     // update provenance information
-                    if (typeid(Comparator) != typeid(WeakComparator) && less(k, *pos)) {
-                        update(*pos, k);
-                        return true;
+                    if (typeid(Comparator) != typeid(WeakComparator)) {
+                        return update(*(pos - 1), k);
                     }
 
                     return false;
@@ -1458,9 +1459,8 @@ class btree {
             // early exit for sets
             if (isSet && pos != a && weak_equal(*(pos - 1), k)) {
                 // update provenance information
-                if (typeid(Comparator) != typeid(WeakComparator) && less(k, *(pos - 1))) {
-                    update(*(pos - 1), k);
-                    return true;
+                if (typeid(Comparator) != typeid(WeakComparator)) {
+                    return update(*(pos - 1), k);
                 }
 
                 return false;

src/include/souffle/datastructure/BTreeUtil.h

@@ -215,7 +215,9 @@ struct default_strategy<std::tuple<Ts...>> : public linear {};
  */
 template <typename T>
 struct updater {
-    void update(T& /* old_t */, const T& /* new_t */) {}
+    bool update(T& /* old_t */, const T& /* new_t */) {
+        return false;
+    }
 };
 
 }  // end of namespace detail

src/interpreter/Util.h

@@ -259,9 +259,18 @@ using Brie = Trie<Arity>;
 // Updater for Provenance
 template <std::size_t Arity>
 struct ProvenanceUpdater {
-    void update(t_tuple<Arity>& old_t, const t_tuple<Arity>& new_t) {
-        old_t[Arity - 2] = new_t[Arity - 2];
-        old_t[Arity - 1] = new_t[Arity - 1];
+    bool update(t_tuple<Arity>& old_t, const t_tuple<Arity>& new_t) {
+        bool changed = false;
+        if  (old_t[Arity - 2] > new_t[Arity - 2]) {
+            old_t[Arity - 2] = new_t[Arity - 2];
+            changed = true;
+        }
+        if (old_t[Arity - 1] > new_t[Arity - 1]) {
+            old_t[Arity - 1] = new_t[Arity - 1];
+            changed = true;
+        }
+
+        return changed;
     }
 };
 

src/parser/parser.yy

@@ -132,6 +132,9 @@
 %token PRINTSIZE_QUALIFIER       "relation qualifier printsize"
 %token BRIE_QUALIFIER            "BRIE datastructure qualifier"
 %token BTREE_QUALIFIER           "BTREE datastructure qualifier"
+%token BTREE_MIN_QUALIFIER       "BTREE_MIN datastructure qualifier"
+%token BTREE_MAX_QUALIFIER       "BTREE_MAX datastructure qualifier"
+%token BTREE_SUM_QUALIFIER       "BTREE_SUM datastructure qualifier"
 %token BTREE_DELETE_QUALIFIER    "BTREE_DELETE datastructure qualifier"
 %token EQREL_QUALIFIER           "equivalence relation qualifier"
 %token OVERRIDABLE_QUALIFIER     "relation qualifier overidable"
@@ -567,6 +570,18 @@ relation_tags
     {
       $$ = driver.addReprTag(RelationTag::BTREE, @2, $1);
     }
+  | relation_tags BTREE_MIN_QUALIFIER
+    {
+      $$ = driver.addReprTag(RelationTag::BTREE_MIN, @2, $1);
+    }
+  | relation_tags BTREE_MAX_QUALIFIER
+    {
+      $$ = driver.addReprTag(RelationTag::BTREE_MAX, @2, $1);
+    }
+  | relation_tags BTREE_SUM_QUALIFIER
+    {
+      $$ = driver.addReprTag(RelationTag::BTREE_SUM, @2, $1);
+    }
   | relation_tags BTREE_DELETE_QUALIFIER
     {
       $$ = driver.addReprTag(RelationTag::BTREE_DELETE, @2, $1);

src/parser/scanner.ll

@@ -187,6 +187,9 @@ WS [ \t\r\v\f]
 "no_magic"                            { return yy::parser::make_NO_MAGIC_QUALIFIER(yylloc); }
 "brie"                                { return yy::parser::make_BRIE_QUALIFIER(yylloc); }
 "btree_delete"                        { return yy::parser::make_BTREE_DELETE_QUALIFIER(yylloc); }
+"btree_min"                           { return yy::parser::make_BTREE_MIN_QUALIFIER(yylloc); }
+"btree_max"                           { return yy::parser::make_BTREE_MAX_QUALIFIER(yylloc); }
+"btree_sum"                           { return yy::parser::make_BTREE_SUM_QUALIFIER(yylloc); }
 "btree"                               { return yy::parser::make_BTREE_QUALIFIER(yylloc); }
 "min"                                 { return yy::parser::make_MIN(yylloc); }
 "max"                                 { return yy::parser::make_MAX(yylloc); }

src/ram/AggregateExistenceCheck.h

@@ -0,0 +1,55 @@
+/*
+ * Souffle - A Datalog Compiler
+ * Copyright (c) 2021, The Souffle Developers. All rights reserved
+ * Licensed under the Universal Permissive License v 1.0 as shown at:
+ * - https://opensource.org/licenses/UPL
+ * - <souffle root>/licenses/SOUFFLE-UPL.txt
+ */
+
+/************************************************************************
+ *
+ * @file AggregateExistenceCheck.h
+ *
+ * Defines a class for evaluating conditions in the Relational Algebra
+ * Machine.
+ *
+ ***********************************************************************/
+
+#pragma once
+
+#include "ram/AbstractExistenceCheck.h"
+#include "ram/Expression.h"
+#include "ram/Relation.h"
+#include "souffle/utility/MiscUtil.h"
+#include <memory>
+#include <sstream>
+#include <utility>
+#include <vector>
+
+namespace souffle::ram {
+
+/**
+ * @class AggregateExistenceCheck
+ * @brief Aggregate Existence check for a relation
+ */
+class AggregateExistenceCheck : public AbstractExistenceCheck {
+public:
+    AggregateExistenceCheck(std::string rel, VecOwn<Expression> vals)
+            : AbstractExistenceCheck(rel, std::move(vals)) {}
+
+    AggregateExistenceCheck* cloning() const override {
+        VecOwn<Expression> newValues;
+        for (auto& cur : values) {
+            newValues.emplace_back(cur->cloning());
+        }
+        return new AggregateExistenceCheck(relation, std::move(newValues));
+    }
+
+protected:
+    void print(std::ostream& os) const override {
+        os << "AGG ";
+        AbstractExistenceCheck::print(os);
+    }
+};
+
+}  // namespace souffle::ram