feat: Skip macros in C++ during parsing

docs/API/knut/cppdocument.md

@@ -298,6 +298,7 @@ The returned QueryMatch instances contain the following captures:
 - `declaration`: The full declaration of the method
 - `function`: The function declaration, without the return type
 - `name`: The name of the function
+- `return-type`: The return type of the function without any reference/pointer specifiers (i.e. `&`/`*`)
 
 #### <a name="queryMethodDefinition"></a>array&lt;[QueryMatch](../knut/querymatch.md)> **queryMethodDefinition**(string scope, string methodName)
 

src/core/codedocument.cpp

@@ -848,4 +848,10 @@ AstNode CodeDocument::astNodeAt(int pos)
     return {};
 }
 
+QList<treesitter::Range> CodeDocument::includedRanges() const
+{
+    // An empty list tells the parser to include the entire document.
+    return {};
+}
+
 } // namespace Core

src/core/codedocument.h

@@ -15,6 +15,7 @@
 #include "querymatch.h"
 #include "symbol.h"
 #include "textdocument.h"
+#include "treesitter/parser.h"
 #include "treesitter/query.h"
 
 #include <functional>
@@ -79,6 +80,8 @@ class CodeDocument : public TextDocument
 
     Q_INVOKABLE Core::AstNode astNodeAt(int pos);
 
+    virtual QList<treesitter::Range> includedRanges() const;
+
 public slots:
     void selectSymbol(const QString &name, int options = NoFindFlags);
 

src/core/codedocument_p.cpp

@@ -48,7 +48,12 @@ treesitter::Parser &TreeSitterHelper::parser()
 std::optional<treesitter::Tree> &TreeSitterHelper::syntaxTree()
 {
     if (!m_tree) {
-        m_tree = parser().parseString(m_document->text());
+        auto &parser = this->parser();
+        if (!parser.setIncludedRanges(m_document->includedRanges())) {
+            spdlog::warn("TreeSitterHelper::syntaxTree: Unable to set the included ranges on the treesitter parser!");
+            parser.setIncludedRanges({});
+        }
+        m_tree = parser.parseString(m_document->text());
         if (!m_tree) {
             spdlog::warn("CodeDocument::syntaxTree: Failed to parse document {}!", m_document->fileName());
         }

src/core/core/settings.json

@@ -30,6 +30,13 @@
             "LANG_NEUTRAL": "[default]"
         }
     },
+    "cpp": {
+        "excluded_macros": [
+            "AFX_EXT_CLASS",
+            "[A-Z_]*EXPORT[A-Z_]*",
+            "Q_OBJECT"
+        ]
+    },
     "mime_types": {
         "c": "cpp_type",
         "cpp": "cpp_type",

src/core/cppdocument.cpp

@@ -692,6 +692,7 @@ MessageMap CppDocument::mfcExtractMessageMap(const QString &className /* = ""*/)
  * - `declaration`: The full declaration of the method
  * - `function`: The function declaration, without the return type
  * - `name`: The name of the function
+ * - `return-type`: The return type of the function without any reference/pointer specifiers (i.e. `&`/`*`)
  */
 Core::QueryMatchList CppDocument::queryMethodDeclaration(const QString &className, const QString &functionName)
 {
@@ -1545,4 +1546,75 @@ QStringList CppDocument::primitiveTypes() const
     return Utils::cppPrimitiveTypes();
 }
 
+QList<treesitter::Range> CppDocument::includedRanges() const
+{
+    auto macros = Settings::instance()->value<QStringList>(Settings::CppExcludedMacros);
+    if (macros.isEmpty()) {
+        return {};
+    }
+
+    QRegularExpression regex(macros.join("|"));
+    if (!regex.isValid()) {
+        spdlog::error("CppDocument::includedRanges: Failed to create regex for excluded macros: {}",
+                      regex.errorString());
+        return {};
+    }
+
+    auto document = textEdit()->document();
+
+    QList<treesitter::Range> ranges;
+    treesitter::Point lastPoint {0, 0};
+    uint32_t lastByte = 0;
+
+    for (auto block = document->firstBlock(); block.isValid(); block = block.next()) {
+        QRegularExpressionMatch match;
+        auto searchFrom = 0;
+        auto index = block.text().indexOf(regex, searchFrom, &match);
+
+        // Run this in a loop to support multiple macros on the same line.
+        while (index != -1) {
+            // We need to construct a range from the end of the last match to the start of the current match.
+            //
+            // Note that the ranges have an inclusive start and an exclusive end..
+            //
+            // Also Note that the column seems to be in bytes, not characters.
+            // This is why we multiply by sizeof(QChar) to get the correct column.
+            // At least that's what the TreeSitterInspector shows us.
+            auto endPoint = treesitter::Point {.row = static_cast<uint32_t>(block.blockNumber()),
+                                               .column = static_cast<uint32_t>(index * sizeof(QChar))};
+            ranges.push_back({.start_point = lastPoint,
+                              .end_point = endPoint,
+                              .start_byte = lastByte,
+                              // No need to add - 1 here, the ranges are exclusive at the end.
+                              .end_byte = static_cast<uint32_t>((block.position() + index) * sizeof(QChar))});
+
+            auto matchLength = match.capturedLength();
+            lastByte = static_cast<uint32_t>((block.position() + index + matchLength) * sizeof(QChar));
+            lastPoint = {.row = static_cast<uint32_t>(block.blockNumber()),
+                         .column = static_cast<uint32_t>((index + matchLength) * sizeof(QChar))};
+            if (lastPoint.column == static_cast<uint32_t>(block.length())) {
+                ++lastPoint.row;
+                lastPoint.column = 0;
+            }
+
+            searchFrom = index + matchLength;
+            index = block.text().indexOf(regex, searchFrom, &match);
+        }
+    }
+
+    if (!ranges.isEmpty()) {
+        // Add the last range, up to the end of the document, but only if we have another range.
+        // Leaving the ranges empty will parse the entire document, so that's easiest.
+        auto endPoint =
+            treesitter::Point {.row = static_cast<uint32_t>(document->blockCount() - 1),
+                               .column = static_cast<uint32_t>(document->lastBlock().length() * sizeof(QChar))};
+        ranges.push_back({.start_point = lastPoint,
+                          .end_point = endPoint,
+                          .start_byte = lastByte,
+                          .end_byte = static_cast<uint32_t>(document->characterCount() * sizeof(QChar))});
+    }
+
+    return ranges;
+}
+
 } // namespace Core

src/core/cppdocument.h

@@ -60,6 +60,8 @@ class CppDocument : public CodeDocument
     bool changeBaseClass(CppDocument *header, CppDocument *source, const QString &className,
                          const QString &newClassBaseName);
 
+    QList<treesitter::Range> includedRanges() const override;
+
 public slots:
     Core::CppDocument *openHeaderSource();
 

src/core/settings.h

@@ -49,6 +49,7 @@ class Settings : public QObject
     static inline constexpr char RcAssetFlags[] = "/rc/asset_flags";
     static inline constexpr char RcAssetColors[] = "/rc/asset_transparent_colors";
     static inline constexpr char RcLanguageMap[] = "/rc/language_map";
+    static inline constexpr char CppExcludedMacros[] = "/cpp/excluded_macros";
     static inline constexpr char SaveLogsToFile[] = "/logs/saveToFile";
     static inline constexpr char ScriptPaths[] = "/script_paths";
     static inline constexpr char Tab[] = "/text_editor/tab";

src/gui/optionsdialog.cpp

@@ -19,7 +19,9 @@
 #include <QApplication>
 #include <QDesktopServices>
 #include <QFileDialog>
+#include <QInputDialog>
 #include <QIntValidator>
+#include <QStringListModel>
 #include <QUrl>
 #include <algorithm>
 
@@ -45,6 +47,7 @@ OptionsDialog::OptionsDialog(QWidget *parent)
     initializeRcSettings();
     initializeSaveToLogFileSetting();
     initializeEnableLSPSetting();
+    initializeCppSettings();
 
     updateScriptPaths();
 }
@@ -217,6 +220,42 @@ void OptionsDialog::initializeRcSettings()
     });
 }
 
+void OptionsDialog::initializeCppSettings()
+{
+    auto *model = new QStringListModel(this);
+    model->setStringList(DEFAULT_VALUE(QStringList, CppExcludedMacros));
+    ui->cppExcludedMacros->setModel(model);
+    ui->cppExcludedMacros->setSelectionMode(QAbstractItemView::SingleSelection);
+
+    connect(ui->cppAddExcludedMacro, &QPushButton::clicked, model, [this, model]() {
+        auto ok = false;
+        auto excludeMacro = QInputDialog::getText(this, tr("Exclude a new macro"), tr("Macro name (supports Regex)"),
+                                                  QLineEdit::Normal, "", &ok);
+        if (ok) {
+            if (model->insertRows(0, 1)) {
+                model->setData(model->index(0), excludeMacro);
+            } else {
+                spdlog::warn("OptionsDialog::excludeMacro: Failed to create new row!");
+            }
+        }
+    });
+
+    connect(ui->cppRemoveExcludedMacro, &QPushButton::clicked, model, [this, model]() {
+        auto selection = ui->cppExcludedMacros->selectionModel()->selectedRows();
+        // single selection mode, so should be at most one.
+        if (!selection.isEmpty()) {
+            model->removeRows(selection.front().row(), 1);
+        }
+    });
+
+    connect(model, &QStringListModel::dataChanged, model, [model]() {
+        SET_DEFAULT_VALUE(CppExcludedMacros, model->stringList());
+    });
+    connect(model, &QStringListModel::rowsRemoved, model, [model]() {
+        SET_DEFAULT_VALUE(CppExcludedMacros, model->stringList());
+    });
+}
+
 void OptionsDialog::openUserSettings()
 {
     QDesktopServices::openUrl(QUrl::fromLocalFile(ui->userPath->text()));

src/gui/optionsdialog.h

@@ -35,6 +35,7 @@ class OptionsDialog : public QDialog
     void initializeScriptPathSettings();
     void initializeScriptBehaviorSettings();
     void initializeRcSettings();
+    void initializeCppSettings();
 
     void openUserSettings();
     void openProjectSettings();

src/gui/optionsdialog.ui

@@ -626,6 +626,49 @@
        </item>
       </layout>
      </widget>
+     <widget class="QWidget" name="stackedWidgetPage5">
+      <layout class="QVBoxLayout" name="verticalLayout_4">
+       <item>
+        <widget class="QGroupBox" name="groupBox_7">
+         <property name="title">
+          <string>C++ Files</string>
+         </property>
+         <layout class="QGridLayout" name="gridLayout_11">
+          <item row="0" column="0">
+           <widget class="QLabel" name="cppLabelExcludedMacros">
+            <property name="toolTip">
+             <string>The Macros listed here will be excluded from parsing (supports Regex).</string>
+            </property>
+            <property name="text">
+             <string>Excluded Macros</string>
+            </property>
+            <property name="alignment">
+             <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
+            </property>
+           </widget>
+          </item>
+          <item row="0" column="2">
+           <widget class="QPushButton" name="cppAddExcludedMacro">
+            <property name="text">
+             <string>Add</string>
+            </property>
+           </widget>
+          </item>
+          <item row="1" column="2">
+           <widget class="QPushButton" name="cppRemoveExcludedMacro">
+            <property name="text">
+             <string>Remove</string>
+            </property>
+           </widget>
+          </item>
+          <item row="0" column="1" rowspan="3">
+           <widget class="QListView" name="cppExcludedMacros"/>
+          </item>
+         </layout>
+        </widget>
+       </item>
+      </layout>
+     </widget>
     </widget>
    </item>
    <item row="0" column="1">
@@ -684,6 +727,11 @@
        <string>Rc Files</string>
       </property>
      </item>
+     <item>
+      <property name="text">
+       <string>C++ Files</string>
+      </property>
+     </item>
     </widget>
    </item>
   </layout>

src/gui/treesitterinspector.cpp

@@ -182,6 +182,7 @@ void TreeSitterInspector::changeText()
         Core::LoggerDisabler disableLogging;
         text = m_document->text();
     }
+    m_parser.setIncludedRanges(m_document->includedRanges());
     auto tree = m_parser.parseString(text);
     if (tree.has_value()) {
         m_treemodel.setTree(std::move(tree.value()), makePredicates(), ui->enableUnnamed->isChecked());

src/gui/treesittertreemodel.cpp

@@ -74,11 +74,13 @@ QVariant TreeSitterTreeModel::TreeNode::data(int column) const
             return QString("%1: %2").arg(fieldName, m_node.type());
         }
     case 1:
-        return QString("[%1:%2] - [%3:%4]")
+        return QString("[%1:%2](%3) - [%4:%5](%6)")
             .arg(m_node.startPoint().row)
             .arg(m_node.startPoint().column)
+            .arg(m_node.startPosition())
             .arg(m_node.endPoint().row)
-            .arg(m_node.endPoint().column);
+            .arg(m_node.endPoint().column)
+            .arg(m_node.endPosition());
     default:
         break;
     }

src/treesitter/parser.cpp

@@ -58,6 +58,11 @@ std::optional<Tree> Parser::parseString(const QString &text, const Tree *old_tre
     return tree ? Tree(tree) : std::optional<Tree> {};
 }
 
+bool Parser::setIncludedRanges(const QList<Range> &ranges)
+{
+    return ts_parser_set_included_ranges(m_parser, ranges.data(), ranges.size());
+}
+
 const TSLanguage *Parser::language() const
 {
     return ts_parser_language(m_parser);

src/treesitter/parser.h

@@ -12,13 +12,16 @@
 
 #include "core/document.h"
 #include <QString>
+#include <tree_sitter/api.h>
+#include <vector>
 
 struct TSParser;
 struct TSLanguage;
 
 namespace treesitter {
 
 class Tree;
+using Range = TSRange;
 
 class Parser
 {
@@ -37,6 +40,21 @@ class Parser
 
     std::optional<Tree> parseString(const QString &text, const Tree *old_tree = nullptr) const;
 
+    /**
+     * Parse only the given ranges.
+     * Note: if the ranges are empty, the entire document is parsed.
+     *
+     * From tree_sitter/api.h:
+     * [The] given ranges must be ordered from earliest to latest in the document,
+     * and they must not overlap. That is, the following must hold for all
+     * `i` < `length - 1`: ranges[i].end_byte <= ranges[i + 1].start_byte
+     *
+     * If this requirement is not satisfied, the operation will fail, the ranges
+     * will not be assigned, and this function will return `false`. On success,
+     * this function returns `true`
+     */
+    bool setIncludedRanges(const QList<Range> &ranges);
+
     const TSLanguage *language() const;
 
     static TSLanguage *getLanguage(Core::Document::Type type);

test_data/tst_cppdocument/treesitterExcludesMacros/AFX_EXT_CLASS.h

@@ -0,0 +1,10 @@
+#pragma once
+
+class AFX_EXT_CLASS TestClass : public AFX_EXT_CLASSBase{
+publicAFX_EXT_CLASS:
+  void testMethod();
+
+private:
+  int AFX_EXT_CLASS m_count;
+};
+AFX_EXT_CLASS