Address Issues 99-100: Remove CodeQL and refactor Treesitter

README.md

@@ -1,6 +1,6 @@
 <picture>
-  <source media="(prefers-color-scheme: dark)" srcset="docs/assets/images/cldk-dark.png">
-  <source media="(prefers-color-scheme: light)" srcset="docs/assets/images/cldk-light.png">
+  <source media="(prefers-color-scheme: dark)" srcset="docs/images/cldk-dark.png">
+  <source media="(prefers-color-scheme: light)" srcset="docs/images/cldk-light.png">
   <img src="docs/assets/img-light.png" alt="Logo">
 </picture>
 
@@ -23,6 +23,12 @@
   <a href="https://discord.gg/zEjz9YrmqN">
     <img src="https://dcbadge.limes.pink/api/server/https://discord.gg/zEjz9YrmqN?style=for-the-badge"/>
   </a>
+  <a href="https://github.com/YOUR_REPO/actions/workflows/test.yml">
+  <img src="https://img.shields.io/badge/Tests-120%2F120%20Passing-green?style=for-the-badge" />
+</a>
+  <a href="https://github.com/codellm-devkit/python-sdk">
+  <img src="https://img.shields.io/badge/Coverage-77%25-green?style=for-the-badge" />
+  </a>
 </p>
 
 Codellm-Devkit (CLDK) is a multilingual program analysis framework that bridges the gap between traditional static analysis tools and Large Language Models (LLMs) specialized for code (CodeLLMs). Codellm-Devkit allows developers to streamline the process of transforming raw code into actionable insights by providing a unified interface for integrating outputs from various analysis tools and preparing them for effective use by CodeLLMs.

cldk/analysis/c/c_analysis.py

@@ -142,7 +142,7 @@ def get_functions(self) -> Dict[str, CFunction]:
         """Should return  all functions in the project.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             Dict[str, Dict[str, JCallable]]: Dictionary of dictionaries of all methods in the C code with qualified class name as key and dictionary of methods in that class.
@@ -169,7 +169,7 @@ def get_C_file(self, file_name: str) -> str:
             file_name (str): The name of the file.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             str: C file name containing the given qualified class.
@@ -183,7 +183,7 @@ def get_C_compilation_unit(self, file_path: str) -> CTranslationUnit:
             file_path (str): Absolute path to C source file
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             CTranslationUnit: Compilation unit object for C source file
@@ -197,7 +197,7 @@ def get_functions_in_file(self, file_name: str) -> List[CFunction]:
             file_name (str): The name of the file.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             Dict[str, JCallable]: A dictionary of all constructors of the given class.
@@ -208,7 +208,7 @@ def get_macros(self) -> List[CMacro]:
         """Should return  a list of all macros in the C code.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             List[CMacro]: A list of all macros in the C code.
@@ -222,7 +222,7 @@ def get_macros_in_file(self, file_name: str) -> List[CMacro] | None:
             file_name (str): The name of the file.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             List[CMacro]: A list of all macros in the given file. Returns None if no macros are found.

cldk/analysis/python/treesitter/__init__.py → cldk/analysis/commons/treesitter/__init__.py

@@ -18,6 +18,7 @@
 Treesitter package
 """
 
-from cldk.analysis.python.treesitter.python_sitter import PythonSitter
+from .treesitter_java import TreesitterJava
+from .treesitter_python import TreesitterPython
 
-__all__ = ["PythonSitter"]
+__all__ = ["TreesitterJava", "TreesitterPython"]

cldk/analysis/java/treesitter/java_sitter.py → cldk/analysis/commons/treesitter/treesitter_java.py

@@ -15,20 +15,20 @@
 ################################################################################
 
 """
-JavaSitter module
+TreesitterJava module
 """
 import logging
 from itertools import groupby
 from typing import List, Set, Dict
 from tree_sitter import Language, Node, Parser, Query, Tree
 import tree_sitter_java as tsjava
-from cldk.models.treesitter import Captures
+from cldk.analysis.commons.treesitter.models import Captures
 
 logger = logging.getLogger(__name__)
 
 
 # pylint: disable=too-many-public-methods
-class JavaSitter:
+class TreesitterJava:
     """
     Treesitter for Java usecases.
     """

cldk/analysis/python/treesitter/python_sitter.py → cldk/analysis/commons/treesitter/treesitter_python.py

@@ -15,22 +15,21 @@
 ################################################################################
 
 """
-PythonSitter module
+TreesitterPython module
 """
 
-import glob
 import os
 from pathlib import Path
 from typing import List
 
-from tree_sitter import Language, Parser, Query, Node, Tree
+from tree_sitter import Language, Parser, Node, Tree
 import tree_sitter_python as tspython
 from cldk.models.python.models import PyMethod, PyClass, PyArg, PyImport, PyModule, PyCallSite
-from cldk.models.treesitter import Captures
-from cldk.utils.treesitter.tree_sitter_utils import TreeSitterUtils
+from cldk.analysis.commons.treesitter.models import Captures
+from cldk.analysis.commons.treesitter.utils.treesitter_utils import TreeSitterUtils
 
 
-class PythonSitter:
+class TreesitterPython:
     """
     Tree sitter for Python use cases.
     """
@@ -49,6 +48,7 @@ def is_parsable(self, code: str) -> bool:
         Returns:
             True if the code is parsable, False otherwise
         """
+
         def syntax_error(node):
             if node.type == "ERROR":
                 return True

cldk/utils/treesitter/__init__.py → cldk/analysis/commons/treesitter/utils/__init__.py

@@ -17,3 +17,6 @@
 """
 Treesitter package
 """
+from .treesitter_utils import TreeSitterUtils
+
+__all__ = ["TreeSitterUtils"]

cldk/analysis/java/codeanalyzer/codeanalyzer.py

@@ -28,7 +28,7 @@
 import networkx as nx
 
 from cldk.analysis import AnalysisLevel
-from cldk.analysis.java.treesitter import JavaSitter
+from cldk.analysis.commons.treesitter import TreesitterJava
 from cldk.models.java import JGraphEdges
 from cldk.models.java.enums import CRUDOperationType
 from cldk.models.java.models import JApplication, JCRUDOperation, JCallable, JField, JMethodDetail, JType, JCompilationUnit, JGraphEdgesST
@@ -303,7 +303,7 @@ def _generate_call_graph(self, using_symbol_table) -> nx.DiGraph:
             NotImplementedError("Call graph generation using symbol table is not implemented yet.")
         else:
             sdg = self.get_system_dependency_graph()
-            tsu = JavaSitter()
+            tsu = TreesitterJava()
             edge_list = [
                 (
                     (jge.source.method.signature, jge.source.klass),
@@ -511,6 +511,16 @@ def get_java_file(self, qualified_class_name) -> str:
             if (qualified_class_name) in v.type_declarations.keys():
                 return k
 
+    def get_compilation_units(self) -> List[JCompilationUnit]:
+        """Get all the compilation units in the symbol table.
+
+        Returns:
+            List[JCompilationUnit]: A list of compilation units.
+        """
+        if self.application is None:
+            self.application = self._init_codeanalyzer()
+        return self.get_symbol_table().values()
+
     def get_java_compilation_unit(self, file_path: str) -> JCompilationUnit:
         """Given the path of a Java source file, returns the compilation unit object from the symbol table.
 
@@ -672,7 +682,7 @@ def __call_graph_using_symbol_table(self, qualified_class_name: str, method_sign
             sdg = self.__raw_call_graph_using_symbol_table_target_method(target_class_name=qualified_class_name, target_method_signature=method_signature)
         else:
             sdg = self.__raw_call_graph_using_symbol_table(qualified_class_name=qualified_class_name, method_signature=method_signature)
-        tsu = JavaSitter()
+        tsu = TreesitterJava()
         edge_list = [
             (
                 (jge.source.method.signature, jge.source.klass),
@@ -895,7 +905,7 @@ def get_all_crud_operations(self) -> List[Dict[str, Union[JType, JCallable, List
         """Should return  a dictionary of all CRUD operations in the source code.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             Dict[str, List[str]]: A dictionary of all CRUD operations in the source code.
@@ -912,7 +922,7 @@ def get_all_read_operations(self) -> List[Dict[str, Union[JType, JCallable, List
         """Should return  a list of all read operations in the source code.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             List[Dict[str, Union[str, JCallable, List[CRUDOperation]]]]:: A list of all read operations in the source code.
@@ -934,7 +944,7 @@ def get_all_create_operations(self) -> List[Dict[str, Union[JType, JCallable, Li
         """Should return  a list of all create operations in the source code.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             List[Dict[str, Union[str, JCallable, List[CRUDOperation]]]]: A list of all create operations in the source code.
@@ -956,7 +966,7 @@ def get_all_update_operations(self) -> List[Dict[str, Union[JType, JCallable, Li
         """Should return  a list of all update operations in the source code.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             List[Dict[str, Union[str, JCallable, List[CRUDOperation]]]]: A list of all update operations in the source code.
@@ -979,7 +989,7 @@ def get_all_delete_operations(self) -> List[Dict[str, Union[JType, JCallable, Li
         """Should return  a list of all delete operations in the source code.
 
         Raises:
-            NotImplementedError: Raised when current AnalysisEngine does not support this function.
+            NotImplementedError: Raised when we do not support this function.
 
         Returns:
             List[Dict[str, Union[str, JCallable, List[CRUDOperation]]]]: A list of all delete operations in the source code.