Merge pull request #55 from cdm-processors/assembler-big-refactor

Refactor assembler

cocas/__init__.py

@@ -0,0 +1 @@
+from . import assembler, linker, object_file, object_module

cocas/antlr-generate.sh

@@ -1,2 +1,4 @@
-antlr4 -Dlanguage=Python3 -visitor -no-listener -Xexact-output-dir -o generated \
-    ./grammar/AsmLexer.g4 ./grammar/AsmParser.g4 ./grammar/Macro.g4 ./grammar/ObjectFileParser.g4 ./grammar/ObjectFileLexer.g4
+antlr4 -Dlanguage=Python3 -visitor -no-listener -Xexact-output-dir -o object_file/generated \
+    object_file/grammar/ObjectFileParser.g4 object_file/grammar/ObjectFileLexer.g4
+antlr4 -Dlanguage=Python3 -visitor -no-listener -Xexact-output-dir -o assembler/generated \
+    assembler/grammar/AsmLexer.g4 assembler/grammar/AsmParser.g4 assembler/grammar/Macro.g4

cocas/assembler/__init__.py

@@ -0,0 +1,5 @@
+"""Assembler module. Converts source files into object module structures"""
+
+from .assembler import assemble_files, assemble_module
+from .exceptions import AssemblerException, AssemblerExceptionTag
+from .targets import list_assembler_targets

cocas/assembler/assembler.py

@@ -0,0 +1,91 @@
+import codecs
+from itertools import chain
+from pathlib import Path
+from typing import Any, Optional
+
+import antlr4
+
+from cocas.object_module import ObjectModule
+
+from .ast_builder import build_ast
+from .macro_processor import ExpandMacrosVisitor, process_macros, read_mlb
+from .object_generator import generate_object_module
+from .targets import TargetInstructions, import_target, mlb_path
+
+
+def assemble_module(input_stream: antlr4.InputStream,
+                    target_instructions: TargetInstructions,
+                    macros_library: ExpandMacrosVisitor,
+                    filepath: Path) -> ObjectModule:
+    """
+    Convert lines of an assembler file to object code
+
+    :param input_stream: contents of file
+    :param target_instructions: information how to convert mnemonics to code segments
+    :param macros_library: standard macros of assembler
+    :param filepath: path of the file to use in error handling
+    """
+    macro_expanded_input_stream = process_macros(input_stream, macros_library, filepath)
+    r = build_ast(macro_expanded_input_stream, filepath)
+    return generate_object_module(r, target_instructions)
+
+
+def get_debug_info_path(filepath: Path,
+                        debug: Optional[Any],
+                        relative_path: Optional[Path],
+                        realpath: bool) -> Optional[Path]:
+    if debug:
+        debug_info_path = filepath.expanduser().absolute()
+        if realpath:
+            debug_info_path = debug_info_path.resolve()
+        if relative_path:
+            debug_info_path = debug_info_path.relative_to(relative_path)
+    else:
+        debug_info_path = None
+    return debug_info_path
+
+
+def assemble_files(target: str,
+                   files: list[Path],
+                   debug: bool,
+                   relative_path: Optional[Path],
+                   absolute_path: Optional[Path],
+                   realpath: bool) -> list[tuple[Path, ObjectModule]]:
+    """
+    Open and assemble multiple files into object modules
+
+    :param target: name of processor, should be valid
+    :param files: list of assembler files' paths to process
+    :param debug: if debug information should be collected
+    :param relative_path: if debug paths should be relative to some path
+    :param absolute_path: if relative paths should be converted to absolute
+    :param realpath: if paths should be converted to canonical
+    :return: list of pairs [source file path, object module]
+    """
+    _ = absolute_path
+    target_instructions = import_target(target)
+    macros_library = read_mlb(mlb_path(target))
+    objects = []
+
+    for filepath in files:
+        with filepath.open('rb') as file:
+            data = file.read()
+        data = codecs.decode(data, 'utf8', 'strict')
+        if not data.endswith('\n'):
+            data += '\n'
+        input_stream = antlr4.InputStream(data)
+        obj = assemble_module(input_stream, target_instructions, macros_library, filepath)
+
+        debug_info_path = get_debug_info_path(filepath, debug, relative_path, realpath)
+        if debug_info_path:
+            obj.source_file_path = debug_info_path
+            fp = filepath.absolute().as_posix()
+            dip = debug_info_path.as_posix()
+            for i in chain(obj.asects, obj.rsects):
+                for j in i.code_locations.values():
+                    if j.file == fp:
+                        j.file = dip
+                    else:
+                        j.file = get_debug_info_path(Path(j.file), debug, relative_path, realpath)
+        objects.append((filepath, obj))
+    return objects

cocas/ast_builder.py → cocas/assembler/ast_builder.py

@@ -1,8 +1,11 @@
 from base64 import b64decode
+from pathlib import Path
 
 from antlr4 import CommonTokenStream, InputStream
 
-from cocas.ast_nodes import (
+from cocas.object_module import CodeLocation
+
+from .ast_nodes import (
     AbsoluteSectionNode,
     BreakStatementNode,
     ConditionalStatementNode,
@@ -21,11 +24,8 @@
     UntilLoopNode,
     WhileLoopNode,
 )
-from cocas.error import AntlrErrorListener, CdmException, CdmExceptionTag
-from cocas.generated.AsmLexer import AsmLexer
-from cocas.generated.AsmParser import AsmParser
-from cocas.generated.AsmParserVisitor import AsmParserVisitor
-from cocas.location import CodeLocation
+from .exceptions import AntlrErrorListener, AssemblerException, AssemblerExceptionTag
+from .generated import AsmLexer, AsmParser, AsmParserVisitor
 
 
 # noinspection PyPep8Naming
@@ -126,8 +126,8 @@ def visitConnective_condition(self, ctx: AsmParser.Connective_conditionContext):
         cond = self.visitCondition(ctx.condition())
         cond.conjunction = ctx.conjunction().getText()
         if cond.conjunction != 'and' and cond.conjunction != 'or':
-            raise CdmException(CdmExceptionTag.ASM, self.source_path, ctx.start.line - self.line_offset,
-                               'Expected "and" or "or" in compound condition')
+            raise AssemblerException(AssemblerExceptionTag.ASM, self.source_path, ctx.start.line - self.line_offset,
+                                     'Expected "and" or "or" in compound condition')
         return cond
 
     def visitCondition(self, ctx: AsmParser.ConditionContext):
@@ -218,8 +218,8 @@ def visitStandaloneLabel(self, ctx: AsmParser.StandaloneLabelContext) -> LabelDe
         label_decl = self.visitLabel_declaration(ctx.label_declaration())
         label_decl.external = ctx.Ext() is not None
         if label_decl.entry and label_decl.external:
-            raise CdmException(CdmExceptionTag.ASM, self.source_path, ctx.start.line - self.line_offset,
-                               f'Label {label_decl.label.name} cannot be both external and entry')
+            raise AssemblerException(AssemblerExceptionTag.ASM, self.source_path, ctx.start.line - self.line_offset,
+                                     f'Label {label_decl.label.name} cannot be both external and entry')
         return label_decl
 
     def visitLabel_declaration(self, ctx: AsmParser.Label_declarationContext) -> LabelDeclarationNode:
@@ -253,16 +253,17 @@ def visitArguments(self, ctx: AsmParser.ArgumentsContext):
         return [self.visitArgument(i) for i in ctx.children if isinstance(i, AsmParser.ArgumentContext)]
 
 
-def build_ast(input_stream: InputStream, filepath: str):
+def build_ast(input_stream: InputStream, filepath: Path):
+    str_path = filepath.absolute().as_posix()
     lexer = AsmLexer(input_stream)
     lexer.removeErrorListeners()
-    lexer.addErrorListener(AntlrErrorListener(CdmExceptionTag.ASM, filepath))
+    lexer.addErrorListener(AntlrErrorListener(AssemblerExceptionTag.ASM, str_path))
     token_stream = CommonTokenStream(lexer)
     token_stream.fill()
     parser = AsmParser(token_stream)
     parser.removeErrorListeners()
-    parser.addErrorListener(AntlrErrorListener(CdmExceptionTag.ASM, filepath))
+    parser.addErrorListener(AntlrErrorListener(AssemblerExceptionTag.ASM, str_path))
     cst = parser.program()
-    bav = BuildAstVisitor(filepath)
+    bav = BuildAstVisitor(str_path)
     result = bav.visit(cst)
     return result

cocas/ast_nodes.py → cocas/assembler/ast_nodes.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import Optional
 
-from cocas.location import CodeLocation
+from cocas.object_module import CodeLocation
 
 
 @dataclass
@@ -40,7 +40,7 @@ class RelocatableExpressionNode(LocatableNode):
 
 
 @dataclass
-class LabelDeclarationNode(Node):
+class LabelDeclarationNode(LocatableNode):
     label: LabelNode
     entry: bool
     external: bool