wip: support statements, paragraphs and sections in hw parser

server/parser/src/main/java/org/eclipse/lsp/cobol/core/cst/procedure/Paragraph.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2024 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Broadcom, Inc. - initial API and implementation
+ * DAF Trucks NV – implementation of DaCo COBOL statements
+ * and DAF development standards
+ *
+ */
+package org.eclipse.lsp.cobol.core.cst.procedure;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.eclipse.lsp.cobol.core.cst.base.CstNodeImpl;
+
+/**
+ * Paragraph node
+ */
+@Getter @Setter
+public class Paragraph extends CstNodeImpl {
+ String name;
+}

server/parser/src/main/java/org/eclipse/lsp/cobol/core/cst/ProcedureDivision.java → server/parser/src/main/java/org/eclipse/lsp/cobol/core/cst/procedure/ProcedureDivision.java

@@ -14,7 +14,7 @@
  * and DAF development standards
  *
  */
-package org.eclipse.lsp.cobol.core.cst;
+package org.eclipse.lsp.cobol.core.cst.procedure;
 
 import org.eclipse.lsp.cobol.core.cst.base.CstNodeImpl;
 

server/parser/src/main/java/org/eclipse/lsp/cobol/core/cst/procedure/Section.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2024 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Broadcom, Inc. - initial API and implementation
+ * DAF Trucks NV – implementation of DaCo COBOL statements
+ * and DAF development standards
+ *
+ */
+package org.eclipse.lsp.cobol.core.cst.procedure;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.eclipse.lsp.cobol.core.cst.base.CstNodeImpl;
+
+/**
+ * Section node
+ */
+@Getter @Setter
+public class Section extends CstNodeImpl {
+ String name;
+}

server/parser/src/main/java/org/eclipse/lsp/cobol/core/cst/procedure/Statement.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2024 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Broadcom, Inc. - initial API and implementation
+ * DAF Trucks NV – implementation of DaCo COBOL statements
+ * and DAF development standards
+ *
+ */
+package org.eclipse.lsp.cobol.core.cst.procedure;
+
+import org.eclipse.lsp.cobol.core.cst.base.CstNodeImpl;
+
+/**
+ * Probably temporary container for statements.
+ */
+public class Statement extends CstNodeImpl {
+}

server/parser/src/main/java/org/eclipse/lsp/cobol/core/hw/CobolParser.java

@@ -23,6 +23,10 @@
 import org.eclipse.lsp.cobol.core.cst.*;
 import org.eclipse.lsp.cobol.core.cst.IdentificationDivision;
 import org.eclipse.lsp.cobol.core.cst.base.CstNode;
+import org.eclipse.lsp.cobol.core.cst.procedure.Paragraph;
+import org.eclipse.lsp.cobol.core.cst.procedure.ProcedureDivision;
+import org.eclipse.lsp.cobol.core.cst.procedure.Section;
+import org.eclipse.lsp.cobol.core.cst.procedure.Statement;
 import org.eclipse.lsp4j.Position;
 import org.eclipse.lsp4j.Range;
 
@@ -207,13 +211,76 @@ private void procedureDivisionContent() {
  spaces();
  try {
  while (!isNextDivisionEofOrEop()) {
+ if (isParagraph()) {
+ paragraph();
+ } else if (isSection()) {
+ section();
+ } else {
+ statement();
+ }
+ spaces();
+ }
+ } finally {
+ ctx.popAndAttach();
+ }
+ }
+
+ private void section() {
+ spaces();
+ ctx.push(new Section());
+ consume();
+ spaces();
+ consume("SECTION");
+ spaces();
+ consume(".");
+ spaces();
+ try {
+ while (!isNextDivisionEofOrEop() && !isParagraph() && !isSection()) {
+ statement();
+ spaces();
+ }
+ } finally {
+ ctx.popAndAttach();
+ }
+ }
+
+ private boolean isSection() {
+ return matchSeq(null, "SECTION", ".");
+ }
+
+ private void paragraph() {
+ spaces();
+ ctx.push(new Paragraph());
+ consume();
+ spaces();
+ consume(".");
+ spaces();
+ try {
+ while (!isNextDivisionEofOrEop() && !isParagraph() && !isSection()) {
+ statement();
+ spaces();
+ }
+ } finally {
+ ctx.popAndAttach();
+ }
+ }
+
+ private void statement() {
+ ctx.push(new Statement());
+ try {
+ while (!match(".") && ctx.getLexer().hasMore()) {
  consume();
  }
+ optional(".");
  } finally {
  ctx.popAndAttach();
  }
  }
 
+ private boolean isParagraph() {
+ return matchSeq(null, ".");
+ }
+
  private void dataDivisionContent() {
  spaces();
  ctx.push(new DataDivision());
@@ -333,7 +400,11 @@ private void or(String... expectedTokens) {
 
  boolean sameLexeme(Token lexemeToken, String expectedLexeme, Double threshold) {
  threshold = threshold == null ? settings.getFuzzyMatchThreshold() : threshold;
- if (lexemeToken == null || lexemeToken.getLexeme() == null || expectedLexeme == null) {
+ // TODO: better matchers, for now it's ANY token when it's null
+ if (expectedLexeme == null) {
+ return true;
+ }
+ if (lexemeToken.getLexeme() == null || expectedLexeme == null) {
  return false;
  }
  /*

server/parser/src/main/java/org/eclipse/lsp/cobol/core/hw/antlradapter/AntlrAdapter.java

@@ -25,6 +25,7 @@
 import org.eclipse.lsp.cobol.core.cst.*;
 import org.eclipse.lsp.cobol.core.cst.base.CstNode;
 import org.eclipse.lsp.cobol.core.cst.IdentificationDivision;
+import org.eclipse.lsp.cobol.core.cst.procedure.ProcedureDivision;
 import org.eclipse.lsp.cobol.core.hw.TokenType;
 
 import java.util.*;

server/parser/src/test/java/org/eclipse/lsp/cobol/core/HwDivisionsTest.java

@@ -18,6 +18,7 @@
 
 import org.eclipse.lsp.cobol.core.cst.*;
 import org.eclipse.lsp.cobol.core.cst.IdentificationDivision;
+import org.eclipse.lsp.cobol.core.cst.procedure.ProcedureDivision;
 import org.eclipse.lsp.cobol.core.hw.CobolLexer;
 import org.eclipse.lsp.cobol.core.hw.CobolParser;
 import org.eclipse.lsp.cobol.core.hw.ParserSettings;

server/parser/src/test/java/org/eclipse/lsp/cobol/core/divisions/procedure/ParagraphTest.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2024 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Broadcom, Inc. - initial API and implementation
+ * DAF Trucks NV – implementation of DaCo COBOL statements
+ * and DAF development standards
+ *
+ */
+package org.eclipse.lsp.cobol.core.divisions.procedure;
+
+import org.eclipse.lsp.cobol.core.cst.procedure.Paragraph;
+import org.eclipse.lsp.cobol.core.cst.procedure.ProcedureDivision;
+import org.eclipse.lsp.cobol.core.cst.ProgramUnit;
+import org.eclipse.lsp.cobol.core.hw.CobolLexer;
+import org.eclipse.lsp.cobol.core.hw.CobolParser;
+import org.eclipse.lsp.cobol.core.hw.ParseResult;
+import org.eclipse.lsp.cobol.core.hw.ParserSettings;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test that HW parser can understand paragraphs.
+ */
+class ParagraphTest {
+ @Test
+ void test() {
+ String source = "ID DIVISION. PROGRAM-ID. str1.\n"
+ + "PROCEDURE DIVISION.\n"
+ + " DISPLAY 'OUT'.\n"
+ + " PARAG1.\n"
+ + " DISPLAY 'PARAG1'.\n"
+ + " PARAG2.\n"
+ + " DISPLAY 'PARAG2'.\n";
+ ParseResult parseResult = new CobolParser(new CobolLexer(source), new ParserSettings()).parse();
+ assertTrue(parseResult.getDiagnostics().isEmpty());
+ ProgramUnit pu = (ProgramUnit) parseResult.getSourceUnit().getChildren().get(0);
+ ProcedureDivision pd = (ProcedureDivision) pu.getChildren().get(2);
+ assertEquals(2, pd.getChildren().stream().filter(Paragraph.class::isInstance).count());
+ // TODO: check if everything is in the place
+
+ assertEquals(source, parseResult.getSourceUnit().toText());
+ }
+}

server/parser/src/test/java/org/eclipse/lsp/cobol/core/divisions/procedure/SectionTest.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2024 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * Broadcom, Inc. - initial API and implementation
+ * DAF Trucks NV – implementation of DaCo COBOL statements
+ * and DAF development standards
+ *
+ */
+package org.eclipse.lsp.cobol.core.divisions.procedure;
+
+import org.eclipse.lsp.cobol.core.cst.ProgramUnit;
+import org.eclipse.lsp.cobol.core.cst.procedure.Paragraph;
+import org.eclipse.lsp.cobol.core.cst.procedure.ProcedureDivision;
+import org.eclipse.lsp.cobol.core.cst.procedure.Section;
+import org.eclipse.lsp.cobol.core.hw.CobolLexer;
+import org.eclipse.lsp.cobol.core.hw.CobolParser;
+import org.eclipse.lsp.cobol.core.hw.ParseResult;
+import org.eclipse.lsp.cobol.core.hw.ParserSettings;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test that HW parser can understand sections.
+ */
+class SectionTest {
+ @Test
+ void test() {
+ String source = "IDENTIFICATION DIVISION.\n"
+ + " PROGRAM-ID. SECTST.\n"
+ + "PROCEDURE DIVISION.\n"
+ + " DISPLAY 'OUT'.\n"
+ + " GO TO PARAG1.\n"
+ + " SECT1 SECTION.\n"
+ + " DISPLAY 'SECT1'.\n"
+ + " PARAG1.\n"
+ + " DISPLAY 'PARAG1'.\n"
+ + " PERFORM PARAG2 OF SECT2.\n"
+ + " SECT2 SECTION.\n"
+ + " PARAG2.\n"
+ + " DISPLAY 'PARAG2'.";
+ ParseResult parseResult = new CobolParser(new CobolLexer(source), new ParserSettings()).parse();
+ assertTrue(parseResult.getDiagnostics().isEmpty());
+ ProgramUnit pu = (ProgramUnit) parseResult.getSourceUnit().getChildren().get(0);
+ ProcedureDivision pd = (ProcedureDivision) pu.getChildren().get(2);
+ assertEquals(2, pd.getChildren().stream().filter(Paragraph.class::isInstance).count());
+ assertEquals(2, pd.getChildren().stream().filter(Section.class::isInstance).count());
+ // TODO: check if everything is in the place
+
+ assertEquals(source, parseResult.getSourceUnit().toText());
+ }
+}

server/parser/src/test/java/org/eclipse/lsp/cobol/core/migration/ProgramsTest.java → server/parser/src/test/java/org/eclipse/lsp/cobol/core/migration/CompositionTest.java

@@ -32,12 +32,19 @@
 /**
  * Test antlr composition
  */
-class ProgramsTest {
+class CompositionTest {
  private static final String NESTED_PROGRAM = "ID DIVISION. PROGRAM-ID. Pr2.\n" + "END PROGRAM Pr2.\n";
  private static final String PROGRAM = "ID DIVISION. PROGRAM-ID. Pr1.\n" + NESTED_PROGRAM + "END PROGRAM Pr1.\n";
+ private static final String PROGRAM_PARA = "ID DIVISION. PROGRAM-ID. str1.\n"
+ + "PROCEDURE DIVISION.\n"
+ + " DISPLAY 'OUT'.\n"
+ + " PARAG1.\n"
+ + " DISPLAY 'PARAG1'.\n"
+ + " PARAG2.\n"
+ + " DISPLAY 'PARAG2'.\n";
 
  @Test
- void test() {
+ void nestedPrograms() {
  SourceUnit su = new CobolParser(new CobolLexer(PROGRAM), new ParserSettings()).parse().getSourceUnit();
  org.eclipse.lsp.cobol.core.CobolParser.StartRuleContext migrated = new AntlrAdapter(mock(BaseErrorListener.class), mock(DefaultErrorStrategy.class), mock(ParseTreeListener.class)).sourceUnitToStartRule(su);
  org.eclipse.lsp.cobol.core.CobolParser.StartRuleContext antlr = antlrParse(PROGRAM);
@@ -62,4 +69,9 @@ org.eclipse.lsp.cobol.core.CobolParser.StartRuleContext antlrParse(String input)
  antlrParser.addErrorListener(listener);
  return antlrParser.startRule();
  }
+
+ @Test
+ void paragraphTest() {
+ SourceUnit su = new CobolParser(new CobolLexer(PROGRAM_PARA), new ParserSettings()).parse().getSourceUnit();
+ }
 }