From 9efd1022f596414dfa9820d2ae9399b1c1b3d879 Mon Sep 17 00:00:00 2001 From: Ben Avison Date: Mon, 14 Jan 2019 17:52:08 +0000 Subject: [PATCH 1/6] Add a lexer for CMHG files This file format is parsed by the `cmhg` or `cmunge` tools to generate the binary object header for privileged code modules on RISC OS - the initialism stands for 'C Module Header Generator'. --- lib/rouge/demos/cmhg | 8 ++++++ lib/rouge/lexers/cmhg.rb | 31 +++++++++++++++++++++ spec/lexers/cmhg_spec.rb | 14 ++++++++++ spec/visual/samples/cmhg | 58 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 lib/rouge/demos/cmhg create mode 100644 lib/rouge/lexers/cmhg.rb create mode 100644 spec/lexers/cmhg_spec.rb create mode 100644 spec/visual/samples/cmhg diff --git a/lib/rouge/demos/cmhg b/lib/rouge/demos/cmhg new file mode 100644 index 0000000000..76eeaf9190 --- /dev/null +++ b/lib/rouge/demos/cmhg @@ -0,0 +1,8 @@ +; Header comments + +#include "definitions.h" + +command-keyword-table: command_handler + foo(min-args:0, max-args:0,; comment + international:, + invalid-syntax: "syntaxtoken" help-text: "helptoken") diff --git a/lib/rouge/lexers/cmhg.rb b/lib/rouge/lexers/cmhg.rb new file mode 100644 index 0000000000..c409434980 --- /dev/null +++ b/lib/rouge/lexers/cmhg.rb @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +module Rouge + module Lexers + class CMHG < RegexLexer + title "CMHG" + desc "RISC OS C module header generator source file" + tag 'cmhg' + filenames '*.cmhg' + + def self.preproc_keyword + @preproc_keyword ||= %w( + define elif else endif error if ifdef ifndef include line pragma undef warning + ) + end + + state :root do + rule %r/;[^\n]*/, Comment + rule %r/^[ \t]*#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})[ \t].*)?/, Comment::Preproc + rule %r/[-a-z]+:/, Keyword::Declaration + rule %r/[a-z_]\w+/i, Name::Entity + rule %r/"[^"]*"/, Literal::String + rule %r/(?:&|0x)\h+/, Literal::Number::Hex + rule %r/\d+/, Literal::Number + rule %r/[,\/()]/, Punctuation + rule %r/[ \t\n]+/, Text + end + end + end +end diff --git a/spec/lexers/cmhg_spec.rb b/spec/lexers/cmhg_spec.rb new file mode 100644 index 0000000000..f7c941b2a6 --- /dev/null +++ b/spec/lexers/cmhg_spec.rb @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +describe Rouge::Lexers::CMHG do + let(:subject) { Rouge::Lexers::CMHG.new } + + describe 'guessing' do + include Support::Guessing + + it 'guesses by filename' do + assert_guess :filename => 'foo.cmhg' + end + end +end diff --git a/spec/visual/samples/cmhg b/spec/visual/samples/cmhg new file mode 100644 index 0000000000..9bf06b1edd --- /dev/null +++ b/spec/visual/samples/cmhg @@ -0,0 +1,58 @@ +; +; CDDL HEADER START +; +; The contents of this file are subject to the terms of the +; Common Development and Distribution License (the "Licence"). +; You may not use this file except in compliance with the Licence. +; +; You can obtain a copy of the licence at +; cddl/RiscOS/Sources/HWSupport/SD/SDIODriver/LICENCE. +; See the Licence for the specific language governing permissions +; and limitations under the Licence. +; +; When distributing Covered Code, include this CDDL HEADER in each +; file and include the Licence file. If applicable, add the +; following below this CDDL HEADER, with the fields enclosed by +; brackets "[]" replaced with your own identifying information: +; Portions Copyright [yyyy] [name of copyright owner] +; +; CDDL HEADER END +; +; Copyright 2012 Ben Avison. All rights reserved. +; Portions Copyright 2013 Jeffrey Lee. +; Use is subject to license terms. +; + +#include "Global/Services.h" +#include "VersionNum" + +initialisation-code: module_initialise + +finalisation-code: module_finalise + +; Service_Hardware +service-call-handler: module_service Service_Hardware, + Service_ModulePostInit + +title-string: SDIODriver + +help-string: SDIODriver Module_MajorVersion_CMHG Module_MinorVersion_CMHG + +command-keyword-table: module_command + SDIODevices(min-args:0, max-args:0, international:, invalid-syntax:"SSDIDEV", help-text:"HSDIDEV"), + SDIOSlots(min-args:0, max-args:0, international:, invalid-syntax:"SSDISLT", help-text:"HSDISLT") + +swi-chunk-base-number: 0x59000 + +swi-handler-code: module_swi + +swi-decoding-table: SDIO, Initialise, Control, Enumerate, ControllerFeatures, ReadRegister, Op, ClaimDeviceVector, ReleaseDeviceVector + +international-help-file:"Resources:$.Resources.SDIODriver.Messages" + +date-string: Module_Date_CMHG + +generic-veneers: module_irq_veneer/module_irq_handler, + module_card_insertion_veneer/module_card_insertion_handler + +vector-handlers: module_tickerv_veneer/module_tickerv_handler From b67fb3f8b9425c9132c0e58ebecd67c79643d8ed Mon Sep 17 00:00:00 2001 From: Ben Avison Date: Tue, 30 Jul 2019 15:02:55 +0100 Subject: [PATCH 2/6] [cmhg] use \s metacharacter --- lib/rouge/lexers/cmhg.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/cmhg.rb b/lib/rouge/lexers/cmhg.rb index c409434980..443e199f62 100644 --- a/lib/rouge/lexers/cmhg.rb +++ b/lib/rouge/lexers/cmhg.rb @@ -24,7 +24,7 @@ def self.preproc_keyword rule %r/(?:&|0x)\h+/, Literal::Number::Hex rule %r/\d+/, Literal::Number rule %r/[,\/()]/, Punctuation - rule %r/[ \t\n]+/, Text + rule %r/\s+/, Text end end end From 334d3f82768a29e0f2e6bee43643c3dcb387612c Mon Sep 17 00:00:00 2001 From: Ben Avison Date: Tue, 30 Jul 2019 15:11:16 +0100 Subject: [PATCH 3/6] [cmhg] include some nonzero examples of Literal::Number in visual sample --- spec/visual/samples/cmhg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/visual/samples/cmhg b/spec/visual/samples/cmhg index 9bf06b1edd..b6b8499006 100644 --- a/spec/visual/samples/cmhg +++ b/spec/visual/samples/cmhg @@ -39,8 +39,8 @@ title-string: SDIODriver help-string: SDIODriver Module_MajorVersion_CMHG Module_MinorVersion_CMHG command-keyword-table: module_command - SDIODevices(min-args:0, max-args:0, international:, invalid-syntax:"SSDIDEV", help-text:"HSDIDEV"), - SDIOSlots(min-args:0, max-args:0, international:, invalid-syntax:"SSDISLT", help-text:"HSDISLT") + SDIODevices(min-args:0, max-args:1, international:, invalid-syntax:"SSDIDEV", help-text:"HSDIDEV"), + SDIOSlots(min-args:0, max-args:255, international:, invalid-syntax:"SSDISLT", help-text:"HSDISLT") swi-chunk-base-number: 0x59000 From 138121719eb9beb342fad2e4b33ebba1f9604509 Mon Sep 17 00:00:00 2001 From: Ben Avison Date: Tue, 30 Jul 2019 15:32:06 +0100 Subject: [PATCH 4/6] [cmhg] break consumption of whitespace at newlines Without this, testing for C preprocessor statements can't identify the leading `#` character as being the first non-whitespace character on a line if it is preceded by whitespace. --- lib/rouge/lexers/cmhg.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rouge/lexers/cmhg.rb b/lib/rouge/lexers/cmhg.rb index 443e199f62..fe63522ac7 100644 --- a/lib/rouge/lexers/cmhg.rb +++ b/lib/rouge/lexers/cmhg.rb @@ -24,7 +24,8 @@ def self.preproc_keyword rule %r/(?:&|0x)\h+/, Literal::Number::Hex rule %r/\d+/, Literal::Number rule %r/[,\/()]/, Punctuation - rule %r/\s+/, Text + rule %r/[ \t]+/, Text + rule %r/\n+/, Text end end end From ba82a3c2fdc86c89a4a6972dc3cacee8ef2e18f5 Mon Sep 17 00:00:00 2001 From: Ben Avison Date: Tue, 30 Jul 2019 15:35:56 +0100 Subject: [PATCH 5/6] [cmhg] fix lexing of #else and #endif where no spaces/tabs follow These two C preprocessor statements have no parameters, so do not require following spaces. Update visual sample to include an example of this (and of whitespace before a `#` character). --- lib/rouge/lexers/cmhg.rb | 2 +- spec/visual/samples/cmhg | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/rouge/lexers/cmhg.rb b/lib/rouge/lexers/cmhg.rb index fe63522ac7..1e6a6e9089 100644 --- a/lib/rouge/lexers/cmhg.rb +++ b/lib/rouge/lexers/cmhg.rb @@ -17,7 +17,7 @@ def self.preproc_keyword state :root do rule %r/;[^\n]*/, Comment - rule %r/^[ \t]*#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})[ \t].*)?/, Comment::Preproc + rule %r/^[ \t]*#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?(?=\n)/, Comment::Preproc rule %r/[-a-z]+:/, Keyword::Declaration rule %r/[a-z_]\w+/i, Name::Entity rule %r/"[^"]*"/, Literal::String diff --git a/spec/visual/samples/cmhg b/spec/visual/samples/cmhg index b6b8499006..90b388cde5 100644 --- a/spec/visual/samples/cmhg +++ b/spec/visual/samples/cmhg @@ -24,7 +24,9 @@ ; #include "Global/Services.h" -#include "VersionNum" +#ifndef Module_Date_CMHG + #include "VersionNum" +#endif initialisation-code: module_initialise From dc19f78145240d097d81564d9f9f7bf313953b5c Mon Sep 17 00:00:00 2001 From: Ben Avison Date: Tue, 30 Jul 2019 22:45:39 +0100 Subject: [PATCH 6/6] [cmhg] lex leading whitespace before preprocessor statement as Text --- lib/rouge/lexers/cmhg.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/rouge/lexers/cmhg.rb b/lib/rouge/lexers/cmhg.rb index 1e6a6e9089..338a3faec9 100644 --- a/lib/rouge/lexers/cmhg.rb +++ b/lib/rouge/lexers/cmhg.rb @@ -17,7 +17,9 @@ def self.preproc_keyword state :root do rule %r/;[^\n]*/, Comment - rule %r/^[ \t]*#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?(?=\n)/, Comment::Preproc + rule %r/^([ \t]*)(#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?)(?=\n)/ do + groups Text, Comment::Preproc + end rule %r/[-a-z]+:/, Keyword::Declaration rule %r/[a-z_]\w+/i, Name::Entity rule %r/"[^"]*"/, Literal::String