Add support for different encodings

Add an '--encoding' flag to the CLI options to allow setting a custom
encoding.

When no encoding is passed, attempt to read the encoding from BOM
Tread as UTF-8 if all else fails.
This uses the encoding_rs_io crate to do this

Author: ghaith

Cargo.lock

@@ -18,6 +18,8 @@ structopt = "0.3"
 indexmap = "1.6"
 chrono = "0.4"
 glob = "0.3.0"
+encoding_rs = "0.8"
+encoding_rs_io = "0.1"
 codespan-reporting = "0.11.1"
 
 

Cargo.toml

@@ -0,0 +1,3 @@
+PROGRAM ä
+(* Cömment *)
+END_PROGRAM

examples/encoding.st

@@ -0,0 +1,3 @@
+PROGRAM ä
+(* Cömment *)
+END_PROGRAM

examples/encoding_utf_16.st

@@ -1,5 +1,6 @@
 // Copyright (c) 2021 Ghaith Hachem and Mathias Rieder
 use std::path::Path;
+use encoding_rs::Encoding;
 use structopt::{clap::ArgGroup, StructOpt};
 
 #[derive(PartialEq, Debug)]
@@ -68,6 +69,14 @@ pub struct CompileParameters {
     )]
     pub target: Option<String>,
 
+    #[structopt(
+        long,
+        name = "encoding",
+        help = "The file encoding used to read the input-files, as defined by the Encoding Standard",
+        parse(try_from_str = parse_encoding),
+    )]
+    pub encoding: Option<&'static Encoding>,
+
     #[structopt(
         name = "input-files",
         help = "Read input from <input-files>, may be a glob expression like 'src/**/*' or a sequence of files",
@@ -78,7 +87,12 @@ pub struct CompileParameters {
     pub input: Vec<String>,
 }
 
+fn parse_encoding(encoding : &str) -> Result<&'static Encoding, String> {
+    Encoding::for_label(encoding.as_bytes()).ok_or(format!("Unknown encoding {}", encoding))
+}
+
 impl CompileParameters {
+
     pub fn parse(args: Vec<String>) -> Result<CompileParameters, ParameterError> {
         CompileParameters::from_iter_safe(args)
     }
@@ -130,6 +144,7 @@ impl CompileParameters {
 #[cfg(test)]
 mod cli_tests {
     use super::{CompileParameters, FormatOption, ParameterError};
+    use pretty_assertions::assert_eq;
     use structopt::clap::ErrorKind;
 
     fn expect_argument_error(args: Vec<String>, expected_error_kind: ErrorKind) {
@@ -255,6 +270,14 @@ mod cli_tests {
         assert_eq!(parameters.output_format_or_default(), super::DEFAULT_FORMAT);
     }
 
+    #[test]
+    fn encoding_resolution() {
+        let parameters = CompileParameters::parse(vec_of_strings!("input.st", "--ir", "--encoding", "cp1252")).unwrap();
+        assert_eq!(parameters.encoding, Some(encoding_rs::WINDOWS_1252));
+        let parameters = CompileParameters::parse(vec_of_strings!("input.st", "--ir", "--encoding", "windows-1252")).unwrap();
+        assert_eq!(parameters.encoding, Some(encoding_rs::WINDOWS_1252));
+    }
+
     #[test]
     fn valid_output_formats() {
         let parameters = CompileParameters::parse(vec_of_strings!("input.st", "--ir")).unwrap();