Fix comments reggression (#18)

This PR fixes #12

Added a WhiteSpace ParseState and check for a comment after a
whitespace. Also return the current position of the iter to correctly
strip the comments of from the line to prevent quoted comments or
non-whitespace surrounded hash chars.

Also updated the tests to match these cases.
And added an example code to easily show the output.

.env

@@ -1,2 +1,25 @@
+# Start of .env file
+# Comment line with single ' quote
+# Comment line with double " quote
+ # Comment line, starts with space with double " quote
+
 CODEGEN_TEST_VAR1="hello!"
 CODEGEN_TEST_VAR2="'quotes within quotes'"
+CODEGEN_TEST_VAR3="double quoted with # hash in value"
+CODEGEN_TEST_VAR4='single quoted with # hash in value'
+CODEGEN_TEST_VAR5=not_quoted_with_#_hash_in_value
+CODEGEN_TEST_VAR6=not_quoted_with_comment_beheind # var6 comment
+CODEGEN_TEST_VAR7=not\ quoted\ with\ escaped\ space
+CODEGEN_TEST_VAR8="double quoted with comment beheind" # var7 comment
+ CODEGEN_TEST_VAR9="Variable starts with a whitespace"
+CODEGEN_TEST_VAR10= "Value starts with a whitespace after ="
+CODEGEN_TEST_VAR11 ="Variable ends with a whitespace before ="
+CODEGEN_TEST_MULTILINE1="First Line
+Second Line"
+CODEGEN_TEST_MULTILINE2="# First Line Comment
+Second Line
+#Third Line Comment
+Fourth Line
+" # multline2 comment
+
+# End of .env file

dotenv/examples/list_variables.rs

@@ -0,0 +1,10 @@
+use dotenvy::{dotenv_iter, Error};
+
+fn main() -> Result<(), Error> {
+    dotenvy::dotenv()?;
+    for item in dotenv_iter()? {
+        let (key, val) = item?;
+        println!("{key}={val}");
+    }
+    Ok(())
+}

dotenv/src/iter.rs

@@ -46,30 +46,38 @@ enum ParseState {
     WeakOpen,
     WeakOpenEscape,
     Comment,
+    WhiteSpace,
 }
 
-fn eval_end_state(prev_state: ParseState, buf: &str) -> ParseState {
+fn eval_end_state(prev_state: ParseState, buf: &str) -> (usize, ParseState) {
     let mut cur_state = prev_state;
+    let mut cur_pos: usize = 0;
 
-    for c in buf.chars() {
+    for (pos, c) in buf.char_indices() {
+        cur_pos = pos;
         cur_state = match cur_state {
+            ParseState::WhiteSpace => match c {
+                '#' => return (cur_pos, ParseState::Comment),
+                '\\' => ParseState::Escape,
+                '"' => ParseState::WeakOpen,
+                '\'' => ParseState::StrongOpen,
+                _ => ParseState::Complete,
+            },
             ParseState::Escape => ParseState::Complete,
             ParseState::Complete => match c {
-                '#' => return ParseState::Comment,
+                c if c.is_whitespace() && c != '\n' && c != '\r' => ParseState::WhiteSpace,
                 '\\' => ParseState::Escape,
                 '"' => ParseState::WeakOpen,
                 '\'' => ParseState::StrongOpen,
                 _ => ParseState::Complete,
             },
             ParseState::WeakOpen => match c {
-                '#' => return ParseState::Comment,
                 '\\' => ParseState::WeakOpenEscape,
                 '"' => ParseState::Complete,
                 _ => ParseState::WeakOpen,
             },
             ParseState::WeakOpenEscape => ParseState::WeakOpen,
             ParseState::StrongOpen => match c {
-                '#' => return ParseState::Comment,
                 '\\' => ParseState::StrongOpenEscape,
                 '\'' => ParseState::Complete,
                 _ => ParseState::StrongOpen,
@@ -79,7 +87,7 @@ fn eval_end_state(prev_state: ParseState, buf: &str) -> ParseState {
             ParseState::Comment => panic!("should have returned early"),
         };
     }
-    cur_state
+    (cur_pos, cur_state)
 }
 
 impl<B: BufRead> Iterator for QuotedLines<B> {
@@ -89,6 +97,7 @@ impl<B: BufRead> Iterator for QuotedLines<B> {
         let mut buf = String::new();
         let mut cur_state = ParseState::Complete;
         let mut buf_pos;
+        let mut cur_pos;
         loop {
             buf_pos = buf.len();
             match self.buf.read_line(&mut buf) {
@@ -100,7 +109,12 @@ impl<B: BufRead> Iterator for QuotedLines<B> {
                     }
                 },
                 Ok(_n) => {
-                    cur_state = eval_end_state(cur_state, &buf[buf_pos..]);
+                    // Skip lines which start with a # before iteration
+                    // This optimizes parsing a bit.
+                    if buf.trim_start().starts_with('#') {
+                        return Some(Ok(String::with_capacity(0)));
+                    }
+                    (cur_pos, cur_state) = eval_end_state(cur_state, &buf[buf_pos..]);
 
                     match cur_state {
                         ParseState::Complete => {
@@ -112,16 +126,14 @@ impl<B: BufRead> Iterator for QuotedLines<B> {
                             }
                             return Some(Ok(buf));
                         }
-                        ParseState::Escape => {}
-                        ParseState::StrongOpen => {}
-                        ParseState::StrongOpenEscape => {}
-                        ParseState::WeakOpen => {}
-                        ParseState::WeakOpenEscape => {}
+                        ParseState::Escape
+                        | ParseState::StrongOpen
+                        | ParseState::StrongOpenEscape
+                        | ParseState::WeakOpen
+                        | ParseState::WeakOpenEscape
+                        | ParseState::WhiteSpace => {}
                         ParseState::Comment => {
-                            // Find the start of the comment
-                            let idx = buf.find(|c| c == '#').unwrap();
-                            // Drop the trailing comment text
-                            buf.truncate(idx);
+                            buf.truncate(buf_pos + cur_pos);
                             return Some(Ok(buf));
                         }
                     }

dotenv/tests/test-multiline-comment.rs

@@ -12,15 +12,46 @@ fn test_issue_12() {
 # Comment line with single ' quote
 # Comment line with double " quote
  # Comment line with double " quote and starts with a space
-TESTKEY=test_val # A '" comment
+TESTKEY1=test_val # 1 '" comment
+TESTKEY2=test_val_with_#_hash # 2 '" comment
+TESTKEY3="test_val quoted with # hash" # 3 '" comment
+TESTKEY4="Line 1
+# Line 2
+Line 3" # 4 Multiline "' comment
+TESTKEY5="Line 4
+# Line 5
+Line 6
+" # 5 Multiline "' comment
 # End of .env file
 "#,
     )
     .expect("should write test env");
 
     dotenv().expect("should succeed");
     assert_eq!(
-        env::var("TESTKEY").expect("test env key not set"),
+        env::var("TESTKEY1").expect("testkey1 env key not set"),
         "test_val"
     );
+    assert_eq!(
+        env::var("TESTKEY2").expect("testkey2 env key not set"),
+        "test_val_with_#_hash"
+    );
+    assert_eq!(
+        env::var("TESTKEY3").expect("testkey3 env key not set"),
+        "test_val quoted with # hash"
+    );
+    assert_eq!(
+        env::var("TESTKEY4").expect("testkey4 env key not set"),
+        r#"Line 1
+# Line 2
+Line 3"#
+    );
+    assert_eq!(
+        env::var("TESTKEY5").expect("testkey5 env key not set"),
+        r#"Line 4
+# Line 5
+Line 6
+"#
+    );
+
 }