Column width experiments

.envrc

@@ -0,0 +1 @@
+use_nix

.gitignore

@@ -9,3 +9,4 @@
 /dist-newstyle
 /.ghc.environment.*
 /result
+/.direnv

README.md

@@ -47,7 +47,7 @@ Haskell dependencies will be built by Cabal.
 
 ## Usage
 
-* `nixfmt < input.nix` – reads Nix code form `stdin`, formats it, and outputs to `stdout`
+* `nixfmt < input.nix` – reads Nix code from `stdin`, formats it, and outputs to `stdout`
 * `nixfmt file.nix` – format the file in place
 
 

flake.nix

@@ -84,6 +84,7 @@
               # nixfmt: expand
               cabal-install
               stylish-haskell
+              shellcheck
             ]);
           });
 

main/Main.hs

@@ -43,20 +43,20 @@ data Nixfmt = Nixfmt
 
 options :: Nixfmt
 options =
-  let defaultWidth = 80
+  let defaultWidth = 180
       addDefaultHint value message =
         message ++ "\n[default: " ++ show value ++ "]"
    in Nixfmt
         { files = [] &= args &= typ "FILES"
         , width =
             defaultWidth &=
             help (addDefaultHint defaultWidth "Maximum width in characters")
-        , check = False &= help "Check whether files are formatted"
+        , check = False &= help "Check whether files are formatted without modifying them"
         , quiet = False &= help "Do not report errors"
         , verify =
             False &=
             help
-              "Check that the output parses and formats the same as the input"
+              "Apply sanity checks on the output after formatting"
         } &=
       summary ("nixfmt v" ++ showVersion version) &=
       help "Format Nix source code"

nixfmt.cabal

@@ -88,9 +88,11 @@ library
   build-depends:
       base             >= 4.12.0 && < 4.17
     , megaparsec       >= 9.0.1 && < 9.3
+    , mtl
     , parser-combinators >= 1.0.3 && < 1.4
     , scientific       >= 0.3.0 && < 0.4.0
     , text             >= 1.2.3 && < 1.3
+    , transformers
   default-language:    Haskell2010
   ghc-options:
     -Wall

src/Nixfmt.hs

@@ -12,7 +12,7 @@ module Nixfmt
     ) where
 
 import Data.Bifunctor (bimap, first)
-import Data.Text (Text)
+import Data.Text (Text, unpack)
 import qualified Text.Megaparsec as Megaparsec (parse)
 import Text.Megaparsec.Error (errorBundlePretty)
 
@@ -40,7 +40,7 @@ formatVerify width path unformatted = do
     if formattedOnceParsed /= unformattedParsed
     then pleaseReport "Parses differently after formatting."
     else if formattedOnce /= formattedTwice
-    then pleaseReport "Nixfmt is not idempotent."
+    then flip first (pleaseReport "Nixfmt is not idempotent.") $ \x -> (x <> "\nAfter one formatting:\n" <> unpack formattedOnce <> "\nAfter two:\n" <> unpack formattedTwice)
     else Right formattedOnce
     where
         parse = first errorBundlePretty . Megaparsec.parse file path

src/Nixfmt/Lexer.hs

@@ -4,35 +4,40 @@
  - SPDX-License-Identifier: MPL-2.0
  -}
 
-{-# LANGUAGE LambdaCase, OverloadedStrings #-}
+{-# LANGUAGE BlockArguments, FlexibleContexts, LambdaCase, OverloadedStrings #-}
 
-module Nixfmt.Lexer (lexeme) where
+module Nixfmt.Lexer (lexeme, pushTrivia, takeTrivia, whole) where
 
+import Control.Monad.State (MonadState, evalStateT, get, modify, put)
 import Data.Char (isSpace)
 import Data.List (dropWhileEnd)
 import Data.Maybe (fromMaybe)
 import Data.Text as Text
-  (Text, intercalate, length, lines, null, pack, replace, replicate, strip,
-  stripEnd, stripPrefix, stripStart, takeWhile)
+  (Text, length, lines, null, pack, replace, replicate, strip, stripEnd,
+  stripPrefix, stripStart, takeWhile, unwords)
+import Data.Void (Void)
 import Text.Megaparsec
-  (SourcePos(..), anySingle, chunk, getSourcePos, hidden, many, manyTill, some,
-  try, unPos, (<|>))
+  (Parsec, SourcePos(..), Pos, anySingle, chunk, getSourcePos, hidden, many,
+  manyTill, some, try, unPos, (<|>))
 import Text.Megaparsec.Char (eol)
 
-import Nixfmt.Types (Ann(..), Parser, TrailingComment(..), Trivia, Trivium(..))
+import Nixfmt.Types
+  (Ann(..), Whole(..), Parser, TrailingComment(..), Trivia, Trivium(..))
 import Nixfmt.Util (manyP)
+-- import Debug.Trace (traceShow, traceShowId)
 
 data ParseTrivium
     = PTNewlines     Int
-    | PTLineComment  Text
+      -- Track the column where the comment starts
+    | PTLineComment  Text Pos
     | PTBlockComment [Text]
     deriving (Show)
 
 preLexeme :: Parser a -> Parser a
 preLexeme p = p <* manyP (\x -> isSpace x && x /= '\n' && x /= '\r')
 
 newlines :: Parser ParseTrivium
-newlines = PTNewlines <$> Prelude.length <$> some (preLexeme eol)
+newlines = PTNewlines . Prelude.length <$> some (preLexeme eol)
 
 splitLines :: Text -> [Text]
 splitLines = dropWhile Text.null . dropWhileEnd Text.null
@@ -50,8 +55,11 @@ fixLines n (h : t) = strip h
     : map (stripIndentation $ commonIndentationLength n $ filter (/="") t) t
 
 lineComment :: Parser ParseTrivium
-lineComment = preLexeme $ chunk "#" *>
-    (PTLineComment <$> manyP (\x -> x /= '\n' && x /= '\r'))
+lineComment = preLexeme $ do
+    SourcePos{sourceColumn = col} <- getSourcePos
+    _ <- chunk "#"
+    text <- manyP (\x -> x /= '\n' && x /= '\r')
+    return (PTLineComment text col)
 
 blockComment :: Parser ParseTrivium
 blockComment = try $ preLexeme $ do
@@ -60,40 +68,71 @@ blockComment = try $ preLexeme $ do
     chars <- manyTill anySingle $ chunk "*/"
     return $ PTBlockComment $ fixLines (unPos pos) $ splitLines $ pack chars
 
+-- This should be called with zero or one elements, as per `span isTrailing`
 convertTrailing :: [ParseTrivium] -> Maybe TrailingComment
 convertTrailing = toMaybe . join . map toText
-    where toText (PTLineComment c)    = strip c
+    where toText (PTLineComment c _)    = strip c
           toText (PTBlockComment [c]) = strip c
           toText _                    = ""
-          join = intercalate " " . filter (/="")
+          join = Text.unwords . filter (/="")
           toMaybe "" = Nothing
           toMaybe c  = Just $ TrailingComment c
 
 convertLeading :: [ParseTrivium] -> Trivia
 convertLeading = concatMap (\case
     PTNewlines 1       -> []
     PTNewlines _       -> [EmptyLine]
-    PTLineComment c    -> [LineComment c]
+    PTLineComment c _  -> [LineComment c]
     PTBlockComment []  -> []
     PTBlockComment [c] -> [LineComment $ " " <> strip c]
     PTBlockComment cs  -> [BlockComment cs])
 
 isTrailing :: ParseTrivium -> Bool
-isTrailing (PTLineComment _)    = True
+isTrailing (PTLineComment _ _)  = True
 isTrailing (PTBlockComment [])  = True
 isTrailing (PTBlockComment [_]) = True
 isTrailing _                    = False
 
-convertTrivia :: [ParseTrivium] -> (Maybe TrailingComment, Trivia)
-convertTrivia pts =
+convertTrivia :: [ParseTrivium] -> Pos -> (Maybe TrailingComment, Trivia)
+convertTrivia pts nextCol =
     let (trailing, leading) = span isTrailing pts
-    in (convertTrailing trailing, convertLeading leading)
+    in case (trailing, leading) of
+        -- Special case: if the trailing comment visually forms a block with the start of the following line,
+        -- then treat it like part of those comments instead of a distinct trailing comment.
+        -- This happens especially often after `{` or `[` tokens, where the comment of the first item
+        -- starts on the same line ase the opening token.
+        ([PTLineComment _ pos], (PTNewlines 1):(PTLineComment _ pos'):_) | pos == pos' -> (Nothing, convertLeading pts)
+        ([PTLineComment _ pos], [(PTNewlines 1)]) | pos == nextCol -> (Nothing, convertLeading pts)
+        _ -> (convertTrailing trailing, convertLeading leading)
 
 trivia :: Parser [ParseTrivium]
 trivia = many $ hidden $ lineComment <|> blockComment <|> newlines
 
+-- The following primitives to interact with the state monad that stores trivia
+-- are designed to prevent trivia from being dropped or duplicated by accident.
+
+takeTrivia :: MonadState Trivia m => m Trivia
+takeTrivia = get <* put []
+
+pushTrivia :: MonadState Trivia m => Trivia -> m ()
+pushTrivia t = modify (<>t)
+
 lexeme :: Parser a -> Parser (Ann a)
 lexeme p = do
+    lastLeading <- takeTrivia
     token <- preLexeme p
-    (trailing, leading) <- convertTrivia <$> trivia
-    return $ Ann token trailing leading
+    parsedTrivia <- trivia
+    -- This is the position of the next lexeme after the currently parsed one
+    SourcePos{sourceColumn = col} <- getSourcePos
+    let (trailing, nextLeading) = convertTrivia parsedTrivia col
+    pushTrivia nextLeading
+    return $ Ann lastLeading token trailing
+
+-- | Tokens normally have only leading trivia and one trailing comment on the same
+-- line. A whole x also parses and stores final trivia after the x. A whole also
+-- does not interact with the trivia state of its surroundings.
+whole :: Parser a -> Parsec Void Text (Whole a)
+whole pa = flip evalStateT [] do
+    preLexeme $ pure ()
+    pushTrivia . convertLeading =<< trivia
+    Whole <$> pa <*> takeTrivia