From bcc09668c6ecaf07095c027800a139638ec04f33 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Fri, 18 Oct 2024 18:32:13 +0200 Subject: [PATCH 01/10] nix: add function type, import regex library --- src/languages/nix.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/languages/nix.js b/src/languages/nix.js index cea7b7c489..2d7757a20f 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -6,7 +6,9 @@ Website: http://nixos.org/nix Category: system */ +/** @type LanguageFn */ export default function(hljs) { + const regex = hljs.regex; const KEYWORDS = { keyword: [ "rec", From c9d9d4fe24a9e23df79bebf19dad21c4fc76539e Mon Sep 17 00:00:00 2001 From: h7x4 Date: Sun, 13 Oct 2024 23:56:35 +0200 Subject: [PATCH 02/10] nix: update keywords - Add a bunch of new keywords - List all `builtins` explicitly - Sort alphabetically, to make it easier to spot missing builtins - Remove `add`, not a real builtin - Move `or` to `keywords`, this is not a `literal` --- src/languages/nix.js | 171 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 148 insertions(+), 23 deletions(-) diff --git a/src/languages/nix.js b/src/languages/nix.js index 2d7757a20f..393728a354 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -11,47 +11,171 @@ export default function(hljs) { const regex = hljs.regex; const KEYWORDS = { keyword: [ - "rec", - "with", - "let", - "in", - "inherit", "assert", - "if", "else", - "then" + "if", + "in", + "inherit", + "let", + "or", + "rec", + "then", + "with", ], literal: [ "true", "false", - "or", - "and", - "null" + "null", ], built_in: [ - "import", + // toplevel builtins "abort", "baseNameOf", + "builtins", + "derivation", + "derivationStrict", "dirOf", + "fetchGit", + "fetchMercurial", + "fetchTarball", + "fetchTree", + "fromTOML", + "import", "isNull", - "builtins", "map", + "placeholder", "removeAttrs", + "scopedImport", "throw", "toString", - "derivation" - ] - }; - const ANTIQUOTE = { - className: 'subst', - begin: /\$\{/, - end: /\}/, - keywords: KEYWORDS + ], }; - const ESCAPED_DOLLAR = { - className: 'char.escape', - begin: /''\$/, + + const BUILTINS = { + scope: 'built_in', + match: regex.either(...[ + "abort", + "add", + "addDrvOutputDependencies", + "addErrorContext", + "all", + "any", + "appendContext", + "attrNames", + "attrValues", + "baseNameOf", + "bitAnd", + "bitOr", + "bitXor", + "break", + "builtins", + "catAttrs", + "ceil", + "compareVersions", + "concatLists", + "concatMap", + "concatStringsSep", + "convertHash", + "currentSystem", + "currentTime", + "deepSeq", + "derivation", + "derivationStrict", + "dirOf", + "div", + "elem", + "elemAt", + "false", + "fetchGit", + "fetchMercurial", + "fetchTarball", + "fetchTree", + "fetchurl", + "filter", + "filterSource", + "findFile", + "flakeRefToString", + "floor", + "foldl'", + "fromJSON", + "fromTOML", + "functionArgs", + "genList", + "genericClosure", + "getAttr", + "getContext", + "getEnv", + "getFlake", + "groupBy", + "hasAttr", + "hasContext", + "hashFile", + "hashString", + "head", + "import", + "intersectAttrs", + "isAttrs", + "isBool", + "isFloat", + "isFunction", + "isInt", + "isList", + "isNull", + "isPath", + "isString", + "langVersion", + "length", + "lessThan", + "listToAttrs", + "map", + "mapAttrs", + "match", + "mul", + "nixPath", + "nixVersion", + "null", + "parseDrvName", + "parseFlakeRef", + "partition", + "path", + "pathExists", + "placeholder", + "readDir", + "readFile", + "readFileType", + "removeAttrs", + "replaceStrings", + "scopedImport", + "seq", + "sort", + "split", + "splitVersion", + "storeDir", + "storePath", + "stringLength", + "sub", + "substring", + "tail", + "throw", + "toFile", + "toJSON", + "toPath", + "toString", + "toXML", + "trace", + "traceVerbose", + "true", + "tryEval", + "typeOf", + "unsafeDiscardOutputDependency", + "unsafeDiscardStringContext", + "unsafeGetAttrPos", + "warn", + "zipAttrsWith", + ].map(b => `builtins\\.${b}`)), + relevance: 10, }; + const ATTRS = { begin: /[a-zA-Z0-9-_]+(\s*=)/, returnBegin: true, @@ -82,6 +206,7 @@ export default function(hljs) { hljs.NUMBER_MODE, hljs.HASH_COMMENT_MODE, hljs.C_BLOCK_COMMENT_MODE, + BUILTINS, STRING, ATTRS ]; From e22f735afbcb062f75e9778a51ac5f7e6876c230 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 00:00:03 +0200 Subject: [PATCH 03/10] nix: fix string handling - The different string variants have different modes of escapes. Split `STRING.contains` into their different variants to reflect this. - Add escape logic for `'''` - Add escape logic for backslash escaped characters. --- src/languages/nix.js | 50 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/languages/nix.js b/src/languages/nix.js index 393728a354..a7ae0b647a 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -185,23 +185,57 @@ export default function(hljs) { className: 'attr', begin: /\S+/, relevance: 0.2 - } - ] + }, + ], + }; + + const NORMAL_ESCAPED_DOLLAR = { + scope: 'char.escape', + match: /\\\$/, + }; + const INDENTED_ESCAPED_DOLLAR = { + scope: 'char.escape', + match: /''\$/, + }; + const ANTIQUOTE = { + scope: 'subst', + begin: /\$\{/, + end: /\}/, + keywords: KEYWORDS, + }; + const ESCAPED_DOUBLEQUOTE = { + scope: 'char.escape', + match: /'''/, + }; + const ESCAPED_LITERAL = { + scope: 'char.escape', + match: /\\(?!\$)./, }; const STRING = { - className: 'string', - contains: [ ESCAPED_DOLLAR, ANTIQUOTE ], + scope: 'string', variants: [ { begin: "''", - end: "''" + end: "''", + contains: [ + INDENTED_ESCAPED_DOLLAR, + ANTIQUOTE, + ESCAPED_DOUBLEQUOTE, + ESCAPED_LITERAL, + ], }, { begin: '"', - end: '"' - } - ] + end: '"', + contains: [ + NORMAL_ESCAPED_DOLLAR, + ANTIQUOTE, + ESCAPED_LITERAL, + ], + }, + ], }; + const EXPRESSIONS = [ hljs.NUMBER_MODE, hljs.HASH_COMMENT_MODE, From 1bb94a0693c8cb90e1305e5988c985b67e0d3af7 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 00:04:53 +0200 Subject: [PATCH 04/10] nix: handle path, lookup paths, and operators --- src/languages/nix.js | 92 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/src/languages/nix.js b/src/languages/nix.js index a7ae0b647a..a4ac3f2c51 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -176,8 +176,90 @@ export default function(hljs) { relevance: 10, }; + const IDENTIFIER_REGEX = '[A-Za-z_][A-Za-z0-9_\'-]*'; + + const LOOKUP_PATH = { + scope: 'symbol', + match: new RegExp(`<${IDENTIFIER_REGEX}(/${IDENTIFIER_REGEX})*>`), + }; + + const PATH_PIECE = "[A-Za-z0-9_\\+\\.-]+" + const PATH = { + scope: 'symbol', + match: new RegExp(`(\\.\\.|\\.|~)?/(${PATH_PIECE})?(/${PATH_PIECE})*(?=[\\s;])`), + }; + + const OPERATOR_WITHOUT_MINUS_REGEX = regex.either(...[ + '==', + '=', + '\\+\\+', + '\\+', + '<=', + '<\\|', + '<', + '>=', + '>', + '->', + '//', + '/', + '!=', + '!', + '\\|\\|', + '\\|>', + '\\?', + '\\*', + '&&', + ]); + + const OPERATOR = { + scope: 'operator', + match: regex.concat(OPERATOR_WITHOUT_MINUS_REGEX, /(?!-)/), + relevance: 0, + }; + + // '-' is being handled by itself to ensure we are able to tell the difference + // between a dash in an identifier and a minus operator + const NUMBER = { + scope: 'number', + match: new RegExp(`${hljs.NUMBER_RE}(?!-)`), + relevance: 0, + }; + const MINUS_OPERATOR = { + variants: [ + { + scope: 'operator', + beforeMatch: /\s/, + // The (?!>) is used to ensure this doesn't collide with the '->' operator + begin: /-(?!>)/, + }, + { + begin: [ + new RegExp(`${hljs.NUMBER_RE}`), + /-/, + /(?!>)/, + ], + beginScope: { + 1: 'number', + 2: 'operator' + }, + }, + { + begin: [ + OPERATOR_WITHOUT_MINUS_REGEX, + /-/, + /(?!>)/, + ], + beginScope: { + 1: 'operator', + 2: 'operator' + }, + }, + ], + relevance: 0, + }; + const ATTRS = { - begin: /[a-zA-Z0-9-_]+(\s*=)/, + begin: new RegExp(`${IDENTIFIER_REGEX}(\\s*=)`), returnBegin: true, relevance: 0, contains: [ @@ -237,12 +319,16 @@ export default function(hljs) { }; const EXPRESSIONS = [ - hljs.NUMBER_MODE, + NUMBER, hljs.HASH_COMMENT_MODE, hljs.C_BLOCK_COMMENT_MODE, BUILTINS, STRING, - ATTRS + LOOKUP_PATH, + PATH, + ATTRS, + MINUS_OPERATOR, + OPERATOR, ]; ANTIQUOTE.contains = EXPRESSIONS; return { From 2d1fbd4573ace2e388e1ef3323d2324f0d5a0e17 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 00:05:12 +0200 Subject: [PATCH 05/10] nix: handle markdown comments Also ensure comments are handled before any other expressions --- src/languages/nix.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/languages/nix.js b/src/languages/nix.js index a4ac3f2c51..f77a2c5ca5 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -322,6 +322,14 @@ export default function(hljs) { NUMBER, hljs.HASH_COMMENT_MODE, hljs.C_BLOCK_COMMENT_MODE, + hljs.COMMENT( + /\/\*\*(?!\/)/, + /\*\//, + { + subLanguage: 'markdown', + relevance: 0 + } + ), BUILTINS, STRING, LOOKUP_PATH, From 45b559a8c5aa11386a22bdc41ebcdfa0a8f73234 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 00:06:35 +0200 Subject: [PATCH 06/10] nix: handle REPL keywords --- src/languages/nix.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/languages/nix.js b/src/languages/nix.js index f77a2c5ca5..f54a5f09a6 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -338,11 +338,26 @@ export default function(hljs) { MINUS_OPERATOR, OPERATOR, ]; + ANTIQUOTE.contains = EXPRESSIONS; + + const REPL = [ + { + scope: 'meta.prompt', + match: /^nix-repl>(?=\s)/, + relevance: 10, + }, + { + scope: 'meta', + beforeMatch: /\s+/, + begin: /:([a-z]+|\?)/, + }, + ]; + return { name: 'Nix', aliases: [ "nixos" ], keywords: KEYWORDS, - contains: EXPRESSIONS + contains: EXPRESSIONS.concat(REPL), }; } From aad34f7de1079702d6494ffc0b1b439b76513cde Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 21:08:25 +0200 Subject: [PATCH 07/10] nix: handle basic function params --- src/languages/nix.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/languages/nix.js b/src/languages/nix.js index f54a5f09a6..ab453ef5f1 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -318,6 +318,11 @@ export default function(hljs) { ], }; + const FUNCTION_PARAMS = { + scope: 'params', + match: new RegExp(`${IDENTIFIER_REGEX}\\s*:(?=\\s)`), + }; + const EXPRESSIONS = [ NUMBER, hljs.HASH_COMMENT_MODE, @@ -334,6 +339,7 @@ export default function(hljs) { STRING, LOOKUP_PATH, PATH, + FUNCTION_PARAMS, ATTRS, MINUS_OPERATOR, OPERATOR, From 313d27ebbdd2387f217da0f7dde8789f181f020e Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 22:03:50 +0200 Subject: [PATCH 08/10] nix: better parsing for attrsets --- src/languages/nix.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/languages/nix.js b/src/languages/nix.js index ab453ef5f1..2b63cd99d4 100644 --- a/src/languages/nix.js +++ b/src/languages/nix.js @@ -259,15 +259,16 @@ export default function(hljs) { }; const ATTRS = { - begin: new RegExp(`${IDENTIFIER_REGEX}(\\s*=)`), + beforeMatch: /(^|\{|;)\s*/, + begin: new RegExp(`${IDENTIFIER_REGEX}(\\.${IDENTIFIER_REGEX})*\\s*=(?!=)`), returnBegin: true, relevance: 0, contains: [ { - className: 'attr', - begin: /\S+/, - relevance: 0.2 - }, + scope: 'attr', + match: new RegExp(`${IDENTIFIER_REGEX}(\\.${IDENTIFIER_REGEX})*(?=\\s*=)`), + relevance: 0.2, + } ], }; From 9f548dac0e3c0bea20f311bd20cfcb829939f53a Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 00:33:19 +0200 Subject: [PATCH 09/10] nix: update tests --- test/markup/nix/default.expect.txt | 104 +++++++++++++++++++++++++---- test/markup/nix/default.txt | 98 ++++++++++++++++++++++++--- 2 files changed, 181 insertions(+), 21 deletions(-) diff --git a/test/markup/nix/default.expect.txt b/test/markup/nix/default.expect.txt index bd90c5c422..57759a7a78 100644 --- a/test/markup/nix/default.expect.txt +++ b/test/markup/nix/default.expect.txt @@ -1,25 +1,105 @@ -{ stdenv, foo, bar ? false, ... }: +{ + stdenv, + pkgs ? import <nixpkgs> {}, + home-manager ? import <a/b/c/home-manager>, + foo ? "hello", + bar ? false, + ... +}: /* * foo */ let - a = 1; # just a comment - b = null; - c = toString 10; -in stdenv.mkDerivation rec { - name = "foo-${version}"; - version = "1.3"; + a = 1; # just a comment + b = null; + c = toString 10; - configureFlags = [ "--with-foo2" ] ++ stdenv.lib.optional bar "--with-foo=${ with stdenv.lib; foo }" + inherit (builtins) concatLists; + inherit (lib) genAttrs; +in - postInstall = '' +{ + number = 1; + floating_point = 2.3; + + dash_usage = { + negative = -1; + negative_space = - 1; + + arithmetic = 1-2; + arithmetic2 = -1 - 2; + arithmetic3 = - 1 -2-3; + arithmetic4 =-1+-2 + - 3; + + attrname = a-b; + }; + + normal_string = "asdf"; + interpolated_string = "hello ${toString 1 + (2 * 3)} world"; + escaped_interpolation_string = "\${escaped} and ''${"not" + "escaped"}"; + indentedString = '' + hello ${ if true then "--${test}" else false } - ''${ escaped } + \${"not" + "escaped"} and ''${escaped} + '''escaped single ticks''' + world + ''; + + attrsWithoutSpace="hello ${world}"; + + concatenatedList = [ "--with-foo2" ] ++ stdenv.lib.optional bar "--with-foo=${ with stdenv.lib; foo }"; + + paths = with lib; { + home = ~/x/y/z; + here = ./x/y/z; + up = ../x/y/z; + root = /x/y/z; + }; + + long.nested.attr.path = '' + ${ + # comment inside antiquote + /* comment before */ toString 1 /* comment after */ + } ''; - meta = with stdenv.lib; { - homepage = https://nixos.org; + someBuiltins = [ + (removeAttrs { x = 1; ${"_" + y.z or "none"} = 2; } [ "x" ]) + (builtins.concatLists [ [1] [2] ]) + (builtins.nonExistent "hello") + ]; + + someOperators = [ + (a |> b <| c) + (x || y && z -> w) + (x < y && y > x) + ]; + + function = a: b: a // b; + unformattedFunction = a : b : a == b; + + interpolatedAttrs = { + ${null} = 1; + x.${y}.z = 2; + }; + + invalidAttrs = { + 1invalidAttr = 1; + notAPath = //; + trailingSlashPath = /asdf/; + notAFunction = x:x; }; } + +# REPL tests + +nix-repl> 1 + 2 + +3 +nix-repl> :b pkgs.writeText "file.txt" "content" + +This derivation produced the following outputs: + out -> /nix/store/v5a715bk02cgvb0fv9kby0nsyy1prpy2-file.txt +[2 built] diff --git a/test/markup/nix/default.txt b/test/markup/nix/default.txt index 21b2080ae9..6bcaeae412 100644 --- a/test/markup/nix/default.txt +++ b/test/markup/nix/default.txt @@ -1,4 +1,11 @@ -{ stdenv, foo, bar ? false, ... }: +{ + stdenv, + pkgs ? import {}, + home-manager ? import , + foo ? "hello", + bar ? false, + ... +}: /* * foo @@ -8,18 +15,91 @@ let a = 1; # just a comment b = null; c = toString 10; -in stdenv.mkDerivation rec { - name = "foo-${version}"; - version = "1.3"; - configureFlags = [ "--with-foo2" ] ++ stdenv.lib.optional bar "--with-foo=${ with stdenv.lib; foo }" + inherit (builtins) concatLists; + inherit (lib) genAttrs; +in - postInstall = '' +{ + number = 1; + floating_point = 2.3; + + dash_usage = { + negative = -1; + negative_space = - 1; + + arithmetic = 1-2; + arithmetic2 = -1 - 2; + arithmetic3 = - 1 -2-3; + arithmetic4 =-1+-2 + - 3; + + attrname = a-b; + }; + + normal_string = "asdf"; + interpolated_string = "hello ${toString 1 + (2 * 3)} world"; + escaped_interpolation_string = "\${escaped} and ''${"not" + "escaped"}"; + indentedString = '' + hello ${ if true then "--${test}" else false } - ''${ escaped } + \${"not" + "escaped"} and ''${escaped} + '''escaped single ticks''' + world + ''; + + attrsWithoutSpace="hello ${world}"; + + concatenatedList = [ "--with-foo2" ] ++ stdenv.lib.optional bar "--with-foo=${ with stdenv.lib; foo }"; + + paths = with lib; { + home = ~/x/y/z; + here = ./x/y/z; + up = ../x/y/z; + root = /x/y/z; + }; + + long.nested.attr.path = '' + ${ + # comment inside antiquote + /* comment before */ toString 1 /* comment after */ + } ''; - meta = with stdenv.lib; { - homepage = https://nixos.org; + someBuiltins = [ + (removeAttrs { x = 1; ${"_" + y.z or "none"} = 2; } [ "x" ]) + (builtins.concatLists [ [1] [2] ]) + (builtins.nonExistent "hello") + ]; + + someOperators = [ + (a |> b <| c) + (x || y && z -> w) + (x < y && y > x) + ]; + + function = a: b: a // b; + unformattedFunction = a : b : a == b; + + interpolatedAttrs = { + ${null} = 1; + x.${y}.z = 2; + }; + + invalidAttrs = { + 1invalidAttr = 1; + notAPath = //; + trailingSlashPath = /asdf/; + notAFunction = x:x; }; } + +# REPL tests + +nix-repl> 1 + 2 + +3 +nix-repl> :b pkgs.writeText "file.txt" "content" + +This derivation produced the following outputs: + out -> /nix/store/v5a715bk02cgvb0fv9kby0nsyy1prpy2-file.txt +[2 built] From d3876a2d78d68ad790e4f9d434b61f7aad4b7da2 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Mon, 14 Oct 2024 00:42:30 +0200 Subject: [PATCH 10/10] CHANGES.md: update with changes to nix --- CHANGES.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8fe38020cb..9dcc789656 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,14 @@ Core Grammars: - enh(csharp) add Contextual keywords `file`, `args`, `dynamic`, `record`, `required` and `scoped` [Alvin Joy][] - enh(lua) add 'pluto' as an alias [Sainan] - enh(bash) add reserved keywords `time` and `coproc` [Álvaro Mondéjar][] +- enh(nix) update keywords [h7x4][] +- enh(nix) support paths [h7x4][] +- enh(nix) support lookup paths [h7x4][] +- enh(nix) support operators [h7x4][] +- enh(nix) support REPL keywords [h7x4][] +- enh(nix) support markdown comments [h7x4][] +- enh(nix) support basic function params [h7x4][] +- enh(nix) better parsing of attrsets [h7x4][] - fix(c) - Fixed hex numbers with decimals [Dxuian] - fix(typescript) - Fixedoptional property not highlighted correctly [Dxuian] - fix(ruby) - fix `|=` operator false positives (as block arguments) [Aboobacker MK] @@ -24,7 +32,12 @@ Core Grammars: - fix(cpp) added flat_set and flat_map as a part of cpp 23 version [Lavan] - fix(yaml) - Fixed special chars in yaml [Dxuian] - fix(basic) - Fixed closing quotation marks not required for a PRINT statement [Somya] - +- fix(nix) remove `add` builtin [h7x4][] +- fix(nix) mark `or` as builtin instead of literal [h7x4][] +- fix(nix) handle `'''` string escapes [h7x4][] +- fix(nix) handle backslash string escapes [h7x4][] +- fix(nix) don't mix escapes for `"` and `''` strings [h7x4][] + New Grammars: - added 3rd party TTCN-3 grammar to SUPPORTED_LANGUAGES [Osmocom][]