From 4d8f39f4457b2fbe2c9bbeb9609d24005d68f440 Mon Sep 17 00:00:00 2001 From: Martin Hradil Date: Fri, 29 Oct 2021 19:36:32 +0000 Subject: [PATCH 1/5] lint:po - lint locale/*.po files to ensure same vars and components ({foo}, {0}, <0>, <0>, ) --- .github/workflows/pr-checks.yml | 4 ++ locale/lint.pl | 100 ++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 105 insertions(+) create mode 100755 locale/lint.pl diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 90c6c6b2b3..c574db4a11 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -18,6 +18,10 @@ jobs: node-version: '14' cache: 'npm' + - name: "Install lint:po dependencies" + run: | + sudo apt install -y libstring-escape-perl libarray-utils-perl perl + - name: "Checks" run: | # fail if npm install had to change package-lock.json diff --git a/locale/lint.pl b/locale/lint.pl new file mode 100755 index 0000000000..dbdeb0c571 --- /dev/null +++ b/locale/lint.pl @@ -0,0 +1,100 @@ +#!/usr/bin/env perl +use v5.30; +use String::Escape qw( unqqbackslash ); # apt install libstring-escape-perl +use Array::Utils qw( array_diff array_minus ); # apt install libarray-utils-perl + +sub process { + my ($msgid, $msgstr) = @_; + return 1 if not defined $msgid or not defined $msgstr; # handling eof while still in state 2 + return 1 if $msgid eq ""; # magic + return 1 if $msgstr eq ""; # untranslated + + my @msgidvars = (); + while ($msgid =~ /\{(\w+|\d+)\}|<\/?\d+\/?>/g) { + push @msgidvars, $&; + } + my @msgstrvars = (); + while ($msgstr =~ /\{(\w+|\d+)\}|<\/?\d+\/?>/g) { + push @msgstrvars, $&; + } + + my @diff = array_diff(@msgidvars, @msgstrvars); + if (@diff) { + my @missing = array_minus(@msgidvars, @msgstrvars); + my @extra = array_minus(@msgstrvars, @msgidvars); + my $message = ""; + $message .= " Missing from msgstr: @missing\n" if @missing; + $message .= " Unexpected in msgstr: @extra\n" if @extra; + $message .= " "; + + warn "Difference between msgid=\"$msgid\" and msgstr=\"$msgstr\":\n$message"; + return 0; + } + + return 1; +} + +my $state = 0; +my $msgid; +my $msgstr; +my $exit = 0; + +while (<>) { + next if /^#/; + + chomp; + s/^\s*//; + + # 0 = expecting `msgid` + if ($state == 0) { + next if /^$/; + + if (/^msgid\s+/) { + $msgid = unqqbackslash($'); + $state = 1; + next; + } + + warn "($state) Unexpected input: $_"; + $exit |= 1; + } + + # 1 = expecting `msgstr`, or more bits of previous msgid + if ($state == 1) { + if (/^msgstr\s+/) { + $msgstr = unqqbackslash($'); + $state = 2; + next; + } + + if (/^"/) { + $msgid .= unqqbackslash($_); + next; + } + + warn "($state) Unexpected input: $_"; + $exit |= 1; + } + + # 2 = expecting newline, or more bits of previous msgstr + if ($state == 2) { + if (/^$/) { + $exit |= 2 unless process($msgid, $msgstr); + $state = 0; + $msgid = undef; + $msgstr = undef; + next; + } + + if (/^"/) { + $msgstr .= unqqbackslash($_); + next; + } + + warn "($state) Unexpected input: $_"; + $exit |= 1; + } +} + +$exit |= 2 unless process($msgid, $msgstr); +exit $exit; diff --git a/package.json b/package.json index 37f39ef358..6d7e6c5455 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "lint:ls": "ls-lint", "lint:sass": "stylelint 'src/**/*.scss' --config .stylelintrc.json", "lint:ts": "tsc", + "lint:po": "for f in locale/*po; do locale/lint.pl \"$f\" 2>&1 | sed 's/<>/'\"$(basename \"$f\")\"'/' ; done", "prettier": "prettier --write 'src/**' 'config/**' 'test/**'", "prettier:check": "prettier -l 'src/**' 'config/**' 'test/**'", "prod": "NODE_ENV=production webpack serve --config custom.dev.config", From 74f8528cfae74301e914dbd4dd579e184b1d167f Mon Sep 17 00:00:00 2001 From: Martin Hradil Date: Fri, 29 Oct 2021 20:23:39 +0000 Subject: [PATCH 2/5] lint:po - convert to ruby * support filenames * support python %(foo)s format strings * add github action error formatter * use line number of the first line of msgstr --- .github/workflows/pr-checks.yml | 4 -- locale/lint.pl | 100 ---------------------------- locale/lint.rb | 114 ++++++++++++++++++++++++++++++++ package.json | 2 +- 4 files changed, 115 insertions(+), 105 deletions(-) delete mode 100755 locale/lint.pl create mode 100755 locale/lint.rb diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index c574db4a11..90c6c6b2b3 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -18,10 +18,6 @@ jobs: node-version: '14' cache: 'npm' - - name: "Install lint:po dependencies" - run: | - sudo apt install -y libstring-escape-perl libarray-utils-perl perl - - name: "Checks" run: | # fail if npm install had to change package-lock.json diff --git a/locale/lint.pl b/locale/lint.pl deleted file mode 100755 index dbdeb0c571..0000000000 --- a/locale/lint.pl +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env perl -use v5.30; -use String::Escape qw( unqqbackslash ); # apt install libstring-escape-perl -use Array::Utils qw( array_diff array_minus ); # apt install libarray-utils-perl - -sub process { - my ($msgid, $msgstr) = @_; - return 1 if not defined $msgid or not defined $msgstr; # handling eof while still in state 2 - return 1 if $msgid eq ""; # magic - return 1 if $msgstr eq ""; # untranslated - - my @msgidvars = (); - while ($msgid =~ /\{(\w+|\d+)\}|<\/?\d+\/?>/g) { - push @msgidvars, $&; - } - my @msgstrvars = (); - while ($msgstr =~ /\{(\w+|\d+)\}|<\/?\d+\/?>/g) { - push @msgstrvars, $&; - } - - my @diff = array_diff(@msgidvars, @msgstrvars); - if (@diff) { - my @missing = array_minus(@msgidvars, @msgstrvars); - my @extra = array_minus(@msgstrvars, @msgidvars); - my $message = ""; - $message .= " Missing from msgstr: @missing\n" if @missing; - $message .= " Unexpected in msgstr: @extra\n" if @extra; - $message .= " "; - - warn "Difference between msgid=\"$msgid\" and msgstr=\"$msgstr\":\n$message"; - return 0; - } - - return 1; -} - -my $state = 0; -my $msgid; -my $msgstr; -my $exit = 0; - -while (<>) { - next if /^#/; - - chomp; - s/^\s*//; - - # 0 = expecting `msgid` - if ($state == 0) { - next if /^$/; - - if (/^msgid\s+/) { - $msgid = unqqbackslash($'); - $state = 1; - next; - } - - warn "($state) Unexpected input: $_"; - $exit |= 1; - } - - # 1 = expecting `msgstr`, or more bits of previous msgid - if ($state == 1) { - if (/^msgstr\s+/) { - $msgstr = unqqbackslash($'); - $state = 2; - next; - } - - if (/^"/) { - $msgid .= unqqbackslash($_); - next; - } - - warn "($state) Unexpected input: $_"; - $exit |= 1; - } - - # 2 = expecting newline, or more bits of previous msgstr - if ($state == 2) { - if (/^$/) { - $exit |= 2 unless process($msgid, $msgstr); - $state = 0; - $msgid = undef; - $msgstr = undef; - next; - } - - if (/^"/) { - $msgstr .= unqqbackslash($_); - next; - } - - warn "($state) Unexpected input: $_"; - $exit |= 1; - } -} - -$exit |= 2 unless process($msgid, $msgstr); -exit $exit; diff --git a/locale/lint.rb b/locale/lint.rb new file mode 100755 index 0000000000..ef4a48a2d0 --- /dev/null +++ b/locale/lint.rb @@ -0,0 +1,114 @@ +#!/usr/bin/env ruby +require 'json' + +if ARGV.empty? + $stderr.puts "#{$0}: no files" + $stderr.puts "use: #{$0} " + exit 1 +end + +# find any {num} {str} %(str)s +REGEX = /(\{(\w+|\d+)\})|(<\/?\d+\/?>)|(%(\(\w+\))?.)/ + +# s.undump if only it worked with utf8 strings +def unqqbackslash(s) + JSON.parse("#{s}") +end + +def process_pair(msgid, msgstr, file, line) + # handling eof while still in state 2; magic id, untranslated strings + return true if msgid.nil? or msgstr.nil? + return true if msgid.empty? + return true if msgstr.empty? + + msgidvars = msgid.scan(REGEX).map { |m| m.compact.first } + msgstrvars = msgstr.scan(REGEX).map { |m| m.compact.first } + + missing = msgidvars - msgstrvars + extra = msgstrvars - msgidvars + if missing.any? or extra.any? + message = "" + if missing.any? + msg = "Missing from msgstr: #{missing.join(' ')}" + $stderr.puts "::error file=#{file},line=#{line}::#{msg.gsub(/[\r\n]/, '').gsub('%', '%25')}" if ENV['GITHUB_ACTIONS'] + message += " #{msg}\n" + end + if extra.any? + msg = "Unexpected in msgstr: #{extra.join(' ')}" + $stderr.puts "::error file=#{file},line=#{line}::#{msg.gsub(/[\r\n]/, '').gsub('%', '%25')}" if ENV['GITHUB_ACTIONS'] + message += " #{msg}\n" + end + message += " at #{file}:#{line}" + + $stderr.puts "Difference between msgid=\"#{msgid}\" and msgstr=\"#{msgstr}\":\n#{message}\n\n" + return false + end + + return true +end + +errors = false + +ARGV.each do |filename| + state = 0 + msgid = nil + msgstr = nil + msgstrlineno = 0 + + IO.readlines(filename).each_with_index do |line, lineno| + next if line =~ /^#/ + line = line.chomp.sub(/^\s*/, '') + + case state + when 0 # expecting `msgid` + next if line =~ /^$/; + + if line =~ /^msgid\s+/ + msgid = unqqbackslash($') + state = 1 + next + end + + warn "(#{state}) Unexpected input: #{line}" + errors = true + + when 1 # expecting `msgstr`, or more bits of previous msgid + if line =~ /^msgstr\s+/ + msgstr = unqqbackslash($') + msgstrlineno = lineno + 1 + state = 2 + next + end + + if line =~ /^"/ + msgid += unqqbackslash(line) + next + end + + warn "(#{state}) Unexpected input: #{line}" + errors = true + + when 2 # expecting newline, or more bits of previous msgstr + if line =~ /^$/ + errors = true unless process_pair(msgid, msgstr, filename, msgstrlineno) + state = 0 + msgid = nil + msgstr = nil + msgstrlineno = 0 + next + end + + if line =~ /^"/ + msgstr += unqqbackslash(line) + next + end + + warn "(#{state}) Unexpected input: #{line}" + errors = true + end + end + + errors = true unless process_pair(msgid, msgstr, filename, msgstrlineno) +end + +exit(errors ? 1 : 0) diff --git a/package.json b/package.json index 6d7e6c5455..44b32f08e5 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "lint:ls": "ls-lint", "lint:sass": "stylelint 'src/**/*.scss' --config .stylelintrc.json", "lint:ts": "tsc", - "lint:po": "for f in locale/*po; do locale/lint.pl \"$f\" 2>&1 | sed 's/<>/'\"$(basename \"$f\")\"'/' ; done", + "lint:po": "locale/lint.rb locale/*.po", "prettier": "prettier --write 'src/**' 'config/**' 'test/**'", "prettier:check": "prettier -l 'src/**' 'config/**' 'test/**'", "prod": "NODE_ENV=production webpack serve --config custom.dev.config", From 613263409ee742333dd2d0140e2a0fe51d55846a Mon Sep 17 00:00:00 2001 From: Martin Hradil Date: Thu, 18 Nov 2021 01:34:23 +0000 Subject: [PATCH 3/5] lint:po - convert to python MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and python \w+ also matches `opération` --- locale/lint.py | 117 +++++++++++++++++++++++++++++++++++++++++++++++++ locale/lint.rb | 114 ----------------------------------------------- package.json | 2 +- 3 files changed, 118 insertions(+), 115 deletions(-) create mode 100755 locale/lint.py delete mode 100755 locale/lint.rb diff --git a/locale/lint.py b/locale/lint.py new file mode 100755 index 0000000000..c6a9fa39be --- /dev/null +++ b/locale/lint.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +import json +import re +import os +import sys + +if len(sys.argv) <= 1: + print(f"{sys.argv[0]}: no files", file=sys.stderr) + print(f"use: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + +# find any {num} {str} %(str)s +REGEX = r'(\{(\w+|\d+)\})|(<\/?\d+\/?>)|(%(\(\w+\))?.)' + +# s.encode('utf-8').decode('unicode-escape') if only it worked with utf8 strings +def unqqbackslash(s): + return json.loads(s) + +def process_pair(msgid, msgstr, file, line): + # handling eof while still in state 2; magic id, untranslated strings + if msgid == None or msgstr == None or len(msgid) == 0 or len(msgstr) == 0: + return True + + # findall(...).map((arr) => arr.compact.first) + msgidvars = [ [truthy for truthy in match if truthy][0] for match in re.findall(REGEX, msgid) ] + msgstrvars = [ [truthy for truthy in match if truthy][0] for match in re.findall(REGEX, msgstr) ] + + missing = list( set(msgidvars) - set(msgstrvars) ) + extra = list( set(msgstrvars) - set(msgidvars) ) + + if len(missing) or len(extra): + message = "" + if len(missing): + msg = f"Missing from msgstr: {' '.join(missing)}" + if os.environ.get('GITHUB_ACTIONS'): + s = msg.replace("\r", "").replace("\n", "").replace('%', '%25') + print(f"::error file={file},line={line}::{s}", file=sys.stderr) + message += f" {msg}\n" + if len(extra): + msg = f"Unexpected in msgstr: {' '.join(extra)}" + if os.environ.get('GITHUB_ACTIONS'): + s = msg.replace("\r", "").replace("\n", "").replace('%', '%25') + print(f"::error file={file},line={line}::{s}", file=sys.stderr) + message += f" {msg}\n" + message += f" at {file}:{line}" + + print(f"Difference between msgid=\"{msgid}\" and msgstr=\"{msgstr}\":\n{message}\n", file=sys.stderr) + return False + + return True + +errors = False + +for filename in sys.argv[1:]: + state = 0 + msgid = None + msgstr = None + msgstrlineno = 0 + lines = None + + with open(filename) as f: + lines = f.read().splitlines() + + for lineno, line in enumerate(lines): + if re.match(r'^#', line): + continue + + line = line.strip() + + if state == 0: # expecting `msgid` + if re.match(r'^$', line): + continue + + if m := re.match(r'^msgid\s+(.*)$', line): + msgid = unqqbackslash(m[1]) + state = 1 + continue + + warnings.warn(f"({state}) Unexpected input: {line}") + errors = True + + elif state == 1: # expecting `msgstr`, or more bits of previous msgid + if m := re.match(r'^msgstr\s+(.*)$', line): + msgstr = unqqbackslash(m[1]) + msgstrlineno = lineno + 1 + state = 2 + continue + + if re.match(r'^"', line): + msgid += unqqbackslash(line) + continue + + warnings.warn(f"({state}) Unexpected input: {line}") + errors = True + + elif state == 2: # expecting newline, or more bits of previous msgstr + if re.match(r'^$', line): + if not process_pair(msgid, msgstr, filename, msgstrlineno): + errors = True + + state = 0 + msgid = None + msgstr = None + msgstrlineno = 0 + continue + + if re.match(r'^"', line): + msgstr += unqqbackslash(line) + continue + + warnings.warn(f"({state}) Unexpected input: {line}") + errors = True + + if not process_pair(msgid, msgstr, filename, msgstrlineno): + errors = True + +sys.exit(1 if errors else 0) diff --git a/locale/lint.rb b/locale/lint.rb deleted file mode 100755 index ef4a48a2d0..0000000000 --- a/locale/lint.rb +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env ruby -require 'json' - -if ARGV.empty? - $stderr.puts "#{$0}: no files" - $stderr.puts "use: #{$0} " - exit 1 -end - -# find any {num} {str} %(str)s -REGEX = /(\{(\w+|\d+)\})|(<\/?\d+\/?>)|(%(\(\w+\))?.)/ - -# s.undump if only it worked with utf8 strings -def unqqbackslash(s) - JSON.parse("#{s}") -end - -def process_pair(msgid, msgstr, file, line) - # handling eof while still in state 2; magic id, untranslated strings - return true if msgid.nil? or msgstr.nil? - return true if msgid.empty? - return true if msgstr.empty? - - msgidvars = msgid.scan(REGEX).map { |m| m.compact.first } - msgstrvars = msgstr.scan(REGEX).map { |m| m.compact.first } - - missing = msgidvars - msgstrvars - extra = msgstrvars - msgidvars - if missing.any? or extra.any? - message = "" - if missing.any? - msg = "Missing from msgstr: #{missing.join(' ')}" - $stderr.puts "::error file=#{file},line=#{line}::#{msg.gsub(/[\r\n]/, '').gsub('%', '%25')}" if ENV['GITHUB_ACTIONS'] - message += " #{msg}\n" - end - if extra.any? - msg = "Unexpected in msgstr: #{extra.join(' ')}" - $stderr.puts "::error file=#{file},line=#{line}::#{msg.gsub(/[\r\n]/, '').gsub('%', '%25')}" if ENV['GITHUB_ACTIONS'] - message += " #{msg}\n" - end - message += " at #{file}:#{line}" - - $stderr.puts "Difference between msgid=\"#{msgid}\" and msgstr=\"#{msgstr}\":\n#{message}\n\n" - return false - end - - return true -end - -errors = false - -ARGV.each do |filename| - state = 0 - msgid = nil - msgstr = nil - msgstrlineno = 0 - - IO.readlines(filename).each_with_index do |line, lineno| - next if line =~ /^#/ - line = line.chomp.sub(/^\s*/, '') - - case state - when 0 # expecting `msgid` - next if line =~ /^$/; - - if line =~ /^msgid\s+/ - msgid = unqqbackslash($') - state = 1 - next - end - - warn "(#{state}) Unexpected input: #{line}" - errors = true - - when 1 # expecting `msgstr`, or more bits of previous msgid - if line =~ /^msgstr\s+/ - msgstr = unqqbackslash($') - msgstrlineno = lineno + 1 - state = 2 - next - end - - if line =~ /^"/ - msgid += unqqbackslash(line) - next - end - - warn "(#{state}) Unexpected input: #{line}" - errors = true - - when 2 # expecting newline, or more bits of previous msgstr - if line =~ /^$/ - errors = true unless process_pair(msgid, msgstr, filename, msgstrlineno) - state = 0 - msgid = nil - msgstr = nil - msgstrlineno = 0 - next - end - - if line =~ /^"/ - msgstr += unqqbackslash(line) - next - end - - warn "(#{state}) Unexpected input: #{line}" - errors = true - end - end - - errors = true unless process_pair(msgid, msgstr, filename, msgstrlineno) -end - -exit(errors ? 1 : 0) diff --git a/package.json b/package.json index 44b32f08e5..b7d43df56d 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "lint:ls": "ls-lint", "lint:sass": "stylelint 'src/**/*.scss' --config .stylelintrc.json", "lint:ts": "tsc", - "lint:po": "locale/lint.rb locale/*.po", + "lint:po": "locale/lint.py locale/*.po", "prettier": "prettier --write 'src/**' 'config/**' 'test/**'", "prettier:check": "prettier -l 'src/**' 'config/**' 'test/**'", "prod": "NODE_ENV=production webpack serve --config custom.dev.config", From cd235ee8498691b8e98cb6b1c435ceafb879f96f Mon Sep 17 00:00:00 2001 From: Martin Hradil Date: Thu, 18 Nov 2021 05:11:23 +0000 Subject: [PATCH 4/5] lint:po - convert to js fix acceented chars, decouple error reporting from processing, DRY up state handling `\w+` -> `[\p{Alpha}\p{Number}_]` & `/u` --- locale/lint.js | 129 +++++++++++++++++++++++++++++++++++++++++++++++++ locale/lint.py | 117 -------------------------------------------- package.json | 2 +- 3 files changed, 130 insertions(+), 118 deletions(-) create mode 100755 locale/lint.js delete mode 100755 locale/lint.py diff --git a/locale/lint.js b/locale/lint.js new file mode 100755 index 0000000000..370b9de59d --- /dev/null +++ b/locale/lint.js @@ -0,0 +1,129 @@ +#!/usr/bin/env node +const { readFileSync } = require('fs'); + +if (process.argv[0] === 'node' || process.argv[0].match(/\/node$/)) { + // `node script args` vs `./script args` + process.argv.shift(); +} + +if (process.argv.length < 2) { + console.error(`${process.argv[0]}: no files`); + console.error(`use: ${process.argv[0]} `); + process.exit(1); +} + +// find any {num} {str} %(str)s +const REGEX = /(\{([\p{Alpha}\p{Number}_]+|\d+)\})|(<\/?\d+\/?>)|(%(\([\p{Alpha}\p{Number}_]+\))?.)/ug; + +// extract all vars from string +const extract = (str) => Array.from( str.matchAll(REGEX) ).map((m) => m[0]); + +// set difference +const difference = (arr1, arr2) => arr1.filter(x => !arr2.includes(x)); + +// "\"foo\\n\\tbar\"" => "foo\n\tbar" +const unqqbackslash = (s) => JSON.parse(s); + +const errors = []; +const fail = (message, data = {}) => { + if (process.env.GITHUB_ACTIONS) { + const s = message.replace(/[\r\n]/g, '').replace(/%/g, '%25'); + console.error(`::error file=${data.file},line=${data.line}::${s}`); + } + + errors.push({ ...data, message }); +}; + +const process_pair = (msgid, msgstr, file, line) => { + // handling eof while still in state 2; magic id, untranslated strings + if (!msgid || !msgstr) { + return; + } + + const msgidvars = extract(msgid); + const msgstrvars = extract(msgstr); + + const missing = difference(msgidvars, msgstrvars); + const extra = difference(msgstrvars, msgidvars); + + if (missing.length) { + fail(`Missing from msgstr: ${missing.join(' ')}`, { file, line, msgid, msgstr }); + } + if (extra.length) { + fail(`Unexpected in msgstr: ${extra.join(' ')}`, { file, line, msgid, msgstr }); + } +}; + +const runState = (state, line, data = {}) => { + const done = state.find(([ regex, callback ]) => { + const match = line.match(regex); + if (match) { + callback(match); + } + return match; + }); + + if (! done) { + fail(`(${state}) Unexpected input: ${line}`, { ...data, text: line, state }); + } +}; + +process.argv.slice(1).forEach((filename) => { + let state = 0; + let msgid = null; + let msgstr = null; + let msgstrlineno = 0; + + const lines = readFileSync(filename, 'utf8').split('\n'); + + lines.forEach((line, lineno) => { + if (line.match(/^#/)) { + return; + } + + line = line.trim(); + + const states = [ + // 0 - expecting `msgid` + [ + [/^$/, () => null], + [/^msgid\s+(.*)$/, (match) => { + msgid = unqqbackslash(match[1]); + state = 1; + }], + ], + // 1 - expecting `msgstr`, or more bits of previous msgid + [ + [/^msgstr\s+(.*)$/, (match) => { + msgstr = unqqbackslash(match[1]); + msgstrlineno = lineno + 1; + state = 2; + }], + [/^"/, () => { + msgid += unqqbackslash(line); + }], + ], + // 2 - expecting newline, or more bits of previous msgstr + [ + [/^$/, () => { + process_pair(msgid, msgstr, filename, msgstrlineno); + + state = 0; + msgid = null; + msgstr = null; + msgstrlineno = 0; + }], + [/^"/, () => { + msgstr += unqqbackslash(line); + }] + ], + ]; + + runState(states[state], line, { file: filename, line: lineno + 1 }); + }); + + process_pair(msgid, msgstr, filename, msgstrlineno); +}); + +console.log(errors); +process.exit(errors.length ? 1 : 0); diff --git a/locale/lint.py b/locale/lint.py deleted file mode 100755 index c6a9fa39be..0000000000 --- a/locale/lint.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -import json -import re -import os -import sys - -if len(sys.argv) <= 1: - print(f"{sys.argv[0]}: no files", file=sys.stderr) - print(f"use: {sys.argv[0]} ", file=sys.stderr) - sys.exit(1) - -# find any {num} {str} %(str)s -REGEX = r'(\{(\w+|\d+)\})|(<\/?\d+\/?>)|(%(\(\w+\))?.)' - -# s.encode('utf-8').decode('unicode-escape') if only it worked with utf8 strings -def unqqbackslash(s): - return json.loads(s) - -def process_pair(msgid, msgstr, file, line): - # handling eof while still in state 2; magic id, untranslated strings - if msgid == None or msgstr == None or len(msgid) == 0 or len(msgstr) == 0: - return True - - # findall(...).map((arr) => arr.compact.first) - msgidvars = [ [truthy for truthy in match if truthy][0] for match in re.findall(REGEX, msgid) ] - msgstrvars = [ [truthy for truthy in match if truthy][0] for match in re.findall(REGEX, msgstr) ] - - missing = list( set(msgidvars) - set(msgstrvars) ) - extra = list( set(msgstrvars) - set(msgidvars) ) - - if len(missing) or len(extra): - message = "" - if len(missing): - msg = f"Missing from msgstr: {' '.join(missing)}" - if os.environ.get('GITHUB_ACTIONS'): - s = msg.replace("\r", "").replace("\n", "").replace('%', '%25') - print(f"::error file={file},line={line}::{s}", file=sys.stderr) - message += f" {msg}\n" - if len(extra): - msg = f"Unexpected in msgstr: {' '.join(extra)}" - if os.environ.get('GITHUB_ACTIONS'): - s = msg.replace("\r", "").replace("\n", "").replace('%', '%25') - print(f"::error file={file},line={line}::{s}", file=sys.stderr) - message += f" {msg}\n" - message += f" at {file}:{line}" - - print(f"Difference between msgid=\"{msgid}\" and msgstr=\"{msgstr}\":\n{message}\n", file=sys.stderr) - return False - - return True - -errors = False - -for filename in sys.argv[1:]: - state = 0 - msgid = None - msgstr = None - msgstrlineno = 0 - lines = None - - with open(filename) as f: - lines = f.read().splitlines() - - for lineno, line in enumerate(lines): - if re.match(r'^#', line): - continue - - line = line.strip() - - if state == 0: # expecting `msgid` - if re.match(r'^$', line): - continue - - if m := re.match(r'^msgid\s+(.*)$', line): - msgid = unqqbackslash(m[1]) - state = 1 - continue - - warnings.warn(f"({state}) Unexpected input: {line}") - errors = True - - elif state == 1: # expecting `msgstr`, or more bits of previous msgid - if m := re.match(r'^msgstr\s+(.*)$', line): - msgstr = unqqbackslash(m[1]) - msgstrlineno = lineno + 1 - state = 2 - continue - - if re.match(r'^"', line): - msgid += unqqbackslash(line) - continue - - warnings.warn(f"({state}) Unexpected input: {line}") - errors = True - - elif state == 2: # expecting newline, or more bits of previous msgstr - if re.match(r'^$', line): - if not process_pair(msgid, msgstr, filename, msgstrlineno): - errors = True - - state = 0 - msgid = None - msgstr = None - msgstrlineno = 0 - continue - - if re.match(r'^"', line): - msgstr += unqqbackslash(line) - continue - - warnings.warn(f"({state}) Unexpected input: {line}") - errors = True - - if not process_pair(msgid, msgstr, filename, msgstrlineno): - errors = True - -sys.exit(1 if errors else 0) diff --git a/package.json b/package.json index b7d43df56d..fe2a1fd5fa 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "lint:ls": "ls-lint", "lint:sass": "stylelint 'src/**/*.scss' --config .stylelintrc.json", "lint:ts": "tsc", - "lint:po": "locale/lint.py locale/*.po", + "lint:po": "locale/lint.js locale/*.po", "prettier": "prettier --write 'src/**' 'config/**' 'test/**'", "prettier:check": "prettier -l 'src/**' 'config/**' 'test/**'", "prod": "NODE_ENV=production webpack serve --config custom.dev.config", From 234f8468f0a5886638335c06e05750dd4b6fd5c6 Mon Sep 17 00:00:00 2001 From: Martin Hradil Date: Sun, 5 Dec 2021 01:38:30 +0000 Subject: [PATCH 5/5] lint:po - use released pip package instead of a local script easiest way to share between repos `pip install lint-po` --- .github/workflows/pr-checks.yml | 3 + locale/lint.js | 129 -------------------------------- package.json | 2 +- 3 files changed, 4 insertions(+), 130 deletions(-) delete mode 100755 locale/lint.js diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 90c6c6b2b3..2ebdf670bf 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -30,6 +30,9 @@ jobs: git diff --exit-code package-lock.json popd + # dependencies + pip install lint-po + # run linters npm run lint diff --git a/locale/lint.js b/locale/lint.js deleted file mode 100755 index 370b9de59d..0000000000 --- a/locale/lint.js +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env node -const { readFileSync } = require('fs'); - -if (process.argv[0] === 'node' || process.argv[0].match(/\/node$/)) { - // `node script args` vs `./script args` - process.argv.shift(); -} - -if (process.argv.length < 2) { - console.error(`${process.argv[0]}: no files`); - console.error(`use: ${process.argv[0]} `); - process.exit(1); -} - -// find any {num} {str} %(str)s -const REGEX = /(\{([\p{Alpha}\p{Number}_]+|\d+)\})|(<\/?\d+\/?>)|(%(\([\p{Alpha}\p{Number}_]+\))?.)/ug; - -// extract all vars from string -const extract = (str) => Array.from( str.matchAll(REGEX) ).map((m) => m[0]); - -// set difference -const difference = (arr1, arr2) => arr1.filter(x => !arr2.includes(x)); - -// "\"foo\\n\\tbar\"" => "foo\n\tbar" -const unqqbackslash = (s) => JSON.parse(s); - -const errors = []; -const fail = (message, data = {}) => { - if (process.env.GITHUB_ACTIONS) { - const s = message.replace(/[\r\n]/g, '').replace(/%/g, '%25'); - console.error(`::error file=${data.file},line=${data.line}::${s}`); - } - - errors.push({ ...data, message }); -}; - -const process_pair = (msgid, msgstr, file, line) => { - // handling eof while still in state 2; magic id, untranslated strings - if (!msgid || !msgstr) { - return; - } - - const msgidvars = extract(msgid); - const msgstrvars = extract(msgstr); - - const missing = difference(msgidvars, msgstrvars); - const extra = difference(msgstrvars, msgidvars); - - if (missing.length) { - fail(`Missing from msgstr: ${missing.join(' ')}`, { file, line, msgid, msgstr }); - } - if (extra.length) { - fail(`Unexpected in msgstr: ${extra.join(' ')}`, { file, line, msgid, msgstr }); - } -}; - -const runState = (state, line, data = {}) => { - const done = state.find(([ regex, callback ]) => { - const match = line.match(regex); - if (match) { - callback(match); - } - return match; - }); - - if (! done) { - fail(`(${state}) Unexpected input: ${line}`, { ...data, text: line, state }); - } -}; - -process.argv.slice(1).forEach((filename) => { - let state = 0; - let msgid = null; - let msgstr = null; - let msgstrlineno = 0; - - const lines = readFileSync(filename, 'utf8').split('\n'); - - lines.forEach((line, lineno) => { - if (line.match(/^#/)) { - return; - } - - line = line.trim(); - - const states = [ - // 0 - expecting `msgid` - [ - [/^$/, () => null], - [/^msgid\s+(.*)$/, (match) => { - msgid = unqqbackslash(match[1]); - state = 1; - }], - ], - // 1 - expecting `msgstr`, or more bits of previous msgid - [ - [/^msgstr\s+(.*)$/, (match) => { - msgstr = unqqbackslash(match[1]); - msgstrlineno = lineno + 1; - state = 2; - }], - [/^"/, () => { - msgid += unqqbackslash(line); - }], - ], - // 2 - expecting newline, or more bits of previous msgstr - [ - [/^$/, () => { - process_pair(msgid, msgstr, filename, msgstrlineno); - - state = 0; - msgid = null; - msgstr = null; - msgstrlineno = 0; - }], - [/^"/, () => { - msgstr += unqqbackslash(line); - }] - ], - ]; - - runState(states[state], line, { file: filename, line: lineno + 1 }); - }); - - process_pair(msgid, msgstr, filename, msgstrlineno); -}); - -console.log(errors); -process.exit(errors.length ? 1 : 0); diff --git a/package.json b/package.json index fe2a1fd5fa..0b97ac3c8e 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "lint:ls": "ls-lint", "lint:sass": "stylelint 'src/**/*.scss' --config .stylelintrc.json", "lint:ts": "tsc", - "lint:po": "locale/lint.js locale/*.po", + "lint:po": "lint-po locale/*.po", "prettier": "prettier --write 'src/**' 'config/**' 'test/**'", "prettier:check": "prettier -l 'src/**' 'config/**' 'test/**'", "prod": "NODE_ENV=production webpack serve --config custom.dev.config",