diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a50faa..7aa11cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ - `csvtk`: - grouping subcommands in help message. - add a new global flag `--quiet`. [#261](https://github.com/shenwei356/csvtk/issues/261) + - add a new global flag `-U, --delete-header` for disable outputing the header row. Supported commands: concat, csv2tab/tab2csv, csv2xlsx/xlsx2csv, cut, filter, filter2, freq, fold/unfold, gather, fmtdate, grep, head, join, mutate, mutate2, replace, round, sample. [#258](https://github.com/shenwei356/csvtk/issues/258) + - support more commands with `-Z/--show-row-number`: head. - `csvtk dim`: - fix duplicated rows for multiple input files, this bug was introduced in v0.27.0. - `csvtk concat`: @@ -11,6 +13,8 @@ - fix flag checking of `-k` and `-v`. - `csvtk sort`: - fix ordering when given multiple custom levels. + - `csvtk filter/filter2`: + - fix printing row number with `-Z`. - `csvtk xls2csv`: - output raw data. [#262](https://github.com/shenwei356/csvtk/issues/262) - `csvtk pretty`: diff --git a/csvtk/cmd/concat.go b/csvtk/cmd/concat.go index 6e6768a..9380360 100644 --- a/csvtk/cmd/concat.go +++ b/csvtk/cmd/concat.go @@ -133,7 +133,7 @@ so only columns match that of the first files kept. checkError(writer.Error()) }() - if !config.NoHeaderRow { + if !config.NoHeaderRow && !config.NoOutHeader { colnames := make([]string, len(COLNAMES)) for i, col := range COLNAMES { colnames[i] = COLNAME2OLDNAME[col] diff --git a/csvtk/cmd/csv.go b/csvtk/cmd/csv.go index 04fd510..1d370b0 100644 --- a/csvtk/cmd/csv.go +++ b/csvtk/cmd/csv.go @@ -503,7 +503,7 @@ func (csvReader *CSVReader) Read(opt ReadOption) { Row: row, All: record, // copied values Fields: fields, // the first variable - Selected: items, // copied values + Selected: items, // copied values. Warning: if showRowNumber is true, the first vlaue is the row number IsHeaderRow: isHeaderRow, SelectWithColnames: selectWithColnames, diff --git a/csvtk/cmd/csv2tab.go b/csvtk/cmd/csv2tab.go index a94be3b..2288b82 100644 --- a/csvtk/cmd/csv2tab.go +++ b/csvtk/cmd/csv2tab.go @@ -50,6 +50,8 @@ var csv2tabCmd = &cobra.Command{ writer.Comma = '\t' for _, file := range files { + handleHeaderRow := !config.NoHeaderRow + csvReader, err := newCSVReaderByConfig(config, file) if err != nil { if err == xopen.ErrNoContent { @@ -71,6 +73,13 @@ var csv2tabCmd = &cobra.Command{ checkError(record.Err) } + if handleHeaderRow { + handleHeaderRow = false + if config.NoOutHeader { + continue + } + } + checkError(writer.Write(record.Selected)) } diff --git a/csvtk/cmd/csv2xlsx.go b/csvtk/cmd/csv2xlsx.go index 8e143bb..4e13531 100644 --- a/csvtk/cmd/csv2xlsx.go +++ b/csvtk/cmd/csv2xlsx.go @@ -110,7 +110,7 @@ Attention: } } - if !config.NoHeaderRow { + if !config.NoHeaderRow && !config.NoOutHeader { checkError(xlsx.SetPanes(sheet, &excelize.Panes{ Freeze: true, Split: false, @@ -122,10 +122,19 @@ Attention: } line = 1 + handleHeaderRow := !config.NoHeaderRow for record := range csvReader.Ch { if record.Err != nil { checkError(record.Err) } + + if handleHeaderRow { + handleHeaderRow = false + if config.NoOutHeader { + continue + } + } + for col, val = range record.Selected { cell = fmt.Sprintf("%s%d", ExcelColumnIndex(col), line) if formatNumbers { diff --git a/csvtk/cmd/cut.go b/csvtk/cmd/cut.go index 8934663..ac5c97f 100644 --- a/csvtk/cmd/cut.go +++ b/csvtk/cmd/cut.go @@ -124,11 +124,19 @@ Examples: ShowRowNumber: config.ShowRowNumber, }) + handleHeaderRow := !config.NoHeaderRow for record := range csvReader.Ch { if record.Err != nil { checkError(record.Err) } + if handleHeaderRow { + handleHeaderRow = false + if config.NoOutHeader { + continue + } + } + writer.Write(record.Selected) } diff --git a/csvtk/cmd/filter.go b/csvtk/cmd/filter.go index ccc4abd..43cc538 100644 --- a/csvtk/cmd/filter.go +++ b/csvtk/cmd/filter.go @@ -94,6 +94,8 @@ var filterCmd = &cobra.Command{ checkError(writer.Error()) }() + showRowNumber := printLineNumber || config.ShowRowNumber + for _, file := range files { csvReader, err := newCSVReaderByConfig(config, file) @@ -110,7 +112,7 @@ var filterCmd = &cobra.Command{ csvReader.Read(ReadOption{ FieldStr: fieldStr, FuzzyFields: fuzzyFields, - ShowRowNumber: printLineNumber || config.ShowRowNumber, + ShowRowNumber: showRowNumber, DoNotAllowDuplicatedColumnName: true, }) @@ -120,6 +122,7 @@ var filterCmd = &cobra.Command{ var n int var v float64 var val string + var i int checkFirstLine := true for record := range csvReader.Ch { @@ -131,7 +134,10 @@ var filterCmd = &cobra.Command{ checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line - if printLineNumber { + if config.NoOutHeader { + continue + } + if showRowNumber { unshift(&record.All, "row") } checkError(writer.Write(record.All)) @@ -144,7 +150,11 @@ var filterCmd = &cobra.Command{ flag = false n = 0 - for _, val = range record.Selected { + for i, val = range record.Selected { + if showRowNumber && i == 0 { // skip the row number + continue + } + if !reDigitals.MatchString(val) { flag = false break @@ -187,6 +197,7 @@ var filterCmd = &cobra.Command{ break } } + } if n == len(record.Fields) { // all satisfied @@ -196,7 +207,7 @@ var filterCmd = &cobra.Command{ continue } - if printLineNumber { + if showRowNumber { unshift(&record.All, strconv.Itoa(record.Row)) } checkError(writer.Write(record.All)) diff --git a/csvtk/cmd/filter2.go b/csvtk/cmd/filter2.go index ca58c27..bd74b3a 100644 --- a/csvtk/cmd/filter2.go +++ b/csvtk/cmd/filter2.go @@ -190,6 +190,8 @@ Custom functions: checkError(writer.Error()) }() + showRowNumber := printLineNumber || config.ShowRowNumber + for _, file := range files { csvReader, err := newCSVReaderByConfig(config, file) @@ -207,7 +209,7 @@ Custom functions: FieldStr: fieldStr, FieldStrSep: varSep, FuzzyFields: fuzzyFields, - ShowRowNumber: printLineNumber || config.ShowRowNumber, + ShowRowNumber: showRowNumber, DoNotAllowDuplicatedColumnName: true, }) @@ -263,7 +265,10 @@ Custom functions: colnamesMap[col] = fuzzyField2Regexp(col) } - if printLineNumber { + if config.NoOutHeader { + continue + } + if showRowNumber { unshift(&record.All, "row") } checkError(writer.Write(record.All)) @@ -398,7 +403,7 @@ Custom functions: continue } - if printLineNumber { + if showRowNumber { unshift(&record.All, strconv.Itoa(record.Row)) } checkError(writer.Write(record.All)) diff --git a/csvtk/cmd/fmtdate.go b/csvtk/cmd/fmtdate.go index 74b330b..62e9ee6 100644 --- a/csvtk/cmd/fmtdate.go +++ b/csvtk/cmd/fmtdate.go @@ -152,6 +152,9 @@ Placeholders: checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line + if config.NoOutHeader { + continue + } checkError(writer.Write(record.All)) continue } diff --git a/csvtk/cmd/fold.go b/csvtk/cmd/fold.go index afbe381..bd8fc9e 100644 --- a/csvtk/cmd/fold.go +++ b/csvtk/cmd/fold.go @@ -164,6 +164,9 @@ Example: checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line + if config.NoOutHeader { + continue + } checkError(writer.Write(record.Selected)) continue } diff --git a/csvtk/cmd/freq.go b/csvtk/cmd/freq.go index edcf1f8..7d2ee1d 100644 --- a/csvtk/cmd/freq.go +++ b/csvtk/cmd/freq.go @@ -119,6 +119,9 @@ var freqCmd = &cobra.Command{ if checkFirstLine { checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { + if config.NoOutHeader { + continue + } checkError(writer.Write(append(record.Selected, "frequency"))) continue } diff --git a/csvtk/cmd/gather.go b/csvtk/cmd/gather.go index 2cd8aa4..0753c24 100644 --- a/csvtk/cmd/gather.go +++ b/csvtk/cmd/gather.go @@ -164,8 +164,9 @@ var gatherCmd = &cobra.Command{ if handleHeaderRow { items[nFieldsLeft] = fieldKey items[nFieldsLeft+1] = fieldValue - checkError(writer.Write(items)) - + if !config.NoOutHeader { + checkError(writer.Write(items)) + } handleHeaderRow = false } else { for _, f = range record.Fields { diff --git a/csvtk/cmd/grep.go b/csvtk/cmd/grep.go index 85feccf..b830ed6 100644 --- a/csvtk/cmd/grep.go +++ b/csvtk/cmd/grep.go @@ -210,6 +210,9 @@ Attentions: checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { + if config.NoOutHeader { + continue + } if printLineNumber { unshift(&record.All, "row") } diff --git a/csvtk/cmd/head.go b/csvtk/cmd/head.go index 21b2e4c..7f111dc 100644 --- a/csvtk/cmd/head.go +++ b/csvtk/cmd/head.go @@ -23,6 +23,7 @@ package cmd import ( "encoding/csv" "runtime" + "strconv" "github.com/shenwei356/xopen" "github.com/spf13/cobra" @@ -63,6 +64,8 @@ var headCmd = &cobra.Command{ checkError(writer.Error()) }() + printLineNumber := config.ShowRowNumber + for _, file := range files { csvReader, err := newCSVReaderByConfig(config, file) @@ -87,14 +90,23 @@ var headCmd = &cobra.Command{ checkError(record.Err) } - checkError(writer.Write(record.All)) - if isHeaderLine { isHeaderLine = false + if config.NoOutHeader { + continue + } + if printLineNumber { + unshift(&record.All, "row") + } } else { i++ + if printLineNumber { + unshift(&record.All, strconv.Itoa(record.Row)) + } } + checkError(writer.Write(record.All)) + if number == i { break } diff --git a/csvtk/cmd/helper.go b/csvtk/cmd/helper.go index 4b4d9de..d5d1d83 100644 --- a/csvtk/cmd/helper.go +++ b/csvtk/cmd/helper.go @@ -321,7 +321,7 @@ func getConfigs(cmd *cobra.Command) Config { Tabs: tabs, OutTabs: getFlagBool(cmd, "out-tabs"), NoHeaderRow: noHeaderRow, - NoOutHeader: getFlagBool(cmd, "no-out-header"), + NoOutHeader: getFlagBool(cmd, "delete-header"), ShowRowNumber: getFlagBool(cmd, "show-row-number"), diff --git a/csvtk/cmd/inter.go b/csvtk/cmd/inter.go index 236a4ca..dc4ffac 100644 --- a/csvtk/cmd/inter.go +++ b/csvtk/cmd/inter.go @@ -173,7 +173,7 @@ Attention: return } - if hasHeaderLine { + if hasHeaderLine && !config.NoOutHeader { checkError(writer.Write(selectedColnames)) } for key := range keysMaps { diff --git a/csvtk/cmd/join.go b/csvtk/cmd/join.go index 4874882..7598d53 100644 --- a/csvtk/cmd/join.go +++ b/csvtk/cmd/join.go @@ -530,18 +530,20 @@ Attention: Data = Data2 } - if withHeaderRow { - if filenameAsPrefix { + if !config.NoOutHeader { + if withHeaderRow { + if filenameAsPrefix { + checkError(writer.Write(prefixedHeaderRow)) + } else if addSuffix { + checkError(writer.Write(suffixedHeaderRow)) + } else { + checkError(writer.Write(HeaderRow)) + } + } else if filenameAsPrefix { checkError(writer.Write(prefixedHeaderRow)) } else if addSuffix { checkError(writer.Write(suffixedHeaderRow)) - } else { - checkError(writer.Write(HeaderRow)) } - } else if filenameAsPrefix { - checkError(writer.Write(prefixedHeaderRow)) - } else if addSuffix { - checkError(writer.Write(suffixedHeaderRow)) } for _, record := range Data { checkError(writer.Write(record)) diff --git a/csvtk/cmd/mutate.go b/csvtk/cmd/mutate.go index 0a45e48..253abc3 100644 --- a/csvtk/cmd/mutate.go +++ b/csvtk/cmd/mutate.go @@ -73,7 +73,7 @@ var mutateCmd = &cobra.Command{ } name := getFlagString(cmd, "name") - if !config.NoHeaderRow && name == "" { + if !config.NoHeaderRow && name == "" && !config.NoOutHeader { checkError(fmt.Errorf("flag -n (--name) needed")) } @@ -204,7 +204,9 @@ var mutateCmd = &cobra.Command{ } handleHeaderRow = false - checkError(writer.Write(record2)) + if !config.NoOutHeader { + checkError(writer.Write(record2)) + } continue } diff --git a/csvtk/cmd/mutate2.go b/csvtk/cmd/mutate2.go index 02c8f21..1b00e0c 100644 --- a/csvtk/cmd/mutate2.go +++ b/csvtk/cmd/mutate2.go @@ -101,7 +101,7 @@ Custom functions: runtime.GOMAXPROCS(config.NumCPUs) name := getFlagString(cmd, "name") - if !config.NoHeaderRow && name == "" { + if !config.NoHeaderRow && name == "" && !config.NoOutHeader { checkError(fmt.Errorf("falg -n (--name) needed")) } @@ -300,7 +300,9 @@ Custom functions: record2[at-1] = value } - checkError(writer.Write(record2)) + if !config.NoOutHeader { + checkError(writer.Write(record2)) + } continue } diff --git a/csvtk/cmd/replace.go b/csvtk/cmd/replace.go index 3aa24ba..018896f 100644 --- a/csvtk/cmd/replace.go +++ b/csvtk/cmd/replace.go @@ -194,6 +194,9 @@ Special replacement symbols: checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line + if config.NoOutHeader { + continue + } checkError(writer.Write(record.All)) continue } diff --git a/csvtk/cmd/root.go b/csvtk/cmd/root.go index 300422b..fd4a12d 100644 --- a/csvtk/cmd/root.go +++ b/csvtk/cmd/root.go @@ -140,7 +140,7 @@ func init() { RootCmd.PersistentFlags().BoolP("tabs", "t", false, `specifies that the input CSV file is delimited with tabs. Overrides "-d"`) RootCmd.PersistentFlags().BoolP("out-tabs", "T", false, `specifies that the output is delimited with tabs. Overrides "-D"`) RootCmd.PersistentFlags().BoolP("no-header-row", "H", false, `specifies that the input CSV file does not have header row`) - RootCmd.PersistentFlags().BoolP("no-out-header", "U", false, `do not output header row`) + RootCmd.PersistentFlags().BoolP("delete-header", "U", false, `do not output header row`) RootCmd.PersistentFlags().StringP("out-file", "o", "-", `out file ("-" for stdout, suffix .gz for gzipped out)`) RootCmd.PersistentFlags().BoolP("show-row-number", "Z", false, `show row number as the first column, with header row skipped`) diff --git a/csvtk/cmd/round.go b/csvtk/cmd/round.go index 78ebda0..a59e200 100644 --- a/csvtk/cmd/round.go +++ b/csvtk/cmd/round.go @@ -114,6 +114,9 @@ var roundCmd = &cobra.Command{ checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line + if config.NoOutHeader { + continue + } checkError(writer.Write(record.All)) continue } diff --git a/csvtk/cmd/sample.go b/csvtk/cmd/sample.go index ec3ffdf..043d8b3 100644 --- a/csvtk/cmd/sample.go +++ b/csvtk/cmd/sample.go @@ -106,6 +106,9 @@ var sampleCmd = &cobra.Command{ checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line + if config.NoOutHeader { + continue + } checkError(writer.Write(record.Selected)) continue } diff --git a/csvtk/cmd/sort.go b/csvtk/cmd/sort.go index c5abba0..f5ac949 100644 --- a/csvtk/cmd/sort.go +++ b/csvtk/cmd/sort.go @@ -233,7 +233,7 @@ var sortCmd = &cobra.Command{ } sorts.Quicksort(stringutil.MultiKeyStringSliceList(list)) - if len(headerRow) > 0 { + if len(headerRow) > 0 && !config.NoOutHeader { checkError(writer.Write(headerRow)) } for _, s := range list { diff --git a/csvtk/cmd/tab2csv.go b/csvtk/cmd/tab2csv.go index daff93b..9ada637 100644 --- a/csvtk/cmd/tab2csv.go +++ b/csvtk/cmd/tab2csv.go @@ -69,11 +69,20 @@ var tab2csvCmd = &cobra.Command{ ShowRowNumber: config.ShowRowNumber, }) + handleHeaderRow := !config.NoHeaderRow + for record := range csvReader.Ch { if record.Err != nil { checkError(record.Err) } + if handleHeaderRow { + handleHeaderRow = false + if config.NoOutHeader { + continue + } + } + checkError(writer.Write(record.Selected)) } diff --git a/csvtk/cmd/unfold.go b/csvtk/cmd/unfold.go index e42d863..7d5f413 100644 --- a/csvtk/cmd/unfold.go +++ b/csvtk/cmd/unfold.go @@ -129,6 +129,9 @@ Example: } if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line + if config.NoOutHeader { + continue + } checkError(writer.Write(record.All)) continue } diff --git a/csvtk/cmd/uniq.go b/csvtk/cmd/uniq.go index 9e5fefb..dc6c6c5 100644 --- a/csvtk/cmd/uniq.go +++ b/csvtk/cmd/uniq.go @@ -116,6 +116,9 @@ var uniqCmd = &cobra.Command{ checkFirstLine = false if !config.NoHeaderRow || record.IsHeaderRow { // do not replace head line + if config.NoOutHeader { + continue + } checkError(writer.Write(record.All)) continue } diff --git a/csvtk/cmd/xlsx2csv.go b/csvtk/cmd/xlsx2csv.go index b99a425..e429488 100644 --- a/csvtk/cmd/xlsx2csv.go +++ b/csvtk/cmd/xlsx2csv.go @@ -131,6 +131,7 @@ var xlsx2csvCmd = &cobra.Command{ var notBlank bool var data string var numEmptyRows int + handleHeaderRow := !config.NoHeaderRow for _, row := range rows { if len(row) < nColsMax { row = append(row, emptyRow[0:nColsMax-len(row)]...) @@ -148,6 +149,14 @@ var xlsx2csvCmd = &cobra.Command{ continue } } + + if handleHeaderRow { + handleHeaderRow = false + if config.NoOutHeader { + continue + } + } + checkError(writer.Write(row)) } checkError(xlsx.Close()) diff --git a/doc/docs/usage.md b/doc/docs/usage.md index c8aa5b4..663398e 100644 --- a/doc/docs/usage.md +++ b/doc/docs/usage.md @@ -242,6 +242,7 @@ Flags: -C, --comment-char string lines starting with commment-character will be ignored. if your header row starts with '#', please assign "-C" another rare symbol, e.g. '$' (default "#") + -U, --delete-header do not output header row -d, --delimiter string delimiting character of the input CSV file (default ",") -h, --help help for csvtk -E, --ignore-empty-row ignore empty rows @@ -620,13 +621,13 @@ Flags: -i, --ignore_nan Ignore non-numeric fields to avoid returning NaN -L, --log Calcute correlations on Log10 transformed data -x, --pass passthrough mode (forward input to output) -``` +``` Examples 1. Calculate pairwise correlations between field, ignore non-numeric values - csvtk -t corr -i -f 1,Foo,Bar input.tsv + csvtk -t corr -i -f Foo,Bar input.tsv ## pretty