diff --git a/CHANGELOG.md b/CHANGELOG.md index b276206..f6c359e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ Types of changes - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [0.2.0] + +- `Added` flag `--only` (short `-o`) to only scan a specific list of fields +- `Added` flag `--alias` (short `-a`) to rename fields on the fly +- `Fixed` self reference link are no longer counted in the links counter while scanning + ## [0.1.0] - `Added` initial version diff --git a/internal/app/cli/scan.go b/internal/app/cli/scan.go index d3cebf1..2e35bdb 100644 --- a/internal/app/cli/scan.go +++ b/internal/app/cli/scan.go @@ -28,7 +28,11 @@ import ( ) func NewScanCommand(parent string, stderr *os.File, stdout *os.File, stdin *os.File) *cobra.Command { - var passthrough bool + var ( + passthrough bool + only []string + aliases map[string]string + ) cmd := &cobra.Command{ //nolint:exhaustruct Use: "scan path", @@ -36,13 +40,15 @@ func NewScanCommand(parent string, stderr *os.File, stdout *os.File, stdin *os.F Example: " lino pull database --table client | " + parent + " scan clients", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - if err := scan(cmd, args[0], passthrough); err != nil { + if err := scan(cmd, args[0], passthrough, only, aliases); err != nil { log.Fatal().Err(err).Int("return", 1).Msg("end SILO") } }, } cmd.Flags().BoolVarP(&passthrough, "passthrough", "p", false, "pass stdin to stdout") + cmd.Flags().StringSliceVarP(&only, "only", "o", []string{}, "only scan these columns, exclude all others") + cmd.Flags().StringToStringVarP(&aliases, "alias", "a", map[string]string{}, "use given aliases for each columns") cmd.SetOut(stdout) cmd.SetErr(stderr) @@ -51,7 +57,7 @@ func NewScanCommand(parent string, stderr *os.File, stdout *os.File, stdin *os.F return cmd } -func scan(cmd *cobra.Command, path string, passthrough bool) error { +func scan(cmd *cobra.Command, path string, passthrough bool, only []string, aliases map[string]string) error { backend, err := infra.NewBackend(path) if err != nil { return fmt.Errorf("%w", err) @@ -59,7 +65,7 @@ func scan(cmd *cobra.Command, path string, passthrough bool) error { defer backend.Close() - driver := silo.NewDriver(backend, nil) + driver := silo.NewDriver(backend, nil, silo.WithKeys(only), silo.WithAliases(aliases)) var reader silo.DataRowReader diff --git a/pkg/silo/config.go b/pkg/silo/config.go new file mode 100644 index 0000000..e63689d --- /dev/null +++ b/pkg/silo/config.go @@ -0,0 +1,50 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +import "errors" + +type Config struct { + Include map[string]bool + Aliases map[string]string +} + +func DefaultConfig() *Config { + config := Config{ + Include: map[string]bool{}, + Aliases: map[string]string{}, + } + + return &config +} + +func (cfg *Config) validate() error { + var errs []error + + for key := range cfg.Aliases { + if _, ok := cfg.Include[key]; !ok && len(cfg.Include) > 0 { + errs = append(errs, &ConfigScanAliasIsNotIncludedError{alias: key}) + } + } + + if len(errs) != 0 { + return errors.Join(errs...) + } + + return nil +} diff --git a/pkg/silo/driver.go b/pkg/silo/driver.go index b573519..96c2366 100644 --- a/pkg/silo/driver.go +++ b/pkg/silo/driver.go @@ -26,14 +26,33 @@ import ( ) type Driver struct { + *Config backend Backend writer DumpWriter } -func NewDriver(backend Backend, writer DumpWriter) *Driver { +func NewDriver(backend Backend, writer DumpWriter, options ...Option) *Driver { + errs := []error{} + config := DefaultConfig() + + for _, option := range options { + if err := option.apply(config); err != nil { + errs = append(errs, err) + } + } + + if err := config.validate(); err != nil { + errs = append(errs, err) + } + + if len(errs) > 0 { + panic(errs) + } + return &Driver{ backend: backend, writer: writer, + Config: config, } } @@ -102,7 +121,7 @@ func (d *Driver) Scan(input DataRowReader, observers ...ScanObserver) error { break } - links := Scan(datarow) + links := d.scan(datarow) log.Info().Int("links", len(links)).Interface("row", datarow).Msg("datarow scanned") @@ -120,12 +139,14 @@ func (d *Driver) ingest(datarow DataRow, links []DataLink, observers ...ScanObse return fmt.Errorf("%w: %w", ErrPersistingData, err) } - if err := d.backend.Store(link.E2, link.E1); err != nil { - return fmt.Errorf("%w: %w", ErrPersistingData, err) - } + if link.E1 != link.E2 { + if err := d.backend.Store(link.E2, link.E1); err != nil { + return fmt.Errorf("%w: %w", ErrPersistingData, err) + } - for _, observer := range observers { - observer.IngestedLink(link) + for _, observer := range observers { + observer.IngestedLink(link) + } } } @@ -136,12 +157,16 @@ func (d *Driver) ingest(datarow DataRow, links []DataLink, observers ...ScanObse return nil } -func Scan(datarow DataRow) []DataLink { +func (cfg *Config) scan(datarow DataRow) []DataLink { nodes := []DataNode{} links := []DataLink{} for key, value := range datarow { - if value != nil { + if _, included := cfg.Include[key]; value != nil && (included || len(cfg.Include) == 0) { + if alias, exist := cfg.Aliases[key]; exist { + key = alias + } + nodes = append(nodes, DataNode{Key: key, Data: value}) } } diff --git a/pkg/silo/errors.go b/pkg/silo/errors.go index cf477c6..4956ba1 100644 --- a/pkg/silo/errors.go +++ b/pkg/silo/errors.go @@ -17,10 +17,21 @@ package silo -import "errors" +import ( + "errors" + "fmt" +) var ( ErrReadingNextInput = errors.New("error while reading next input") ErrPersistingData = errors.New("error while persisting data") ErrReadingPersistedData = errors.New("error while reading persisted data") ) + +type ConfigScanAliasIsNotIncludedError struct { + alias string +} + +func (e *ConfigScanAliasIsNotIncludedError) Error() string { + return fmt.Sprintf("configuration error : alias [%s] is not included", e.alias) +} diff --git a/pkg/silo/options.go b/pkg/silo/options.go new file mode 100644 index 0000000..ed1c7b7 --- /dev/null +++ b/pkg/silo/options.go @@ -0,0 +1,76 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +type Option interface { + applier +} + +type option func(*Config) error + +func (f option) apply(cfg *Config) error { + return f(cfg) +} + +type applier interface { + apply(cfg *Config) error +} + +func Alias(key, alias string) Option { //nolint:ireturn + applier := func(cfg *Config) error { + cfg.Aliases[key] = alias + + return nil + } + + return option(applier) +} + +func Include(key string) Option { //nolint:ireturn + applier := func(cfg *Config) error { + cfg.Include[key] = true + + return nil + } + + return option(applier) +} + +func WithAliases(aliases map[string]string) Option { //nolint:ireturn + applier := func(cfg *Config) error { + for key, alias := range aliases { + cfg.Aliases[key] = alias + } + + return nil + } + + return option(applier) +} + +func WithKeys(keys []string) Option { //nolint:ireturn + applier := func(cfg *Config) error { + for _, key := range keys { + cfg.Include[key] = true + } + + return nil + } + + return option(applier) +}