diff --git a/cmd/mimo/main.go b/cmd/mimo/main.go index f61c34d..22b4d3c 100644 --- a/cmd/mimo/main.go +++ b/cmd/mimo/main.go @@ -142,7 +142,7 @@ func run(_ *cobra.Command, realJSONLineFileName string) error { haserror := false - var report mimo.Report + var report *mimo.Report if report, err = runAnalyse(driver, profiling); err != nil { return fmt.Errorf("%w", err) @@ -175,14 +175,14 @@ func run(_ *cobra.Command, realJSONLineFileName string) error { return nil } -func runAnalyse(driver mimo.Driver, profiling bool) (mimo.Report, error) { +func runAnalyse(driver mimo.Driver, profiling bool) (*mimo.Report, error) { var cpuProfiler interface{ Stop() } if profiling { cpuProfiler = profile.Start(profile.ProfilePath(".")) } - var report mimo.Report + var report *mimo.Report report, err := driver.Analyze() if err != nil { @@ -251,7 +251,7 @@ func selectCounterFactory() mimo.CounterFactory { return counterFactory } -func appendColumnMetric(report mimo.Report, colname string, haserror bool) bool { +func appendColumnMetric(report *mimo.Report, colname string, haserror bool) bool { metrics := report.ColumnMetric(colname) if metrics.Validate() >= 0 { log.Info(). diff --git a/internal/infra/datarowreader_jsonline.go b/internal/infra/datarowreader_jsonline.go index 89f8caa..f14dcf9 100644 --- a/internal/infra/datarowreader_jsonline.go +++ b/internal/infra/datarowreader_jsonline.go @@ -29,9 +29,15 @@ import ( "github.com/cgi-fr/mimo/pkg/mimo" ) +const linebreak byte = 10 + type DataRowReaderJSONLine struct { + input *bufio.Scanner +} + +type DataRowReaderWriterJSONLine struct { input *bufio.Scanner - output io.Writer + output *bufio.Writer } func NewDataRowReaderJSONLineFromFile(filename string) (*DataRowReaderJSONLine, error) { @@ -40,20 +46,47 @@ func NewDataRowReaderJSONLineFromFile(filename string) (*DataRowReaderJSONLine, return nil, fmt.Errorf("%w", err) } - return &DataRowReaderJSONLine{input: bufio.NewScanner(source), output: io.Discard}, nil + return &DataRowReaderJSONLine{input: bufio.NewScanner(source)}, nil } -func NewDataRowReaderJSONLine(input io.Reader, output io.Writer) *DataRowReaderJSONLine { - return &DataRowReaderJSONLine{input: bufio.NewScanner(input), output: output} +func NewDataRowReaderJSONLine(input io.Reader, output io.Writer) *DataRowReaderWriterJSONLine { + return &DataRowReaderWriterJSONLine{input: bufio.NewScanner(input), output: bufio.NewWriter(output)} } func (drr *DataRowReaderJSONLine) ReadDataRow() (mimo.DataRow, error) { var data mimo.DataRow if drr.input.Scan() { - if _, err := drr.output.Write(append(drr.input.Bytes(), '\n')); err != nil { + data = mimo.DataRow{} + if err := json.UnmarshalNoEscape(drr.input.Bytes(), &data); err != nil { return nil, fmt.Errorf("%w", err) } + } + + if err := drr.input.Err(); err != nil { + if errors.Is(err, io.EOF) { + return nil, nil + } + + return nil, fmt.Errorf("%w", err) + } + + return data, nil +} + +func (drr *DataRowReaderJSONLine) Close() error { + return nil +} + +func (drr *DataRowReaderWriterJSONLine) ReadDataRow() (mimo.DataRow, error) { + var data mimo.DataRow + + if drr.input.Scan() { + if drr.output != nil { + if err := drr.writeLine(); err != nil { + return nil, err + } + } data = mimo.DataRow{} if err := json.UnmarshalNoEscape(drr.input.Bytes(), &data); err != nil { @@ -71,3 +104,23 @@ func (drr *DataRowReaderJSONLine) ReadDataRow() (mimo.DataRow, error) { return data, nil } + +func (drr *DataRowReaderWriterJSONLine) writeLine() error { + if _, err := drr.output.Write(drr.input.Bytes()); err != nil { + return fmt.Errorf("%w", err) + } + + if err := drr.output.WriteByte(linebreak); err != nil { + return fmt.Errorf("%w", err) + } + + return nil +} + +func (drr *DataRowReaderWriterJSONLine) Close() error { + if drr.output == nil { + return nil + } + + return fmt.Errorf("%w", drr.output.Flush()) +} diff --git a/internal/infra/html_reports.go b/internal/infra/html_reports.go index 5718c8a..bc1eed0 100644 --- a/internal/infra/html_reports.go +++ b/internal/infra/html_reports.go @@ -43,7 +43,7 @@ func NewReportExporter() ReportExporter { return ReportExporter{tmpl: t} } -func (e ReportExporter) Export(report mimo.Report, filename string) error { +func (e ReportExporter) Export(report *mimo.Report, filename string) error { if file, err := os.Create(filename); err != nil { return fmt.Errorf("%w", err) } else if err := e.tmpl.Execute(file, report); err != nil { diff --git a/pkg/mimo/driven.go b/pkg/mimo/driven.go index e5cccc6..2af5266 100644 --- a/pkg/mimo/driven.go +++ b/pkg/mimo/driven.go @@ -19,6 +19,7 @@ package mimo type DataRowReader interface { ReadDataRow() (DataRow, error) + Close() error } type EventSubscriber interface { diff --git a/pkg/mimo/driver.go b/pkg/mimo/driver.go index 200dd8c..ca54db2 100644 --- a/pkg/mimo/driver.go +++ b/pkg/mimo/driver.go @@ -27,7 +27,7 @@ type Driver struct { realDataSource DataRowReader maskDataSource DataRowReader subscribers Suscribers - report Report + report *Report } func NewDriver( @@ -49,7 +49,10 @@ func (d *Driver) Configure(c Config) { d.report.config = c } -func (d *Driver) Analyze() (Report, error) { +func (d *Driver) Analyze() (*Report, error) { + defer d.realDataSource.Close() + defer d.maskDataSource.Close() + for { realRow, err := d.realDataSource.ReadDataRow() if err != nil { diff --git a/pkg/mimo/model.go b/pkg/mimo/model.go index f385f05..56a4865 100644 --- a/pkg/mimo/model.go +++ b/pkg/mimo/model.go @@ -342,8 +342,8 @@ type Report struct { func NewReport( subs []EventSubscriber, config Config, multiMapFactory MultimapFactory, counterFactory CounterFactory, -) Report { - return Report{make(map[string]Metrics), subs, config, multiMapFactory, counterFactory} +) *Report { + return &Report{make(map[string]Metrics), subs, config, multiMapFactory, counterFactory} } func (r Report) UpdateDeep(root DataRow, realRow DataRow, maskedRow DataRow, stack []any, path ...string) {