diff --git a/deegree-services/deegree-webservices-handbook/src/main/asciidoc/cli-utility.adoc b/deegree-services/deegree-webservices-handbook/src/main/asciidoc/cli-utility.adoc index 84b411b0a0..dde4017665 100644 --- a/deegree-services/deegree-webservices-handbook/src/main/asciidoc/cli-utility.adoc +++ b/deegree-services/deegree-webservices-handbook/src/main/asciidoc/cli-utility.adoc @@ -170,6 +170,7 @@ arguments: -sqlFeatureStoreId=, the ID of the SQLFeatureStore in the given workspace options: + -reportWriteStatistics=true, create a summary of all written feature types, disabled by default -disabledResources=, a comma separated list url patterns which should not be resolved, not set by default Example: diff --git a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/FeatureStatistics.java b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/FeatureStatistics.java new file mode 100644 index 0000000000..5db0525b24 --- /dev/null +++ b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/FeatureStatistics.java @@ -0,0 +1,74 @@ +/*- + * #%L + * deegree-cli-utility + * %% + * Copyright (C) 2022 grit graphische Informationstechnik Beratungsgesellschaft mbH + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 2.1 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.deegree.tools.featurestoresql.loader; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +import javax.xml.namespace.QName; + +import org.deegree.feature.Feature; +import org.springframework.batch.core.ItemWriteListener; + +/** + * Collects statistics for each type loaded + * + * @author Stephan Reichhelm + */ +public class FeatureStatistics { + private Map counter = new TreeMap<>( Comparator.comparing( QName::getNamespaceURI ).thenComparing( QName::getLocalPart ) ); + + public void reset() { + counter.clear(); + } + + /** + * @param name + * type name to increment + */ + public long increment( QName name ) { + AtomicLong cnt = counter.get( name ); + if ( cnt == null ) { + cnt = new AtomicLong(); + counter.put( name, cnt ); + } + return cnt.incrementAndGet(); + } + + /** + * @param out + * Consumer function that receives the output + */ + public void summary( Consumer out ) { + final long max = counter.values().stream() // + .map( AtomicLong::get ) // + .max( Long::compareTo )// + .orElse( 1L ); + final int len = String.valueOf( max ).length(); + final String format = "%" + len + "d %s"; + counter.forEach( ( k, v ) -> out.accept( String.format( format, v.get(), k.toString() ) ) ); + } +} diff --git a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/FeatureStoreWriter.java b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/FeatureStoreWriter.java index 14dab3a649..1e3896a9b8 100644 --- a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/FeatureStoreWriter.java +++ b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/FeatureStoreWriter.java @@ -3,6 +3,7 @@ * deegree-cli-utility * %% * Copyright (C) 2016 - 2021 lat/lon GmbH + * Copyright (C) 2022 grit graphische Informationstechnik Beratungsgesellschaft mbH * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as @@ -63,20 +64,20 @@ public FeatureStoreWriter( SQLFeatureStore sqlFeatureStore, Summary summary ) { } @Override - public void write( List features ) + public void write( List features ) throws Exception { FeatureCollection featureCollection = new GenericFeatureCollection(); - for ( Object feature : features ) { - Feature featureToAdd = (Feature) feature; - LOG.info( "Adding feature with GML ID '"+featureToAdd.getId()+"' of type '"+featureToAdd.getType().getName()+"' to chunk" ); + for ( Feature featureToAdd : features ) { + LOG.debug( "Adding feature with GML ID '{}' of type '{}' to chunk", featureToAdd.getId(), + featureToAdd.getType().getName() ); featureCollection.add( featureToAdd ); + summary.increaseNumberOfFeatures( featureToAdd.getType().getName() ); } - LOG.info( "Trying to write " + featureCollection.size() + " features" ); + LOG.info( "Trying to write {} features", featureCollection.size() ); SQLFeatureStoreTransaction transaction = (SQLFeatureStoreTransaction) sqlFeatureStore.getTransaction(); transaction.performInsert( featureCollection, USE_EXISTING.withSkipResolveReferences( true ) ); LOG.info( "Insert performed." ); - summary.increaseNumberOfFeatures( featureCollection.size() ); } } \ No newline at end of file diff --git a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderConfiguration.java b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderConfiguration.java index 4bff224460..69ea57045b 100644 --- a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderConfiguration.java +++ b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderConfiguration.java @@ -3,6 +3,7 @@ * deegree-cli-utility * %% * Copyright (C) 2016 - 2021 lat/lon GmbH + * Copyright (C) 2022 grit graphische Informationstechnik Beratungsgesellschaft mbH * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as @@ -38,6 +39,7 @@ import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.core.launch.support.RunIdIncrementer; @@ -64,9 +66,16 @@ public class GmlLoaderConfiguration { @Autowired private StepBuilderFactory stepBuilderFactory; + @JobScope @Bean - public Summary summary() { - return new Summary(); + public Summary summary( @Value("#{jobParameters[reportWriteStatistics] ?: false}") boolean reportWriteStatistics ) { + Summary summary = new Summary(); + + if ( reportWriteStatistics ) { + summary.setStatistics( new FeatureStatistics() ); + } + + return summary; } @Bean diff --git a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderHelpUsage.java b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderHelpUsage.java index aefceba67a..e71079b862 100644 --- a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderHelpUsage.java +++ b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/GmlLoaderHelpUsage.java @@ -25,10 +25,12 @@ * GmlLoader CLI usage info. * * @author Lyn Goltz + * @author Stephan Reichhelm */ public class GmlLoaderHelpUsage { public static void printUsage() { + // see also the webservices-handbook chapter for the CLI System.out.println( "Usage: java -jar deegree-tools-gml.jar GmlLoader -pathToFile= -workspaceName= -sqlFeatureStoreId= [options]" ); System.out.println( "Description: Imports a GML file directly into a given deegree SQLFeatureStore" ); System.out.println(); @@ -38,6 +40,7 @@ public static void printUsage() { System.out.println( " -sqlFeatureStoreId=, the ID of the SQLFeatureStore in the given workspace" ); System.out.println(); System.out.println( "options:" ); + System.out.println( " -reportWriteStatistics=true, create a summary of all written feature types, disabled by default"); System.out.println( " -disabledResources=, a comma separated list url patterns which should not be resolved, not set by default" ); System.out.println(); System.out.println( "Example:" ); diff --git a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/ReportWriter.java b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/ReportWriter.java index 590ade848d..69dc1a6084 100644 --- a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/ReportWriter.java +++ b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/ReportWriter.java @@ -3,6 +3,7 @@ * deegree-cli-utility * %% * Copyright (C) 2016 - 2021 lat/lon GmbH + * Copyright (C) 2022 grit graphische Informationstechnik Beratungsgesellschaft mbH * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as @@ -43,6 +44,7 @@ * Job listener to write final report. * * @author Lyn Goltz + * @author Stephan Reichhelm */ public class ReportWriter extends JobExecutionListenerSupport { @@ -72,6 +74,7 @@ public void afterJob( JobExecution jobExecution ) { ExitStatus exitStatus = stepExecution.getExitStatus(); try ( PrintWriter writer = new PrintWriter( outFile.toFile() ) ) { writer.println( "Start: " + getStartTime( stepExecution ) ); + writer.println( "End: " + getEndTime( stepExecution ) ); writer.println( "Time needed: " + getTimeNeeded( stepExecution ) ); if ( ExitStatus.FAILED.getExitCode().equals( exitStatus.getExitCode() ) ) { @@ -87,6 +90,12 @@ public void afterJob( JobExecution jobExecution ) { writer.println( "Status: FAILED" ); } else if ( ExitStatus.COMPLETED.getExitCode().equals( exitStatus.getExitCode() ) ) { writer.println( "Number of processed features: " + summary.getNumberOfFeatures() ); + if ( summary.getStatistics() != null ) { + writer.println( "Written feature statistics:" ); + writer.println( "==========================="); + summary.getStatistics().summary( writer::println ); + writer.println(); + } writer.println( "Status: SUCCESS" ); } else { writer.println( "Status: " + exitStatus ); @@ -128,5 +137,4 @@ private String getEndTime( StepExecution stepExecution ) { return DATE_FORMAT.format( stepExecution.getEndTime() ); return "UNKNOWN"; } - } \ No newline at end of file diff --git a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/Summary.java b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/Summary.java index ff0fc6dff7..861aeba5e5 100644 --- a/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/Summary.java +++ b/deegree-tools/deegree-tools-gml/src/main/java/org/deegree/tools/featurestoresql/loader/Summary.java @@ -3,6 +3,7 @@ * deegree-cli-utility * %% * Copyright (C) 2016 - 2021 lat/lon GmbH + * Copyright (C) 2022 grit graphische Informationstechnik Beratungsgesellschaft mbH * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as @@ -24,12 +25,13 @@ import java.util.HashSet; import java.util.Set; -import org.springframework.batch.core.StepExecution; +import javax.xml.namespace.QName; /** * Report summary. * * @author Lyn Goltz + * @author Stephan Reichhelm */ public class Summary { @@ -39,7 +41,7 @@ public class Summary { private Set unresolvableReferences = new HashSet<>(); - private StepExecution stepExecution; + private FeatureStatistics statistics = null; /** * @param increaseBy @@ -49,6 +51,17 @@ public void increaseNumberOfFeatures( int increaseBy ) { this.numberOfFeatures = this.numberOfFeatures + increaseBy; } + /** + * @param increaseBy + * integer to add (positive integer or null) + */ + public void increaseNumberOfFeatures( QName typeName ) { + if ( this.statistics != null ) { + statistics.increment( typeName ); + } + this.numberOfFeatures++; + } + /** * @return the number of features processed (positive integer or null) */ @@ -100,4 +113,18 @@ public String getCommitFailed() { return this.commitFailed; } + /** + * @return the statistics, may be null + */ + public FeatureStatistics getStatistics() { + return statistics; + } + + /** + * @param statistics + * the statistics to use inside this summary + */ + public void setStatistics( FeatureStatistics statistics ) { + this.statistics = statistics; + } } \ No newline at end of file