From f119c49c4d649e97015a442da26af1c0bcfe16b0 Mon Sep 17 00:00:00 2001 From: Rajmund Takacs Date: Mon, 26 Feb 2024 16:52:59 +0100 Subject: [PATCH 01/73] NIFI-12843: Fix incorrect read of parquet data, when record.count is inherited This closes #8452. Signed-off-by: Tamas Palfy --- .../parquet/record/ParquetRecordReader.java | 9 +- .../nifi/parquet/TestParquetReader.java | 90 ------------------- 2 files changed, 6 insertions(+), 93 deletions(-) diff --git a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/main/java/org/apache/nifi/parquet/record/ParquetRecordReader.java b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/main/java/org/apache/nifi/parquet/record/ParquetRecordReader.java index 380081a3b067..0ed4680c4b5d 100644 --- a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/main/java/org/apache/nifi/parquet/record/ParquetRecordReader.java +++ b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/main/java/org/apache/nifi/parquet/record/ParquetRecordReader.java @@ -61,10 +61,13 @@ public ParquetRecordReader( final Long offset = Optional.ofNullable(variables.get(ParquetAttribute.RECORD_OFFSET)) .map(Long::parseLong) .orElse(null); + final String recordCount = variables.get(ParquetAttribute.RECORD_COUNT); - recordsToRead = Optional.ofNullable(variables.get(ParquetAttribute.RECORD_COUNT)) - .map(Long::parseLong) - .orElse(null); + if (offset != null && recordCount != null) { + recordsToRead = Long.parseLong(recordCount); + } else { + recordsToRead = null; + } final long fileStartOffset = Optional.ofNullable(variables.get(ParquetAttribute.FILE_RANGE_START_OFFSET)) .map(Long::parseLong) diff --git a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/parquet/TestParquetReader.java b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/parquet/TestParquetReader.java index 1b50d131d0cf..2a22f75345a1 100644 --- a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/parquet/TestParquetReader.java +++ b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/parquet/TestParquetReader.java @@ -17,7 +17,6 @@ package org.apache.nifi.parquet; import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static java.util.stream.Collectors.toMap; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -79,30 +78,6 @@ public void testReadUsers() throws IOException, MalformedRecordException { .forEach(i -> assertEquals(ParquetTestUtils.createUser(i), convertRecordToUser(results.get(i)))); } - @Test - public void testReadUsersPartiallyWithLimitedRecordCount() throws IOException, MalformedRecordException { - final int numUsers = 25; - final int expectedRecords = 3; - final File parquetFile = ParquetTestUtils.createUsersParquetFile(numUsers); - final List results = getRecords(parquetFile, singletonMap(ParquetAttribute.RECORD_COUNT, "3")); - - assertEquals(expectedRecords, results.size()); - IntStream.range(0, expectedRecords) - .forEach(i -> assertEquals(ParquetTestUtils.createUser(i), convertRecordToUser(results.get(i)))); - } - - @Test - public void testReadUsersPartiallyWithOffset() throws IOException, MalformedRecordException { - final int numUsers = 1000025; // intentionally so large, to test input with many record groups - final int expectedRecords = 5; - final File parquetFile = ParquetTestUtils.createUsersParquetFile(numUsers); - final List results = getRecords(parquetFile, singletonMap(ParquetAttribute.RECORD_OFFSET, "1000020")); - - assertEquals(expectedRecords, results.size()); - IntStream.range(0, expectedRecords) - .forEach(i -> assertEquals(ParquetTestUtils.createUser(i + 1000020), convertRecordToUser(results.get(i)))); - } - @Test public void testReadUsersPartiallyWithOffsetAndLimitedRecordCount() throws IOException, MalformedRecordException { final int numUsers = 1000025; // intentionally so large, to test input with many record groups @@ -120,28 +95,6 @@ public void testReadUsersPartiallyWithOffsetAndLimitedRecordCount() throws IOExc .forEach(i -> assertEquals(ParquetTestUtils.createUser(i + 1000020), convertRecordToUser(results.get(i)))); } - @Test - public void testReadUsersPartiallyWithLimitedRecordCountWithinFileRange() - throws IOException, MalformedRecordException { - final int numUsers = 1000; - final int expectedRecords = 3; - final File parquetFile = ParquetTestUtils.createUsersParquetFile(numUsers); - final List results = getRecords( - parquetFile, - new HashMap() { - { - put(ParquetAttribute.RECORD_COUNT, "3"); - put(ParquetAttribute.FILE_RANGE_START_OFFSET, "16543"); - put(ParquetAttribute.FILE_RANGE_END_OFFSET, "24784"); - } - } - ); - - assertEquals(expectedRecords, results.size()); - IntStream.range(0, expectedRecords) - .forEach(i -> assertEquals(ParquetTestUtils.createUser(i + 663), convertRecordToUser(results.get(i)))); - } - @Test public void testReadUsersPartiallyWithOffsetWithinFileRange() throws IOException, MalformedRecordException { final int numUsers = 1000; @@ -213,25 +166,6 @@ public void testReader() throws InitializationException, IOException { "MapRecord[{name=Bob9, favorite_number=9, favorite_color=blue9}]"); } - @Test - public void testPartialReaderWithLimitedRecordCount() throws InitializationException, IOException { - final TestRunner runner = TestRunners.newTestRunner(TestParquetProcessor.class); - final ParquetReader parquetReader = new ParquetReader(); - - runner.addControllerService("reader", parquetReader); - runner.enableControllerService(parquetReader); - - runner.enqueue(Paths.get(PARQUET_PATH), singletonMap(ParquetAttribute.RECORD_COUNT, "2")); - - runner.setProperty(TestParquetProcessor.READER, "reader"); - - runner.run(); - runner.assertAllFlowFilesTransferred(TestParquetProcessor.SUCCESS, 1); - runner.getFlowFilesForRelationship(TestParquetProcessor.SUCCESS).get(0).assertContentEquals( - "MapRecord[{name=Bob0, favorite_number=0, favorite_color=blue0}]\n" + - "MapRecord[{name=Bob1, favorite_number=1, favorite_color=blue1}]"); - } - @Test public void testPartialReaderWithOffsetAndLimitedRecordCount() throws InitializationException, IOException { final TestRunner runner = TestRunners.newTestRunner(TestParquetProcessor.class); @@ -256,30 +190,6 @@ public void testPartialReaderWithOffsetAndLimitedRecordCount() throws Initializa "MapRecord[{name=Bob7, favorite_number=7, favorite_color=blue7}]"); } - @Test - public void testPartialReaderWithOffsetOnly() throws InitializationException, IOException { - final TestRunner runner = TestRunners.newTestRunner(TestParquetProcessor.class); - final ParquetReader parquetReader = new ParquetReader(); - - runner.addControllerService("reader", parquetReader); - runner.enableControllerService(parquetReader); - - runner.enqueue(Paths.get(PARQUET_PATH), singletonMap(ParquetAttribute.RECORD_OFFSET, "3")); - - runner.setProperty(TestParquetProcessor.READER, "reader"); - - runner.run(); - runner.assertAllFlowFilesTransferred(TestParquetProcessor.SUCCESS, 1); - runner.getFlowFilesForRelationship(TestParquetProcessor.SUCCESS).get(0).assertContentEquals( - "MapRecord[{name=Bob3, favorite_number=3, favorite_color=blue3}]\n" + - "MapRecord[{name=Bob4, favorite_number=4, favorite_color=blue4}]\n" + - "MapRecord[{name=Bob5, favorite_number=5, favorite_color=blue5}]\n" + - "MapRecord[{name=Bob6, favorite_number=6, favorite_color=blue6}]\n" + - "MapRecord[{name=Bob7, favorite_number=7, favorite_color=blue7}]\n" + - "MapRecord[{name=Bob8, favorite_number=8, favorite_color=blue8}]\n" + - "MapRecord[{name=Bob9, favorite_number=9, favorite_color=blue9}]"); - } - private List getRecords(File parquetFile, Map variables) throws IOException, MalformedRecordException { final List results = new ArrayList<>(); From c29a744644134bb122dbbddc3eb3d6ba3d98508a Mon Sep 17 00:00:00 2001 From: Mark Bathori Date: Tue, 27 Feb 2024 13:47:19 +0100 Subject: [PATCH 02/73] NIFI-12847: Add Enum data type handling to Iceberg record converter Signed-off-by: Pierre Villard This closes #8453. --- .../record/util/DataTypeUtils.java | 2 +- .../iceberg/converter/ArrayElementGetter.java | 4 +++ .../iceberg/converter/RecordFieldGetter.java | 4 +++ .../iceberg/TestIcebergRecordConverter.java | 6 +++- .../TestPutIcebergWithHiveCatalog.java | 2 +- .../src/test/resources/user.avsc | 29 ++++++++++++++----- 6 files changed, 36 insertions(+), 11 deletions(-) diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java index 000435e4100c..d45b5053c325 100644 --- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java +++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java @@ -1103,7 +1103,7 @@ public static boolean isEnumTypeCompatible(final Object value, final EnumDataTyp return enumType.getEnums() != null && enumType.getEnums().contains(value); } - private static Object toEnum(Object value, EnumDataType dataType, String fieldName) { + public static Object toEnum(Object value, EnumDataType dataType, String fieldName) { if(dataType.getEnums() != null && dataType.getEnums().contains(value)) { return value.toString(); } diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/ArrayElementGetter.java b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/ArrayElementGetter.java index f03c8f64a031..7e0be786e73c 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/ArrayElementGetter.java +++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/ArrayElementGetter.java @@ -22,6 +22,7 @@ import org.apache.nifi.serialization.record.field.StandardFieldConverterRegistry; import org.apache.nifi.serialization.record.type.ArrayDataType; import org.apache.nifi.serialization.record.type.ChoiceDataType; +import org.apache.nifi.serialization.record.type.EnumDataType; import org.apache.nifi.serialization.record.util.DataTypeUtils; import org.apache.nifi.serialization.record.util.IllegalTypeConversionException; @@ -96,6 +97,9 @@ public static ElementGetter createElementGetter(DataType dataType) { return converter.convertField(element, Optional.ofNullable(dataType.getFormat()), ARRAY_FIELD_NAME); }; break; + case ENUM: + elementGetter = element -> DataTypeUtils.toEnum(element, (EnumDataType) dataType, ARRAY_FIELD_NAME); + break; case UUID: elementGetter = DataTypeUtils::toUUID; break; diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/RecordFieldGetter.java b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/RecordFieldGetter.java index 2b7c7c09dad6..d0f9d55d87de 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/RecordFieldGetter.java +++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/converter/RecordFieldGetter.java @@ -23,6 +23,7 @@ import org.apache.nifi.serialization.record.field.StandardFieldConverterRegistry; import org.apache.nifi.serialization.record.type.ArrayDataType; import org.apache.nifi.serialization.record.type.ChoiceDataType; +import org.apache.nifi.serialization.record.type.EnumDataType; import org.apache.nifi.serialization.record.type.RecordDataType; import org.apache.nifi.serialization.record.util.DataTypeUtils; import org.apache.nifi.serialization.record.util.IllegalTypeConversionException; @@ -101,6 +102,9 @@ public static FieldGetter createFieldGetter(DataType dataType, String fieldName, case UUID: fieldGetter = record -> DataTypeUtils.toUUID(record.getValue(fieldName)); break; + case ENUM: + fieldGetter = record -> DataTypeUtils.toEnum(record.getValue(fieldName), (EnumDataType) dataType, fieldName); + break; case ARRAY: fieldGetter = record -> DataTypeUtils.toArray(record.getValue(fieldName), fieldName, ((ArrayDataType) dataType).getElementType()); break; diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestIcebergRecordConverter.java b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestIcebergRecordConverter.java index b067a9202dcc..48915ba7f2d4 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestIcebergRecordConverter.java +++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestIcebergRecordConverter.java @@ -166,7 +166,8 @@ public void tearDown() { Types.NestedField.optional(11, "timestamp", Types.TimestampType.withZone()), Types.NestedField.optional(12, "timestampTz", Types.TimestampType.withoutZone()), Types.NestedField.optional(13, "uuid", Types.UUIDType.get()), - Types.NestedField.optional(14, "choice", Types.IntegerType.get()) + Types.NestedField.optional(14, "choice", Types.IntegerType.get()), + Types.NestedField.optional(15, "enum", Types.StringType.get()) ); private static final Schema PRIMITIVES_SCHEMA_WITH_REQUIRED_FIELDS = new Schema( @@ -294,6 +295,7 @@ private static RecordSchema getPrimitivesSchema() { fields.add(new RecordField("timestampTz", RecordFieldType.TIMESTAMP.getDataType())); fields.add(new RecordField("uuid", RecordFieldType.UUID.getDataType())); fields.add(new RecordField("choice", RecordFieldType.CHOICE.getChoiceDataType(RecordFieldType.STRING.getDataType(), RecordFieldType.INT.getDataType()))); + fields.add(new RecordField("enum", RecordFieldType.ENUM.getEnumDataType(Arrays.asList("red", "blue", "yellow")))); return new SimpleRecordSchema(fields); } @@ -469,6 +471,7 @@ private static Record setupPrimitivesTestRecord() { values.put("timestampTz", Timestamp.valueOf(LOCAL_DATE_TIME)); values.put("uuid", UUID.fromString("0000-00-00-00-000000")); values.put("choice", "10"); + values.put("enum", "blue"); return new MapRecord(getPrimitivesSchema(), values); } @@ -590,6 +593,7 @@ public void testPrimitives(FileFormat format) throws IOException { assertEquals(offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC), resultRecord.get(11, OffsetDateTime.class)); assertEquals(LOCAL_DATE_TIME, resultRecord.get(12, LocalDateTime.class)); assertEquals(Integer.valueOf(10), resultRecord.get(14, Integer.class)); + assertEquals("blue", resultRecord.get(15, String.class)); if (format.equals(PARQUET)) { assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, resultRecord.get(13, byte[].class)); diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestPutIcebergWithHiveCatalog.java b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestPutIcebergWithHiveCatalog.java index c61925050ce5..d2b942cf58d7 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestPutIcebergWithHiveCatalog.java +++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestPutIcebergWithHiveCatalog.java @@ -109,7 +109,7 @@ private void initRecordReader() throws InitializationException { RecordSchema recordSchema = AvroTypeUtil.createSchema(inputSchema); for (RecordField recordField : recordSchema.getFields()) { - readerFactory.addSchemaField(recordField.getFieldName(), recordField.getDataType().getFieldType(), recordField.isNullable()); + readerFactory.addSchemaField(recordField); } readerFactory.addRecord(0, "John", "Finance"); diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/user.avsc b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/user.avsc index c537a9e4968f..799c0023b4a9 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/user.avsc +++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/user.avsc @@ -15,12 +15,25 @@ * limitations under the License. */ { - "namespace": "nifi", - "type": "record", - "name": "User", - "fields": [ - {"name": "id", "type": ["long", "null"]}, - {"name": "name", "type": ["string", "null"]}, - {"name": "department", "type": ["string", "null"]} - ] + "namespace": "nifi", + "type": "record", + "name": "User", + "fields": [ + { + "name": "id", + "type": ["long", "null"] + }, + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "department", + "type": { + "name": "Department", + "type": "enum", + "symbols": ["Finance", "Marketing", "Sales"] + } + } + ] } From ecea18f79655c0e34949d94609c8909aeb2d093e Mon Sep 17 00:00:00 2001 From: Juldrixx Date: Fri, 2 Feb 2024 17:44:37 +0100 Subject: [PATCH 03/73] NIFI-12733 Make Apicurio's groupId optional and configurable and use artifactId instead of name as key Signed-off-by: Pierre Villard This closes #8351. --- .../ApicurioSchemaRegistry.java | 24 ++++++---- .../client/ApicurioSchemaRegistryClient.java | 39 +++++----------- .../client/CachingSchemaRegistryClient.java | 21 ++------- .../client/SchemaRegistryApiClient.java | 37 ++++++--------- .../client/SchemaRegistryClient.java | 6 +-- .../schemaregistry/util/SchemaUtils.java | 44 +++++------------- .../ApicurioSchemaRegistryClientTest.java | 46 ++++++++----------- .../CachingSchemaRegistryClientTest.java | 26 ++++++----- .../client/SchemaRegistryApiClientTest.java | 36 ++++++--------- .../schemaregistry/util/SchemaUtilsTest.java | 23 +--------- .../src/test/resources/metadata_response.json | 15 ------ .../resources/schema_response_version_3.json | 15 ++++++ .../schema_response_version_latest.json | 15 ++++++ .../src/test/resources/search_response.json | 16 ------- 14 files changed, 135 insertions(+), 228 deletions(-) delete mode 100644 nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/metadata_response.json create mode 100644 nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_3.json create mode 100644 nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_latest.json delete mode 100644 nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/search_response.json diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/ApicurioSchemaRegistry.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/ApicurioSchemaRegistry.java index 63a68949ec59..d36b34c21b48 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/ApicurioSchemaRegistry.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/ApicurioSchemaRegistry.java @@ -26,6 +26,7 @@ import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.schema.access.SchemaField; import org.apache.nifi.schema.access.SchemaNotFoundException; @@ -43,7 +44,7 @@ @Tags({"schema", "registry", "apicurio", "avro"}) @CapabilityDescription("Provides a Schema Registry that interacts with the Apicurio Schema Registry so that those Schemas that are stored in the Apicurio Schema " - + "Registry can be used in NiFi. When a Schema is looked up by name by this registry, it will find a Schema in the Apicurio Schema Registry with their names.") + + "Registry can be used in NiFi. When a Schema is looked up by name by this registry, it will find a Schema in the Apicurio Schema Registry with their artifact identifiers.") public class ApicurioSchemaRegistry extends AbstractControllerService implements SchemaRegistry { private static final Set schemaFields = EnumSet.of(SchemaField.SCHEMA_NAME, SchemaField.SCHEMA_TEXT, @@ -56,6 +57,15 @@ public class ApicurioSchemaRegistry extends AbstractControllerService implements .addValidator(StandardValidators.URL_VALIDATOR) .required(true) .build(); + static final PropertyDescriptor SCHEMA_GROUP_ID = new PropertyDescriptor.Builder() + .name("Schema Group ID") + .displayName("Schema Group ID") + .description("The artifact Group ID for the schemas") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .defaultValue("default") + .required(true) + .build(); static final PropertyDescriptor CACHE_SIZE = new PropertyDescriptor.Builder() .name("Cache Size") .displayName("Cache Size") @@ -86,6 +96,7 @@ public class ApicurioSchemaRegistry extends AbstractControllerService implements private static final List PROPERTIES = List.of( SCHEMA_REGISTRY_URL, + SCHEMA_GROUP_ID, CACHE_SIZE, CACHE_EXPIRATION, WEB_CLIENT_PROVIDER @@ -102,29 +113,26 @@ protected List getSupportedPropertyDescriptors() { @OnEnabled public void onEnabled(final ConfigurationContext context) { final String schemaRegistryUrl = context.getProperty(SCHEMA_REGISTRY_URL).getValue(); + final String schemaGroupId = context.getProperty(SCHEMA_GROUP_ID).getValue(); final int cacheSize = context.getProperty(CACHE_SIZE).asInteger(); final long cacheExpiration = context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.NANOSECONDS); final WebClientServiceProvider webClientServiceProvider = context.getProperty(WEB_CLIENT_PROVIDER).asControllerService(WebClientServiceProvider.class); - final SchemaRegistryApiClient apiClient = new SchemaRegistryApiClient(webClientServiceProvider, schemaRegistryUrl); + final SchemaRegistryApiClient apiClient = new SchemaRegistryApiClient(webClientServiceProvider, schemaRegistryUrl, schemaGroupId); final SchemaRegistryClient schemaRegistryClient = new ApicurioSchemaRegistryClient(apiClient); client = new CachingSchemaRegistryClient(schemaRegistryClient, cacheSize, cacheExpiration); } @Override public RecordSchema retrieveSchema(SchemaIdentifier schemaIdentifier) throws IOException, SchemaNotFoundException { - final String schemaName = schemaIdentifier.getName().orElseThrow( + final String schemaId = schemaIdentifier.getName().orElseThrow( () -> new SchemaNotFoundException("Cannot retrieve schema because Schema Name is not present") ); final OptionalInt version = schemaIdentifier.getVersion(); - if (version.isPresent()) { - return client.getSchema(schemaName, version.getAsInt()); - } else { - return client.getSchema(schemaName); - } + return client.getSchema(schemaId, version); } @Override diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClient.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClient.java index 793113705a8e..e7c094715603 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClient.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClient.java @@ -17,13 +17,13 @@ package org.apache.nifi.apicurio.schemaregistry.client; import org.apache.nifi.apicurio.schemaregistry.util.SchemaUtils; -import org.apache.nifi.apicurio.schemaregistry.util.SchemaUtils.ResultAttributes; import org.apache.nifi.schema.access.SchemaNotFoundException; import org.apache.nifi.serialization.record.RecordSchema; import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.util.OptionalInt; public class ApicurioSchemaRegistryClient implements SchemaRegistryClient { private final SchemaRegistryApiClient apiClient; @@ -33,37 +33,20 @@ public ApicurioSchemaRegistryClient(SchemaRegistryApiClient apiClient) { } @Override - public RecordSchema getSchema(final String schemaName) throws IOException, SchemaNotFoundException { - final ResultAttributes attributes = getAttributesForSchemaName(schemaName); - final int version = getVersionAttributeFromMetadata(attributes); - return createRecordSchemaForAttributes(attributes, version); + public RecordSchema getSchema(final String schemaId, final OptionalInt version) throws IOException, SchemaNotFoundException { + return createRecordSchemaForAttributes( + schemaId, + version + ); } - @Override - public RecordSchema getSchema(final String schemaName, final int version) throws IOException, SchemaNotFoundException { - final ResultAttributes attributes = getAttributesForSchemaName(schemaName); - return createRecordSchemaForAttributes(attributes, version); - } - - private ResultAttributes getAttributesForSchemaName(String schemaName) throws IOException { - final URI searchUri = apiClient.buildSearchUri(schemaName); - try (final InputStream searchResultStream = apiClient.retrieveResponse(searchUri)) { - return SchemaUtils.getResultAttributes(searchResultStream); - } - } - - private int getVersionAttributeFromMetadata(final ResultAttributes attributes) throws IOException { - final URI metaDataUri = apiClient.buildMetaDataUri(attributes.groupId(), attributes.artifactId()); - try (final InputStream metadataResultStream = apiClient.retrieveResponse(metaDataUri)) { - return SchemaUtils.extractVersionAttributeFromStream(metadataResultStream); - } - } - - private RecordSchema createRecordSchemaForAttributes(ResultAttributes attributes, int version) throws IOException, SchemaNotFoundException { - final URI schemaUri = apiClient.buildSchemaVersionUri(attributes.groupId(), attributes.artifactId(), version); + private RecordSchema createRecordSchemaForAttributes(final String artifactId, final OptionalInt version) throws IOException, SchemaNotFoundException { + final URI schemaUri = version.isPresent() + ? apiClient.buildSchemaVersionUri(artifactId, version.getAsInt()) : + apiClient.buildSchemaArtifactUri(artifactId); try (final InputStream schemaResultStream = apiClient.retrieveResponse(schemaUri)) { - return SchemaUtils.createRecordSchema(schemaResultStream, attributes.name(), version); + return SchemaUtils.createRecordSchema(schemaResultStream, artifactId, version); } } } diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClient.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClient.java index cfb8aff1dd88..ba5d74cb1a47 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClient.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClient.java @@ -23,10 +23,11 @@ import org.apache.nifi.serialization.record.RecordSchema; import java.time.Duration; +import java.util.OptionalInt; public class CachingSchemaRegistryClient implements SchemaRegistryClient { private final SchemaRegistryClient client; - private final LoadingCache, RecordSchema> schemaCache; + private final LoadingCache, RecordSchema> schemaCache; public CachingSchemaRegistryClient(final SchemaRegistryClient toWrap, final int cacheSize, final long expirationNanos) { this.client = toWrap; @@ -34,24 +35,10 @@ public CachingSchemaRegistryClient(final SchemaRegistryClient toWrap, final int schemaCache = Caffeine.newBuilder() .maximumSize(cacheSize) .expireAfterWrite(Duration.ofNanos(expirationNanos)) - .build(key -> { - if (key.getRight() == -1) { - // If the version in the key is -1, fetch the schema by name only. - return client.getSchema(key.getLeft()); - } else { - // If a specific version is provided in the key, fetch the schema with that version. - return client.getSchema(key.getLeft(), key.getRight()); - } - }); + .build(key -> client.getSchema(key.getLeft(), key.getRight())); } - - @Override - public RecordSchema getSchema(final String schemaName) { - return schemaCache.get(Pair.of(schemaName, -1)); - } - @Override - public RecordSchema getSchema(final String schemaName, final int version) { + public RecordSchema getSchema(final String schemaName, final OptionalInt version) { return schemaCache.get(Pair.of(schemaName, version)); } } diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClient.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClient.java index 57488794a897..8d030e29d8e5 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClient.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClient.java @@ -26,10 +26,12 @@ public class SchemaRegistryApiClient { private final WebClientServiceProvider webClientServiceProvider; private final String baseUrl; + private final String groupId; - public SchemaRegistryApiClient(final WebClientServiceProvider webClientServiceProvider, final String baseUrl) { + public SchemaRegistryApiClient(final WebClientServiceProvider webClientServiceProvider, final String baseUrl, final String groupId) { this.webClientServiceProvider = webClientServiceProvider; this.baseUrl = baseUrl; + this.groupId = groupId; } public InputStream retrieveResponse(final URI uri) { @@ -51,37 +53,26 @@ public HttpUriBuilder buildBaseUri() { .addPathSegment("v2"); } - public URI buildSearchUri(final String schemaName) { + private HttpUriBuilder buildBaseSchemaUri() { return buildBaseUri() - .addPathSegment("search") - .addPathSegment("artifacts") - .addQueryParameter("name", schemaName) - .addQueryParameter("limit", "1") - .build(); + .addPathSegment("groups") + .addPathSegment(this.groupId); } - public URI buildMetaDataUri(final String groupId, final String artifactId) { - return buildGroupArtifactsUri(groupId, artifactId) - .addPathSegment("meta") - .build(); + private HttpUriBuilder buildBaseSchemaArtifactUri(final String artifactId) { + return buildBaseSchemaUri() + .addPathSegment("artifacts") + .addPathSegment(artifactId); } - public URI buildSchemaUri(final String groupId, final String artifactId) { - return buildGroupArtifactsUri(groupId, artifactId).build(); + public URI buildSchemaArtifactUri(final String artifactId) { + return buildBaseSchemaArtifactUri(artifactId).build(); } - public URI buildSchemaVersionUri(final String groupId, final String artifactId, final int version) { - return buildGroupArtifactsUri(groupId, artifactId) + public URI buildSchemaVersionUri(final String artifactId, final int version) { + return buildBaseSchemaArtifactUri(artifactId) .addPathSegment("versions") .addPathSegment(String.valueOf(version)) .build(); } - - private HttpUriBuilder buildGroupArtifactsUri(final String groupId, final String artifactId) { - return buildBaseUri() - .addPathSegment("groups") - .addPathSegment(groupId) - .addPathSegment("artifacts") - .addPathSegment(artifactId); - } } diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryClient.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryClient.java index ba21b507341c..39a6cf24a2ca 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryClient.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryClient.java @@ -20,12 +20,10 @@ import org.apache.nifi.serialization.record.RecordSchema; import java.io.IOException; +import java.util.OptionalInt; public interface SchemaRegistryClient { - RecordSchema getSchema(final String schemaName) throws IOException, SchemaNotFoundException; - - RecordSchema getSchema(final String schemaName, final int version) throws IOException, SchemaNotFoundException; - + RecordSchema getSchema(final String schemaId, final OptionalInt version) throws IOException, SchemaNotFoundException; } diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtils.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtils.java index 5202a9e14e5f..4f77104e6346 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtils.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/main/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtils.java @@ -29,7 +29,7 @@ import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; +import java.util.OptionalInt; public class SchemaUtils { @@ -39,50 +39,28 @@ public class SchemaUtils { private SchemaUtils() { } - public static RecordSchema createRecordSchema(final InputStream schemaStream, final String name, final int version) throws SchemaNotFoundException, IOException { + public static RecordSchema createRecordSchema(final InputStream schemaStream, final String artifactId, final OptionalInt version) throws SchemaNotFoundException, IOException { final JsonNode schemaNode = OBJECT_MAPPER.readTree(schemaStream); final String schemaText = schemaNode.toString(); try { final Schema avroSchema = new Schema.Parser().parse(schemaText); - final SchemaIdentifier schemaId = SchemaIdentifier.builder() - .name(name) - .version(version) - .build(); + final SchemaIdentifier.Builder schemaIdBuilder = SchemaIdentifier.builder() + .name(artifactId); + + if (version.isPresent()) { + schemaIdBuilder.version(version.getAsInt()); + } + + final SchemaIdentifier schemaId = schemaIdBuilder.build(); return AvroTypeUtil.createSchema(avroSchema, schemaText, schemaId); } catch (final SchemaParseException e) { final String errorMessage = String.format("Obtained Schema with name [%s] from Apicurio Schema Registry but the Schema Text " + - "that was returned is not a valid Avro Schema", name); + "that was returned is not a valid Avro Schema", artifactId); throw new SchemaNotFoundException(errorMessage, e); } } - public static int extractVersionAttributeFromStream(InputStream in) { - final JsonNode metadataNode; - try { - metadataNode = OBJECT_MAPPER.readTree(in); - } catch (IOException e) { - throw new UncheckedIOException("Failed to read version from HTTP response stream", e); - } - return Integer.parseInt(metadataNode.get("version").asText()); - } - - public static ResultAttributes getResultAttributes(InputStream in) { - final JsonNode jsonNode; - try { - jsonNode = OBJECT_MAPPER.readTree(in); - } catch (IOException e) { - throw new UncheckedIOException("Failed to read result attributes from HTTP response stream", e); - } - final JsonNode artifactNode = jsonNode.get("artifacts").get(0); - final String groupId = artifactNode.get("groupId").asText(); - final String artifactId = artifactNode.get("id").asText(); - final String name = artifactNode.get("name").asText(); - return new ResultAttributes(groupId, artifactId, name); - } - - public record ResultAttributes(String groupId, String artifactId, String name) { - } static ObjectMapper setObjectMapper() { return new ObjectMapper(); diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClientTest.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClientTest.java index 671297d23fa0..9cbbdedbf375 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClientTest.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/ApicurioSchemaRegistryClientTest.java @@ -19,7 +19,6 @@ import org.apache.commons.io.IOUtils; import org.apache.nifi.schema.access.SchemaNotFoundException; import org.apache.nifi.serialization.record.RecordSchema; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -29,6 +28,7 @@ import java.io.InputStream; import java.net.URI; import java.nio.charset.Charset; +import java.util.OptionalInt; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doReturn; @@ -39,55 +39,45 @@ class ApicurioSchemaRegistryClientTest { private static final String TEST_URL = "http://test.apicurio-schema-registry.com:8888"; - private static final String SCHEMA_NAME = "schema1"; + private static final String ARTIFACT_ID = "artifactId1"; private static final int VERSION = 3; - private static final String SEARCH_URL = TEST_URL + "/search"; - private static final String METADATA_URL = TEST_URL + "/meta"; + private static final String SCHEMA_ARTIFACT_URL = TEST_URL + "/schema/" + ARTIFACT_ID; private static final String SCHEMA_VERSION_URL = TEST_URL + "/schema/versions/" + VERSION; - private static final String GROUP_ID = "groupId1"; - private static final String ARTIFACT_ID = "artifactId1"; @Mock private SchemaRegistryApiClient apiClient; private ApicurioSchemaRegistryClient schemaRegistryClient; - @BeforeEach - void setup() { - doReturn(URI.create(SEARCH_URL)).when(apiClient).buildSearchUri(SCHEMA_NAME); - doReturn(URI.create(SCHEMA_VERSION_URL)).when(apiClient).buildSchemaVersionUri(GROUP_ID, ARTIFACT_ID, VERSION); - doReturn(getResource("search_response.json")).when(apiClient).retrieveResponse(URI.create(SEARCH_URL)); - doReturn(getResource("schema_response.json")).when(apiClient).retrieveResponse(URI.create(SCHEMA_VERSION_URL)); - } - @Test - void testGetSchemaWithNameInvokesApiClientAndReturnsRecordSchema() throws IOException, SchemaNotFoundException { - doReturn(URI.create(METADATA_URL)).when(apiClient).buildMetaDataUri(GROUP_ID, ARTIFACT_ID); - doReturn(getResource("metadata_response.json")).when(apiClient).retrieveResponse(URI.create(METADATA_URL)); + void testGetSchemaWithIdInvokesApiClientAndReturnsRecordSchema() throws IOException, SchemaNotFoundException { + doReturn(URI.create(SCHEMA_ARTIFACT_URL)).when(apiClient).buildSchemaArtifactUri(ARTIFACT_ID); + doReturn(getResource("schema_response_version_latest.json")).when(apiClient).retrieveResponse(URI.create(SCHEMA_ARTIFACT_URL)); schemaRegistryClient = new ApicurioSchemaRegistryClient(apiClient); - final RecordSchema schema = schemaRegistryClient.getSchema(SCHEMA_NAME); + final RecordSchema schema = schemaRegistryClient.getSchema(ARTIFACT_ID, OptionalInt.empty()); - verify(apiClient).buildSearchUri(SCHEMA_NAME); - verify(apiClient).buildMetaDataUri(GROUP_ID, ARTIFACT_ID); - verify(apiClient).buildSchemaVersionUri(GROUP_ID, ARTIFACT_ID, VERSION); + verify(apiClient).buildSchemaArtifactUri(ARTIFACT_ID); + verify(apiClient, never()).buildSchemaVersionUri(ARTIFACT_ID, VERSION); - final String expectedSchemaText = IOUtils.toString(getResource("schema_response.json"), Charset.defaultCharset()) + final String expectedSchemaText = IOUtils.toString(getResource("schema_response_version_latest.json"), Charset.defaultCharset()) .replace("\n", "") .replaceAll(" +", ""); assertEquals(expectedSchemaText, schema.getSchemaText().orElseThrow(() -> new AssertionError("Schema Text is empty"))); } @Test - void testGetSchemaWithNameAndVersionInvokesApiClientAndReturnsRecordSchema() throws IOException, SchemaNotFoundException { + void testGetSchemaWithIdAndVersionInvokesApiClientAndReturnsRecordSchema() throws IOException, SchemaNotFoundException { + doReturn(URI.create(SCHEMA_VERSION_URL)).when(apiClient).buildSchemaVersionUri(ARTIFACT_ID, VERSION); + doReturn(getResource("schema_response_version_3.json")).when(apiClient).retrieveResponse(URI.create(SCHEMA_VERSION_URL)); + schemaRegistryClient = new ApicurioSchemaRegistryClient(apiClient); - final RecordSchema schema = schemaRegistryClient.getSchema(SCHEMA_NAME, 3); + final RecordSchema schema = schemaRegistryClient.getSchema(ARTIFACT_ID, OptionalInt.of(VERSION)); - verify(apiClient).buildSearchUri(SCHEMA_NAME); - verify(apiClient, never()).buildMetaDataUri(GROUP_ID, ARTIFACT_ID); - verify(apiClient).buildSchemaVersionUri(GROUP_ID, ARTIFACT_ID, VERSION); + verify(apiClient, never()).buildSchemaArtifactUri(ARTIFACT_ID); + verify(apiClient).buildSchemaVersionUri(ARTIFACT_ID, VERSION); - final String expectedSchemaText = IOUtils.toString(getResource("schema_response.json"), Charset.defaultCharset()) + final String expectedSchemaText = IOUtils.toString(getResource("schema_response_version_3.json"), Charset.defaultCharset()) .replace("\n", "") .replaceAll(" +", ""); assertEquals(expectedSchemaText, schema.getSchemaText().orElseThrow(() -> new AssertionError("Schema Text is empty"))); diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClientTest.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClientTest.java index 24eda7a159c7..ef3f63dbe5e8 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClientTest.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/CachingSchemaRegistryClientTest.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.OptionalInt; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -60,35 +61,36 @@ public void setUp() { @Test void testGetSchemaWithNameInvokesClientAndCacheResult() throws IOException, SchemaNotFoundException { - when(mockClient.getSchema(SCHEMA_NAME)).thenReturn(TEST_SCHEMA); + final OptionalInt version = OptionalInt.empty(); - RecordSchema actualSchema1 = cachingClient.getSchema(SCHEMA_NAME); - RecordSchema actualSchema2 = cachingClient.getSchema(SCHEMA_NAME); + when(mockClient.getSchema(SCHEMA_NAME, version)).thenReturn(TEST_SCHEMA); + + RecordSchema actualSchema1 = cachingClient.getSchema(SCHEMA_NAME, version); + RecordSchema actualSchema2 = cachingClient.getSchema(SCHEMA_NAME, version); assertEquals(TEST_SCHEMA, actualSchema1); assertEquals(TEST_SCHEMA, actualSchema2); - verify(mockClient).getSchema(SCHEMA_NAME); + verify(mockClient).getSchema(SCHEMA_NAME, version); } @Test void testGetSchemaWithNameAndVersionInvokesClientAndCacheResult() throws IOException, SchemaNotFoundException { - String schemaName = "schema"; - int version = 1; + final OptionalInt version = OptionalInt.of(1); - when(mockClient.getSchema(schemaName, version)).thenReturn(TEST_SCHEMA); + when(mockClient.getSchema(SCHEMA_NAME, version)).thenReturn(TEST_SCHEMA); - RecordSchema actualSchema1 = cachingClient.getSchema(schemaName, version); - RecordSchema actualSchema2 = cachingClient.getSchema(schemaName, version); + RecordSchema actualSchema1 = cachingClient.getSchema(SCHEMA_NAME, version); + RecordSchema actualSchema2 = cachingClient.getSchema(SCHEMA_NAME, version); assertEquals(TEST_SCHEMA, actualSchema1); assertEquals(TEST_SCHEMA, actualSchema2); - verify(mockClient).getSchema(schemaName, version); + verify(mockClient).getSchema(SCHEMA_NAME, version); } @Test void testGetSchemaWithNameAndVersionDoesNotCacheDifferentVersions() throws IOException, SchemaNotFoundException { - int version1 = 1; - int version2 = 2; + final OptionalInt version1 = OptionalInt.of(1); + final OptionalInt version2 = OptionalInt.of(2); RecordSchema expectedSchema1 = TEST_SCHEMA; RecordSchema expectedSchema2 = TEST_SCHEMA_2; diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClientTest.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClientTest.java index 8551c6fb58a1..486075a7d04b 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClientTest.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/client/SchemaRegistryApiClientTest.java @@ -35,12 +35,12 @@ class SchemaRegistryApiClientTest { private static final String BASE_URL = "http://test.apicurio-schema-registry.com:8888"; private static final String API_PATH = "/apis/registry/v2"; - private static final String METADATA_PATH = "/meta"; private static final String GROUP_ID = "groupId1"; private static final String ARTIFACT_ID = "artifactId1"; - private static final String SCHEMA_PATH = String.format("/groups/%s/artifacts/%s", GROUP_ID, ARTIFACT_ID); - private static final String SCHEMA_NAME = "schema1"; - private static final String SEARCH_PATH = String.format("/search/artifacts?name=%s&limit=1", SCHEMA_NAME); + private static final int VERSION = 3; + private static final String GROUP_PATH = String.format("/groups/%s", GROUP_ID); + private static final String ARTIFACT_PATH = String.format("/artifacts/%s", ARTIFACT_ID); + private static final String VERSION_PATH = String.format("/versions/%d", VERSION); @Mock private WebClientServiceProvider webClientServiceProvider; @@ -53,7 +53,7 @@ void setup() { @Test void testBuildBaseUrl() { - client = new SchemaRegistryApiClient(webClientServiceProvider, BASE_URL); + client = new SchemaRegistryApiClient(webClientServiceProvider, BASE_URL, GROUP_ID); final HttpUriBuilder httpUriBuilder = client.buildBaseUri(); @@ -61,30 +61,20 @@ void testBuildBaseUrl() { } @Test - void testBuildSearchUri() { - client = new SchemaRegistryApiClient(webClientServiceProvider, BASE_URL); + void testBuildSchemaArtifactUri() { + client = new SchemaRegistryApiClient(webClientServiceProvider, BASE_URL, GROUP_ID); - final URI uri = client.buildSearchUri(SCHEMA_NAME); + final URI uri = client.buildSchemaArtifactUri(ARTIFACT_ID); - assertEquals(BASE_URL + API_PATH + SEARCH_PATH, uri.toString()); + assertEquals(BASE_URL + API_PATH + GROUP_PATH + ARTIFACT_PATH, uri.toString()); } @Test - void testBuildMetadataUri() { - client = new SchemaRegistryApiClient(webClientServiceProvider, BASE_URL); + void testBuildSchemaVersionUri() { + client = new SchemaRegistryApiClient(webClientServiceProvider, BASE_URL, GROUP_ID); - final URI uri = client.buildMetaDataUri(GROUP_ID, ARTIFACT_ID); + final URI uri = client.buildSchemaVersionUri(ARTIFACT_ID, VERSION); - assertEquals(BASE_URL + API_PATH + SCHEMA_PATH + METADATA_PATH, uri.toString()); + assertEquals(BASE_URL + API_PATH + GROUP_PATH + ARTIFACT_PATH + VERSION_PATH, uri.toString()); } - - @Test - void testBuildSchemaUri() { - client = new SchemaRegistryApiClient(webClientServiceProvider, BASE_URL); - - final URI uri = client.buildSchemaUri(GROUP_ID, ARTIFACT_ID); - - assertEquals(BASE_URL + API_PATH + SCHEMA_PATH, uri.toString()); - } - } diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtilsTest.java b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtilsTest.java index 7b745f6344c6..cded22eb7f81 100644 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtilsTest.java +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/java/org/apache/nifi/apicurio/schemaregistry/util/SchemaUtilsTest.java @@ -17,7 +17,6 @@ package org.apache.nifi.apicurio.schemaregistry.util; import org.apache.commons.io.IOUtils; -import org.apache.nifi.apicurio.schemaregistry.util.SchemaUtils.ResultAttributes; import org.apache.nifi.schema.access.SchemaNotFoundException; import org.apache.nifi.serialization.record.RecordSchema; import org.junit.jupiter.api.Test; @@ -25,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; +import java.util.OptionalInt; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -34,7 +34,7 @@ class SchemaUtilsTest { void testCreateRecordSchema() throws SchemaNotFoundException, IOException { final InputStream in = getResource("schema_response.json"); - final RecordSchema schema = SchemaUtils.createRecordSchema(in, "schema1", 3); + final RecordSchema schema = SchemaUtils.createRecordSchema(in, "schema1", OptionalInt.of(3)); assertEquals("schema1", schema.getSchemaName().orElseThrow(() -> new AssertionError("Schema Name is empty"))); assertEquals("schema_namespace_1", schema.getSchemaNamespace().orElseThrow(() -> new AssertionError("Schema Namespace is empty"))); @@ -46,25 +46,6 @@ void testCreateRecordSchema() throws SchemaNotFoundException, IOException { assertEquals(expectedSchemaText, schema.getSchemaText().orElseThrow(() -> new AssertionError("Schema Text is empty"))); } - @Test - void testGetVersionAttribute() { - final InputStream in = getResource("metadata_response.json"); - - int version = SchemaUtils.extractVersionAttributeFromStream(in); - - assertEquals(3, version); - } - - @Test - void testGetResultAttributes() { - final InputStream in = getResource("search_response.json"); - - final ResultAttributes resultAttributes = SchemaUtils.getResultAttributes(in); - - final ResultAttributes expectedAttributes = new ResultAttributes("groupId1", "artifactId1", "schema1"); - assertEquals(expectedAttributes, resultAttributes); - } - private InputStream getResource(final String resourceName) { return this.getClass().getClassLoader().getResourceAsStream(resourceName); } diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/metadata_response.json b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/metadata_response.json deleted file mode 100644 index 96459721caa0..000000000000 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/metadata_response.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "schema1", - "createdBy": "", - "createdOn": "2023-10-16T14:19:21+0000", - "modifiedBy": "", - "modifiedOn": "2023-10-16T14:53:12+0000", - "id": "artifactId1", - "version": "3", - "type": "AVRO", - "globalId": 3, - "state": "ENABLED", - "groupId": "groupId1", - "contentId": 2, - "references": [] -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_3.json b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_3.json new file mode 100644 index 000000000000..9adfffeaa23a --- /dev/null +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_3.json @@ -0,0 +1,15 @@ +{ + "type": "record", + "namespace": "schema_namespace", + "name": "schema_version_3", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "age", + "type": "int" + } + ] +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_latest.json b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_latest.json new file mode 100644 index 000000000000..87f1e60e8df6 --- /dev/null +++ b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/schema_response_version_latest.json @@ -0,0 +1,15 @@ +{ + "type": "record", + "namespace": "schema_namespace", + "name": "schema_version_latest", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "age", + "type": "int" + } + ] +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/search_response.json b/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/search_response.json deleted file mode 100644 index 5db50a7a15a3..000000000000 --- a/nifi-nar-bundles/nifi-apicurio-bundle/nifi-apicurio-schema-registry-service/src/test/resources/search_response.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "artifacts": [ - { - "id": "artifactId1", - "name": "schema1", - "createdOn": "2023-10-16T14:19:21+0000", - "createdBy": "", - "type": "AVRO", - "state": "ENABLED", - "modifiedOn": "2023-10-16T14:19:31+0000", - "modifiedBy": "", - "groupId": "groupId1" - } - ], - "count": 1 -} \ No newline at end of file From 01ca19eccc9a711315797f30ece5aec67cda9a2e Mon Sep 17 00:00:00 2001 From: Lucas <16666115+EndzeitBegins@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:55:45 +0100 Subject: [PATCH 04/73] NIFI-12498 Adjust docs on default behaviour of prioritizers (#8451) --- nifi-docs/src/main/asciidoc/user-guide.adoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc index b8c3129a859b..cee1d3b59203 100644 --- a/nifi-docs/src/main/asciidoc/user-guide.adoc +++ b/nifi-docs/src/main/asciidoc/user-guide.adoc @@ -1432,13 +1432,16 @@ The following prioritizers are available: - *FirstInFirstOutPrioritizer*: Given two FlowFiles, the one that reached the connection first will be processed first. - *NewestFlowFileFirstPrioritizer*: Given two FlowFiles, the one that is newest in the dataflow will be processed first. -- *OldestFlowFileFirstPrioritizer*: Given two FlowFiles, the one that is oldest in the dataflow will be processed first. 'This is the default scheme that is used if no prioritizers are selected'. +- *OldestFlowFileFirstPrioritizer*: Given two FlowFiles, the one that is oldest in the dataflow will be processed first. - *PriorityAttributePrioritizer*: Given two FlowFiles, an attribute called “priority” will be extracted. The one that has the lowest priority value will be processed first. ** Note that an UpdateAttribute processor should be used to add the "priority" attribute to the FlowFiles before they reach a connection that has this prioritizer set. ** If only one has that attribute it will go first. ** Values for the "priority" attribute can be alphanumeric, where "a" will come before "z" and "1" before "9" ** If "priority" attribute cannot be parsed as a long, unicode string ordering will be used. For example: "99" and "100" will be ordered so the FlowFile with "99" comes first, but "A-99" and "A-100" will sort so the FlowFile with "A-100" comes first. +In case no prioritizers are selected or the selected prioritizers do not differentiate between the compared FlowFiles, +processing order is not further specified but an implementation detail that might change between versions of NiFi. + NOTE: With a <> configured, the connection has a queue per node in addition to the local queue. The prioritizer will sort the data in each queue independently. ==== Changing Configuration and Context Menu Options From 74bd798097e15d54b871ac3ef7654a0d3433f99a Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Fri, 9 Feb 2024 16:58:09 -0500 Subject: [PATCH 05/73] NIFI-12773: Added join and anchored RecordPath function Signed-off-by: Chris Sampson This closes #8391 --- nifi-commons/nifi-record-path/pom.xml | 4 + .../nifi/record/path/functions/Anchored.java | 83 ++++++++++++++++++ .../nifi/record/path/functions/Join.java | 84 +++++++++++++++++++ .../record/path/paths/RecordPathCompiler.java | 24 ++++++ .../nifi/record/path/TestRecordPath.java | 83 ++++++++++++++++++ .../src/main/asciidoc/record-path-guide.adoc | 42 ++++++++++ 6 files changed, 320 insertions(+) create mode 100644 nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Anchored.java create mode 100644 nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Join.java diff --git a/nifi-commons/nifi-record-path/pom.xml b/nifi-commons/nifi-record-path/pom.xml index 17d9e762e201..ead593722791 100644 --- a/nifi-commons/nifi-record-path/pom.xml +++ b/nifi-commons/nifi-record-path/pom.xml @@ -103,6 +103,10 @@ nifi-uuid5 2.0.0-SNAPSHOT + + org.apache.nifi + nifi-property-utils + org.antlr antlr-runtime diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Anchored.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Anchored.java new file mode 100644 index 000000000000..2874295d79fc --- /dev/null +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Anchored.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.record.path.functions; + +import org.apache.nifi.record.path.FieldValue; +import org.apache.nifi.record.path.RecordPathEvaluationContext; +import org.apache.nifi.record.path.StandardRecordPathEvaluationContext; +import org.apache.nifi.record.path.paths.RecordPathSegment; +import org.apache.nifi.serialization.record.Record; + +import java.util.Arrays; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public class Anchored extends RecordPathSegment { + + private final RecordPathSegment anchorPath; + private final RecordPathSegment evaluationPath; + + public Anchored(final RecordPathSegment anchorPath, final RecordPathSegment evaluationPath, final boolean absolute) { + super("anchored", null, absolute); + + this.anchorPath = anchorPath; + this.evaluationPath = evaluationPath; + } + + + @Override + public Stream evaluate(final RecordPathEvaluationContext context) { + final Stream anchoredStream = anchorPath.evaluate(context); + + return anchoredStream.flatMap(fv -> { + final Object value = fv.getValue(); + return evaluateFieldValue(value); + }); + } + + private Stream evaluateFieldValue(final Object value) { + if (value == null) { + return Stream.of(); + } + + if (value instanceof Record) { + return evaluateAtRoot((Record) value); + } + + if (value instanceof final Record[] array) { + return Arrays.stream(array).flatMap(this::evaluateAtRoot); + } + + if (value instanceof final Iterable iterable) { + return StreamSupport.stream(iterable.spliterator(), false).flatMap(element -> { + if (!(element instanceof Record)) { + return Stream.of(); + } + + return evaluateAtRoot((Record) element); + }); + } + + return Stream.of(); + } + + private Stream evaluateAtRoot(final Record root) { + final RecordPathEvaluationContext recordPathEvaluateContext = new StandardRecordPathEvaluationContext(root); + return evaluationPath.evaluate(recordPathEvaluateContext); + } +} diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Join.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Join.java new file mode 100644 index 000000000000..da9cee488fac --- /dev/null +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Join.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.record.path.functions; + +import org.apache.nifi.record.path.FieldValue; +import org.apache.nifi.record.path.RecordPathEvaluationContext; +import org.apache.nifi.record.path.StandardFieldValue; +import org.apache.nifi.record.path.paths.RecordPathSegment; +import org.apache.nifi.record.path.util.RecordPathUtils; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +public class Join extends RecordPathSegment { + private final RecordPathSegment delimiterPath; + private final RecordPathSegment[] valuePaths; + + public Join(final RecordPathSegment delimiterPath, final RecordPathSegment[] valuePaths, final boolean absolute) { + super("join", null, absolute); + this.delimiterPath = delimiterPath; + this.valuePaths = valuePaths; + } + + @Override + public Stream evaluate(final RecordPathEvaluationContext context) { + String delimiter = RecordPathUtils.getFirstStringValue(delimiterPath, context); + if (delimiter == null) { + delimiter = ""; + } + + final List values = new ArrayList<>(); + for (final RecordPathSegment valuePath : valuePaths) { + final Stream stream = valuePath.evaluate(context); + + stream.forEach(fv -> { + final Object value = fv.getValue(); + addStringValue(value, values); + }); + } + + final String joined = String.join(delimiter, values); + final RecordField field = new RecordField("join", RecordFieldType.STRING.getDataType()); + final FieldValue responseValue = new StandardFieldValue(joined, field, null); + return Stream.of(responseValue); + } + + private void addStringValue(final Object value, final List values) { + if (value == null) { + values.add("null"); + return; + } + + if (value instanceof final Object[] array) { + for (final Object element : array) { + addStringValue(element, values); + } + } else if (value instanceof final Iterable iterable) { + for (final Object element : iterable) { + addStringValue(element, values); + } + } else { + values.add(DataTypeUtils.toString(value, null)); + } + } +} diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java index c6f23b145ba1..906101429568 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java @@ -35,6 +35,7 @@ import org.apache.nifi.record.path.filter.NotFilter; import org.apache.nifi.record.path.filter.RecordPathFilter; import org.apache.nifi.record.path.filter.StartsWith; +import org.apache.nifi.record.path.functions.Anchored; import org.apache.nifi.record.path.functions.Base64Decode; import org.apache.nifi.record.path.functions.Base64Encode; import org.apache.nifi.record.path.functions.Coalesce; @@ -45,6 +46,7 @@ import org.apache.nifi.record.path.functions.FilterFunction; import org.apache.nifi.record.path.functions.Format; import org.apache.nifi.record.path.functions.Hash; +import org.apache.nifi.record.path.functions.Join; import org.apache.nifi.record.path.functions.MapOf; import org.apache.nifi.record.path.functions.PadLeft; import org.apache.nifi.record.path.functions.PadRight; @@ -129,6 +131,10 @@ public static RecordPathSegment buildPath(final Tree tree, final RecordPathSegme } case CHILD_REFERENCE: { final Tree childTree = tree.getChild(0); + if (childTree == null) { + return new RootPath(); + } + final int childTreeType = childTree.getType(); if (childTreeType == FIELD_NAME) { final String childName = childTree.getChild(0).getText(); @@ -404,6 +410,24 @@ public static RecordPathSegment buildPath(final Tree tree, final RecordPathSegme final RecordPathSegment[] args = getArgPaths(argumentListTree, 1, functionName, absolute); return new Count(args[0], absolute); } + case "join": { + final int numArgs = argumentListTree.getChildCount(); + if (numArgs < 2) { + throw new RecordPathException("Invalid number of arguments: " + functionName + " function takes 2 or more arguments but got " + numArgs); + } + + final RecordPathSegment[] joinPaths = new RecordPathSegment[numArgs - 1]; + for (int i = 0; i < numArgs - 1; i++) { + joinPaths[i] = buildPath(argumentListTree.getChild(i + 1), null, absolute); + } + + final RecordPathSegment delimiterPath = buildPath(argumentListTree.getChild(0), null, absolute); + return new Join(delimiterPath, joinPaths, absolute); + } + case "anchored": { + final RecordPathSegment[] args = getArgPaths(argumentListTree, 2, functionName, absolute); + return new Anchored(args[0], args[1], absolute); + } case "not": case "contains": case "containsRegex": diff --git a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java index 2b6341124c39..f374325528ca 100644 --- a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java +++ b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java @@ -1241,6 +1241,89 @@ public void testConcat() { assertEquals("John Doe: 48", RecordPath.compile("concat(/firstName, ' ', /lastName, ': ', 48)").evaluate(record).getSelectedFields().findFirst().get().getValue()); } + @Test + public void testJoinWithTwoFields() { + final List fields = new ArrayList<>(); + fields.add(new RecordField("fullName", RecordFieldType.INT.getDataType())); + fields.add(new RecordField("lastName", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("firstName", RecordFieldType.LONG.getDataType())); + + final RecordSchema schema = new SimpleRecordSchema(fields); + + final Map values = new HashMap<>(); + values.put("lastName", "Doe"); + values.put("firstName", "John"); + final Record record = new MapRecord(schema, values); + + assertEquals("Doe, John", RecordPath.compile("join(', ', /lastName, /firstName)").evaluate(record).getSelectedFields().findFirst().get().getValue()); + } + + @Test + public void testJoinWithArray() { + final List fields = new ArrayList<>(); + fields.add(new RecordField("names", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()))); + final RecordSchema schema = new SimpleRecordSchema(fields); + + final Map values = new HashMap<>(); + values.put("names", new String[] {"John", "Jane", "Jacob", "Judy"}); + final Record record = new MapRecord(schema, values); + + assertEquals("John,Jane,Jacob,Judy", RecordPath.compile("join(',', /names)").evaluate(record).getSelectedFields().findFirst().get().getValue()); + } + + @Test + public void testJoinWithArrayAndMultipleFields() { + final List personFields = new ArrayList<>(); + personFields.add(new RecordField("lastName", RecordFieldType.STRING.getDataType())); + personFields.add(new RecordField("firstName", RecordFieldType.STRING.getDataType())); + personFields.add(new RecordField("friends", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()))); + final RecordSchema personSchema = new SimpleRecordSchema(personFields); + + final Map values = new HashMap<>(); + values.put("friends", new String[] {"John", "Jane", "Jacob", "Judy"}); + values.put("firstName", "John"); + values.put("lastName", "Doe"); + final Record record = new MapRecord(personSchema, values); + + assertEquals("Doe\nJohn\nJane\nJacob", RecordPath.compile("join('\\n', /lastName, /firstName, /friends[1..2])").evaluate(record).getSelectedFields().findFirst().get().getValue()); + } + + @Test + public void testAnchored() { + final List personFields = new ArrayList<>(); + personFields.add(new RecordField("lastName", RecordFieldType.STRING.getDataType())); + personFields.add(new RecordField("firstName", RecordFieldType.STRING.getDataType())); + final RecordSchema personSchema = new SimpleRecordSchema(personFields); + + final List employeeFields = new ArrayList<>(); + employeeFields.add(new RecordField("self", RecordFieldType.RECORD.getRecordDataType(personSchema))); + employeeFields.add(new RecordField("manager", RecordFieldType.RECORD.getRecordDataType(personSchema))); + employeeFields.add(new RecordField("directReports", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(personSchema)))); + final RecordSchema employeeSchema = new SimpleRecordSchema(employeeFields); + + final Record directReport1 = createPerson("John", "Doe", personSchema); + final Record directReport2 = createPerson("John", "Jingleheimer", personSchema); + final Record directReport3 = createPerson("John", "Jacob", personSchema); + final Record manager = createPerson("Jane", "Smith", personSchema); + final Record employee = new MapRecord(employeeSchema, Map.of( + "self", createPerson("John", "Schmidt", personSchema), + "manager", manager, + "directReports", new Record[] {directReport1, directReport2, directReport3} + )); + + assertEquals("John", RecordPath.compile("anchored(/directReports[0], /firstName)").evaluate(employee).getSelectedFields().findFirst().get().getValue()); + assertEquals(List.of("John", "John", "John"), RecordPath.compile("anchored(/directReports, /firstName)").evaluate(employee).getSelectedFields().map(FieldValue::getValue).toList()); + assertEquals(List.of(), RecordPath.compile("anchored(/self/lastName, / )").evaluate(employee).getSelectedFields().map(FieldValue::getValue).toList()); + } + + private Record createPerson(final String firstName, final String lastName, final RecordSchema schema) { + final Map values = Map.of( + "firstName", firstName, + "lastName", lastName); + return new MapRecord(schema, values); + } + + @Test public void testMapOf() { final List fields = new ArrayList<>(); diff --git a/nifi-docs/src/main/asciidoc/record-path-guide.adoc b/nifi-docs/src/main/asciidoc/record-path-guide.adoc index a4d0323c4cb3..fae0538d646f 100644 --- a/nifi-docs/src/main/asciidoc/record-path-guide.adoc +++ b/nifi-docs/src/main/asciidoc/record-path-guide.adoc @@ -456,6 +456,48 @@ Concatenates all the arguments together. |========================================================== +=== join + +Joins together multiple values with a separator. + +|========================================================== +| RecordPath | Return value +| `join(', ', /workAddress/* )` | 123, 5th Avenue, New York, NY, 10020 +|========================================================== + + +=== anchored + +Allows evaluating a RecordPath while anchoring the root context to a child record. + +|========================================================== +| RecordPath | Return value +| `anchored(/homeAddress, /city)` | Jersey City +|========================================================== + +Additionally, this can be used in conjunction with arrays. For example, if we have the following record: +---- +{ + "id": "1234", + "elements": [{ + "name": "book", + "color": "red" + }, { + "name": "computer", + "color": "black" + }] +} +---- + +We can evaluate hte following Record paths: + +|========================================================== +| RecordPath | Return value +| `anchored(/elements, /name)` | The array containing `book` and `computer` +| `anchored(/elements, concat(/name, ': ', /color))` | The array containing `book: red` and `computer: black` +|========================================================== + + === fieldName Normally, when a path is given to a particular field in a Record, what is returned is the value of that field. It From 1cb0a537118a413622e3ce7b2485820c4910a04c Mon Sep 17 00:00:00 2001 From: Rob Fellows Date: Wed, 28 Feb 2024 16:17:33 -0500 Subject: [PATCH 06/73] [NIFI-12848] - fixed ExpressionChanged error in Status History dialog (#8455) * color the legend text to match the color of the corresponding line in the chart for each node This closes #8455 --- .../src/app/ui/common/status-history/index.ts | 7 +++ .../status-history-chart.component.ts | 14 +++++- .../status-history.component.html | 43 ++++++++++++------- .../status-history.component.ts | 31 ++++++++++--- 4 files changed, 73 insertions(+), 22 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/index.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/index.ts index c8f629b3fc8a..8c7bcb223e0b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/index.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/index.ts @@ -30,8 +30,15 @@ export const NIFI_NODE_CONFIG = { nifiInstanceLabel: 'NiFi' }; +export interface StatsNode { + id: string; + label: string; + color: string; +} + export interface Stats { min: string; max: string; mean: string; + nodes?: StatsNode[]; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history-chart/status-history-chart.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history-chart/status-history-chart.component.ts index 70af55881743..a46444eeb7a2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history-chart/status-history-chart.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history-chart/status-history-chart.component.ts @@ -462,7 +462,12 @@ export class StatusHistoryChart { this.nodeStats$.next({ min: nodeMinValue, max: nodeMaxValue, - mean: nodeMeanValue + mean: nodeMeanValue, + nodes: nodes.map((n) => ({ + id: n.id, + label: n.label, + color: color(n.label) + })) }); // only consider the cluster with data in the brush @@ -482,7 +487,12 @@ export class StatusHistoryChart { this.clusterStats$.next({ min: clusterMinValue, max: clusterMaxValue, - mean: clusterMeanValue + mean: clusterMeanValue, + nodes: cluster.map((n) => ({ + id: n.id, + label: n.label, + color: color(n.label) + })) }); }; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html index ab7e1f68932b..a8595386a331 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html @@ -26,15 +26,15 @@

Status History

} @else { - @if (instances.length > 0) { + @if (instances.length > 0 && fieldDescriptors.length > 0) { @if (componentDetails$ | async; as componentDetails) {
- @for (entry of Object.entries(componentDetails); track entry) { - @if (entry[0] && entry[1]) { + @for (detail of details; track detail) { + @if (detail.key && detail.value) {
-
{{ entry[0] }}
-
{{ entry[1] }}
+
{{ detail.key }}
+
{{ detail.value }}
} } @@ -63,7 +63,12 @@

Status History

[checked]=" !!instanceVisibility['nifi-instance-id'] "> - NiFi + NiFi
@if (!!nodes && nodes.length > 0) { @@ -76,17 +81,25 @@

Status History

{{ nodeStats.mean }} -
@for (node of nodes; track node) { - - {{ node.label }} + @if (node.snapshots?.length) { +
+ + {{ node.label }} +
+ } }
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.ts index e0aa42fceeb0..e5aea4ced268 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.ts @@ -18,7 +18,7 @@ import { AfterViewInit, Component, DestroyRef, inject, Inject, OnInit } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogModule } from '@angular/material/dialog'; import { StatusHistoryService } from '../../../service/status-history.service'; -import { AsyncPipe } from '@angular/common'; +import { AsyncPipe, NgStyle } from '@angular/common'; import { MatButtonModule } from '@angular/material/button'; import { FieldDescriptor, @@ -45,7 +45,7 @@ import * as d3 from 'd3'; import { NiFiCommon } from '../../../service/nifi-common.service'; import { TextTip } from '../tooltips/text-tip/text-tip.component'; import { NifiTooltipDirective } from '../tooltips/nifi-tooltip.directive'; -import { TextTipInput } from '../../../state/shared'; +import { isDefinedAndNotNull, TextTipInput } from '../../../state/shared'; import { MatCheckboxChange, MatCheckboxModule } from '@angular/material/checkbox'; import { Resizable } from '../resizable/resizable.component'; import { Instance, NIFI_NODE_CONFIG, Stats } from './index'; @@ -67,7 +67,8 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; NifiTooltipDirective, MatCheckboxModule, Resizable, - StatusHistoryChart + StatusHistoryChart, + NgStyle ], styleUrls: ['./status-history.component.scss'] }) @@ -79,6 +80,8 @@ export class StatusHistory implements OnInit, AfterViewInit { fieldDescriptors$ = this.store.select(selectStatusHistoryFieldDescriptors); fieldDescriptors: FieldDescriptor[] = []; + details: { key: string; value: string }[] = []; + minDate = ''; maxDate = ''; statusHistoryForm: FormGroup; @@ -86,12 +89,14 @@ export class StatusHistory implements OnInit, AfterViewInit { nodeStats: Stats = { max: 'NA', min: 'NA', - mean: 'NA' + mean: 'NA', + nodes: [] }; clusterStats: Stats = { max: 'NA', min: 'NA', - mean: 'NA' + mean: 'NA', + nodes: [] }; nodes: any[] = []; @@ -186,6 +191,10 @@ export class StatusHistory implements OnInit, AfterViewInit { this.statusHistoryForm.get('fieldDescriptor')?.setValue(descriptors[0]); this.selectedDescriptor = descriptors[0]; }); + + this.componentDetails$.pipe(isDefinedAndNotNull(), take(1)).subscribe((details) => { + this.details = Object.entries(details).map((entry) => ({ key: entry[0], value: entry[1] })); + }); } ngAfterViewInit(): void { @@ -254,4 +263,16 @@ export class StatusHistory implements OnInit, AfterViewInit { this.selectedDescriptor = { ...this.selectedDescriptor }; } } + + getColor(stats: Stats, nodeId: string): string { + if (stats.nodes && stats.nodes.length > 0) { + const nodeColor = stats.nodes?.find((c) => c.id === nodeId); + if (nodeColor) { + return nodeColor.color; + } + } + return 'unset'; + } + + protected readonly NIFI_NODE_CONFIG = NIFI_NODE_CONFIG; } From b368c281e8e6c25e040b083db6ebaaf95ec04c9a Mon Sep 17 00:00:00 2001 From: Paul Grey Date: Wed, 28 Feb 2024 16:39:10 -0500 Subject: [PATCH 07/73] NIFI-12851 - ConsumeKafka, remove limitation on count of subscribed topics Signed-off-by: Pierre Villard This closes #8460. --- .../kafka/pubsub/ConsumeKafkaRecord_2_6.java | 2 +- .../kafka/pubsub/ConsumeKafka_2_6.java | 2 +- .../kafka/pubsub/TestConsumeKafka_2_6.java | 38 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafkaRecord_2_6.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafkaRecord_2_6.java index 782d43de2445..5e263f059950 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafkaRecord_2_6.java +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafkaRecord_2_6.java @@ -451,7 +451,7 @@ protected ConsumerPool createConsumerPool(final ProcessContext context, final Co } if (topicType.equals(TOPIC_NAME.getValue())) { - for (final String topic : topicListing.split(",", 100)) { + for (final String topic : topicListing.split(",")) { final String trimmedName = topic.trim(); if (!trimmedName.isEmpty()) { topics.add(trimmedName); diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafka_2_6.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafka_2_6.java index 043e81a31356..784121e89eb0 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafka_2_6.java +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumeKafka_2_6.java @@ -397,7 +397,7 @@ protected ConsumerPool createConsumerPool(final ProcessContext context, final Co } if (topicType.equals(TOPIC_NAME.getValue())) { - for (final String topic : topicListing.split(",", 100)) { + for (final String topic : topicListing.split(",")) { final String trimmedName = topic.trim(); if (!trimmedName.isEmpty()) { topics.add(trimmedName); diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestConsumeKafka_2_6.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestConsumeKafka_2_6.java index 872df85cb901..42ab9d82ad25 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestConsumeKafka_2_6.java +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/TestConsumeKafka_2_6.java @@ -22,12 +22,20 @@ import org.apache.nifi.kafka.shared.property.SecurityProtocol; import org.apache.nifi.kerberos.KerberosUserService; import org.apache.nifi.kerberos.SelfContainedKerberosUserService; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -44,6 +52,36 @@ public void setup() { mockConsumerPool = mock(ConsumerPool.class); } + @Test + public void validateNoLimitToTopicCount() { + final int expectedCount = 101; + final String topics = String.join(",", Collections.nCopies(expectedCount, "foo")); + final ConsumeKafka_2_6 consumeKafka = new ConsumeKafka_2_6() { + protected ConsumerPool createConsumerPool(final ProcessContext context, final ComponentLog log) { + final ConsumerPool consumerPool = super.createConsumerPool(context, log); + try { + final Field topicsField = ConsumerPool.class.getDeclaredField("topics"); + topicsField.setAccessible(true); + final Object o = topicsField.get(consumerPool); + final List list = assertInstanceOf(List.class, o); + assertEquals(expectedCount, list.size()); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + return consumerPool; + } + }; + + TestRunner runner = TestRunners.newTestRunner(consumeKafka); + runner.setValidateExpressionUsage(false); + runner.setProperty(ConsumeKafka_2_6.BOOTSTRAP_SERVERS, "localhost:1234"); + runner.setProperty(ConsumeKafka_2_6.TOPICS, topics); + runner.setProperty(ConsumeKafka_2_6.GROUP_ID, "foo"); + runner.setProperty(ConsumeKafka_2_6.AUTO_OFFSET_RESET, ConsumeKafka_2_6.OFFSET_EARLIEST); + runner.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + runner.run(); + } + @Test public void validateCustomValidatorSettings() { ConsumeKafka_2_6 consumeKafka = new ConsumeKafka_2_6(); From 8346bd7d0f617f73660cc79a02798ddbe484606d Mon Sep 17 00:00:00 2001 From: ravisingh Date: Wed, 28 Feb 2024 19:43:07 -0800 Subject: [PATCH 08/73] NIFI-12828: Added Mapping for BIT type to return INT and handled boolean case for postgres Signed-off-by: Matt Burgess This closes #8445 --- .../nifi/serialization/record/util/DataTypeUtils.java | 2 ++ .../nifi/serialization/record/TestDataTypeUtils.java | 1 + .../db/schemaregistry/DatabaseTableSchemaRegistry.java | 8 +++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java index d45b5053c325..cd7fe2da8243 100644 --- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java +++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java @@ -1917,6 +1917,8 @@ public static DataType getDataTypeFromSQLTypeValue(final int sqlType) { switch (sqlType) { case Types.BIGINT: return RecordFieldType.BIGINT.getDataType(); + case Types.BIT: + return RecordFieldType.INT.getDataType(); case Types.BOOLEAN: return RecordFieldType.BOOLEAN.getDataType(); case Types.TINYINT: diff --git a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java index 6aca3af079d3..72f3a315bffb 100644 --- a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java +++ b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java @@ -558,6 +558,7 @@ public void testGetSQLTypeValueWithBigDecimal() { @Test public void testGetDataTypeFromSQLTypeValue() { assertEquals(RecordFieldType.STRING.getDataType(), DataTypeUtils.getDataTypeFromSQLTypeValue(Types.CLOB)); + assertEquals(RecordFieldType.INT.getDataType(), DataTypeUtils.getDataTypeFromSQLTypeValue(Types.BIT)); assertEquals(RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.BYTE.getDataType()), DataTypeUtils.getDataTypeFromSQLTypeValue(Types.BLOB)); assertEquals(RecordFieldType.STRING.getDataType(), DataTypeUtils.getDataTypeFromSQLTypeValue(Types.CHAR)); } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-db-schema-registry-bundle/nifi-db-schema-registry-service/src/main/java/org/apache/nifi/db/schemaregistry/DatabaseTableSchemaRegistry.java b/nifi-nar-bundles/nifi-standard-services/nifi-db-schema-registry-bundle/nifi-db-schema-registry-service/src/main/java/org/apache/nifi/db/schemaregistry/DatabaseTableSchemaRegistry.java index b0ee1cb4c2ae..caf5888b11a8 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-db-schema-registry-bundle/nifi-db-schema-registry-service/src/main/java/org/apache/nifi/db/schemaregistry/DatabaseTableSchemaRegistry.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-db-schema-registry-bundle/nifi-db-schema-registry-service/src/main/java/org/apache/nifi/db/schemaregistry/DatabaseTableSchemaRegistry.java @@ -157,7 +157,13 @@ private RecordField createRecordFieldFromColumn(final ResultSet columnResultSet) // COLUMN_DEF must be read first to work around Oracle bug, see NIFI-4279 for details final String defaultValue = columnResultSet.getString("COLUMN_DEF"); final String columnName = columnResultSet.getString("COLUMN_NAME"); - final int dataType = columnResultSet.getInt("DATA_TYPE"); + String typeName = columnResultSet.getString("TYPE_NAME"); + final int dataType; + if (typeName.equalsIgnoreCase("bool")) { + dataType = 16; + } else { + dataType = columnResultSet.getInt("DATA_TYPE"); + } final String nullableValue = columnResultSet.getString("IS_NULLABLE"); final boolean isNullable = "YES".equalsIgnoreCase(nullableValue) || nullableValue.isEmpty(); return new RecordField( From c3c1b834c724462bb7c96d953dd064c3fd823edb Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Wed, 28 Feb 2024 20:07:11 +0100 Subject: [PATCH 09/73] NIFI-12850 - Prevent indexing of overly large filename attribute Signed-off-by: Matt Burgess This closes #8457 --- .../provenance/index/lucene/ConvertEventToLuceneDocument.java | 2 +- .../java/org/apache/nifi/provenance/lucene/IndexingAction.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/ConvertEventToLuceneDocument.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/ConvertEventToLuceneDocument.java index f8706a105740..85e78f4575be 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/ConvertEventToLuceneDocument.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/ConvertEventToLuceneDocument.java @@ -62,7 +62,7 @@ public Document convert(final ProvenanceEventRecord record, final StorageSummary public Document convert(final ProvenanceEventRecord record, final long eventId) { final Document doc = new Document(); addField(doc, SearchableFields.FlowFileUUID, record.getFlowFileUuid()); - addField(doc, SearchableFields.Filename, record.getAttribute(CoreAttributes.FILENAME.key())); + addField(doc, SearchableFields.Filename, LuceneUtil.truncateIndexField(record.getAttribute(CoreAttributes.FILENAME.key()))); addField(doc, SearchableFields.ComponentID, record.getComponentId()); addField(doc, SearchableFields.AlternateIdentifierURI, record.getAlternateIdentifierUri()); addField(doc, SearchableFields.EventType, record.getEventType().name()); diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexingAction.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexingAction.java index 62c7d8ac6a41..24aef5044550 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexingAction.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexingAction.java @@ -56,7 +56,7 @@ private void addField(final Document doc, final SearchableField field, final Str public void index(final StandardProvenanceEventRecord record, final IndexWriter indexWriter, final Integer blockIndex) throws IOException { final Document doc = new Document(); addField(doc, SearchableFields.FlowFileUUID, record.getFlowFileUuid(), Store.NO); - addField(doc, SearchableFields.Filename, record.getAttribute(CoreAttributes.FILENAME.key()), Store.NO); + addField(doc, SearchableFields.Filename, LuceneUtil.truncateIndexField(record.getAttribute(CoreAttributes.FILENAME.key())), Store.NO); addField(doc, SearchableFields.ComponentID, record.getComponentId(), Store.NO); addField(doc, SearchableFields.AlternateIdentifierURI, record.getAlternateIdentifierUri(), Store.NO); addField(doc, SearchableFields.EventType, record.getEventType().name(), Store.NO); From 455159f6ace9c3902937468344a50ad9594d03ca Mon Sep 17 00:00:00 2001 From: Rob Fellows Date: Thu, 29 Feb 2024 16:08:47 -0500 Subject: [PATCH 10/73] [NIFI-12537] Open cluster/node dialog from Summary screen. (#8454) * [NIFI-12537] - Summary screen cluster-related features * common component status table * cluster summary dialogs * address review feedback * ellisis for node columns in cluster dialogs, cluster node dropdown panel no longer wraps, shorted the dropdown width as well. * prettier This closes #8454 --- .../summary/feature/summary.component.html | 14 +- .../summary/feature/summary.component.ts | 20 +- .../pages/summary/feature/summary.module.ts | 3 +- .../component-cluster-status.service.ts | 64 +++++ .../service/process-group-status.service.ts | 18 +- .../component-cluster-status.actions.ts | 43 ++++ .../component-cluster-status.effects.ts | 114 +++++++++ .../component-cluster-status.reducer.ts | 80 ++++++ .../component-cluster-status.selectors.ts | 45 ++++ .../state/component-cluster-status/index.ts | 47 ++++ .../nifi/src/app/pages/summary/state/index.ts | 225 ++++++++++++++++- .../summary/state/summary-listing/index.ts | 165 ++---------- .../summary-listing.actions.ts | 6 + .../summary-listing.effects.ts | 33 +-- .../summary-listing.reducer.ts | 21 +- .../summary-listing.selectors.ts | 18 +- .../cluster-summary-dialog.component.html | 85 +++++++ .../cluster-summary-dialog.component.scss} | 28 ++- .../cluster-summary-dialog.component.spec.ts | 62 +++++ .../cluster-summary-dialog.component.ts | 140 +++++++++++ .../component-cluster-table.component.ts | 120 +++++++++ .../connection-cluster-table.component.html | 130 ++++++++++ .../connection-cluster-table.component.scss | 25 ++ ...connection-cluster-table.component.spec.ts | 40 +++ .../connection-cluster-table.component.ts | 109 ++++++++ .../port-cluster-table.component.html | 107 ++++++++ .../port-cluster-table.component.scss | 25 ++ .../port-cluster-table.component.spec.ts | 40 +++ .../port-cluster-table.component.ts | 127 ++++++++++ ...process-group-cluster-table.component.html | 172 +++++++++++++ ...process-group-cluster-table.component.scss | 25 ++ ...cess-group-cluster-table.component.spec.ts | 40 +++ .../process-group-cluster-table.component.ts | 144 +++++++++++ .../processor-cluster-table.component.html | 151 +++++++++++ .../processor-cluster-table.component.scss | 25 ++ .../processor-cluster-table.component.spec.ts | 40 +++ .../processor-cluster-table.component.ts | 138 ++++++++++ ...process-group-cluster-table.component.html | 110 ++++++++ ...process-group-cluster-table.component.scss | 25 ++ ...cess-group-cluster-table.component.spec.ts | 40 +++ ...e-process-group-cluster-table.component.ts | 120 +++++++++ .../component-status-table.component.ts | 236 ++++++++++++++++++ .../port-status-table.component.html | 10 +- .../port-status-table.component.scss | 4 +- .../port-status-table.component.ts | 166 ++---------- .../summary-table-filter.component.html | 29 ++- .../summary-table-filter.component.scss | 10 + .../summary-table-filter.component.ts | 116 ++++++++- .../connection-status-listing.component.html | 11 +- .../connection-status-listing.component.ts | 40 ++- .../connection-status-table.component.html | 9 + .../connection-status-table.component.scss | 4 +- .../connection-status-table.component.ts | 170 ++----------- .../input-port-status-listing.component.html | 11 +- .../input-port-status-listing.component.ts | 38 ++- .../output-port-status-listing.component.html | 11 +- .../output-port-status-listing.component.ts | 38 ++- ...rocess-group-status-listing.component.html | 11 +- .../process-group-status-listing.component.ts | 40 ++- .../process-group-status-listing.module.ts | 3 +- .../process-group-status-table.component.html | 9 + .../process-group-status-table.component.scss | 4 +- .../process-group-status-table.component.ts | 166 ++---------- .../processor-status-listing.component.html | 11 +- .../processor-status-listing.component.ts | 40 ++- ...rocessor-status-table.component-theme.scss | 42 ++++ .../processor-status-table.component.html | 20 +- .../processor-status-table.component.scss | 15 +- .../processor-status-table.component.ts | 184 ++------------ ...rocess-group-status-listing.component.html | 11 +- ...-process-group-status-listing.component.ts | 40 ++- ...-process-group-status-table.component.html | 9 + ...-process-group-status-table.component.scss | 4 +- ...te-process-group-status-table.component.ts | 170 ++----------- .../pipes/component-type-name.pipe.spec.ts | 25 ++ .../src/app/pipes/component-type-name.pipe.ts | 60 +++++ .../cluster-summary.actions.ts | 12 +- .../cluster-summary.effects.ts | 38 ++- .../cluster-summary.reducer.ts | 10 +- .../cluster-summary.selectors.ts | 5 + .../src/app/state/cluster-summary/index.ts | 5 + .../_component-context.component-theme.scss | 62 +++++ .../component-context.component.html | 31 +++ .../component-context.component.scss | 41 +++ .../component-context.component.spec.ts | 39 +++ .../component-context.component.ts | 67 +++++ .../assets/styles/_listing-table-theme.scss | 1 + .../src/main/nifi/src/styles.scss | 6 + 88 files changed, 4057 insertions(+), 1011 deletions(-) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/service/component-cluster-status.service.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/state/component-cluster-status/component-cluster-status.actions.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/state/component-cluster-status/component-cluster-status.effects.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/state/component-cluster-status/component-cluster-status.reducer.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/state/component-cluster-status/component-cluster-status.selectors.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/state/component-cluster-status/index.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html rename nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/{service/cluster-summary.service.ts => ui/common/cluster-summary-dialog/cluster-summary-dialog.component.scss} (64%) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/component-cluster-table/component-cluster-table.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/port-cluster-table/port-cluster-table.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/port-cluster-table/port-cluster-table.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/port-cluster-table/port-cluster-table.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/port-cluster-table/port-cluster-table.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/process-group-cluster-table/process-group-cluster-table.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/process-group-cluster-table/process-group-cluster-table.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/process-group-cluster-table/process-group-cluster-table.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/process-group-cluster-table/process-group-cluster-table.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/processor-cluster-table/processor-cluster-table.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/processor-cluster-table/processor-cluster-table.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/processor-cluster-table/processor-cluster-table.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/processor-cluster-table/processor-cluster-table.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/remote-process-group-cluster-table/remote-process-group-cluster-table.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/remote-process-group-cluster-table/remote-process-group-cluster-table.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/remote-process-group-cluster-table/remote-process-group-cluster-table.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/remote-process-group-cluster-table/remote-process-group-cluster-table.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/component-status-table/component-status-table.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/_processor-status-table.component-theme.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.ts diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/summary.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/summary.component.html index 04196c4e49f8..02c166050fe3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/summary.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/summary.component.html @@ -19,8 +19,18 @@
-
-

NiFi Summary

+
+

+ @if (selectedClusterNode$ | async; as selectedNode) { + @if (selectedNode.id !== 'All') { + {{ selectedNode.address }} Summary + } @else { + NiFi Summary + } + } @else { + NiFi Summary + } +

+ @if (connectedToCluster) { +
+ }
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.scss index 2a30318aefad..4dbbdf897871 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.scss @@ -23,8 +23,8 @@ } .mat-column-actions { - width: 72px; - min-width: 72px; + width: 80px; + min-width: 80px; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts index c3dcf9989ca2..e42698c244f3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts @@ -15,21 +15,18 @@ * limitations under the License. */ -import { AfterViewInit, Component, EventEmitter, Input, Output, ViewChild } from '@angular/core'; +import { Component, Input } from '@angular/core'; import { CommonModule } from '@angular/common'; -import { MatSortModule, Sort, SortDirection } from '@angular/material/sort'; -import { MultiSort } from '../index'; -import { MatTableDataSource, MatTableModule } from '@angular/material/table'; -import { PortStatusSnapshot, PortStatusSnapshotEntity } from '../../../state/summary-listing'; +import { MatSortModule, Sort } from '@angular/material/sort'; +import { MatTableModule } from '@angular/material/table'; import { SummaryTableFilterModule } from '../summary-table-filter/summary-table-filter.module'; -import { - SummaryTableFilterArgs, - SummaryTableFilterColumn -} from '../summary-table-filter/summary-table-filter.component'; +import { SummaryTableFilterColumn } from '../summary-table-filter/summary-table-filter.component'; import { ComponentType } from '../../../../../state/shared'; import { RouterLink } from '@angular/router'; import { NiFiCommon } from '../../../../../service/nifi-common.service'; -import { MatPaginator, MatPaginatorModule } from '@angular/material/paginator'; +import { MatPaginatorModule } from '@angular/material/paginator'; +import { PortStatusSnapshot, PortStatusSnapshotEntity } from '../../../state'; +import { ComponentStatusTable } from '../component-status-table/component-status-table.component'; export type SupportedColumns = 'name' | 'runStatus' | 'in' | 'out'; @@ -40,40 +37,22 @@ export type SupportedColumns = 'name' | 'runStatus' | 'in' | 'out'; templateUrl: './port-status-table.component.html', styleUrls: ['./port-status-table.component.scss'] }) -export class PortStatusTable implements AfterViewInit { - private _initialSortColumn: SupportedColumns = 'name'; - private _initialSortDirection: SortDirection = 'asc'; +export class PortStatusTable extends ComponentStatusTable { private _portType!: 'input' | 'output'; filterableColumns: SummaryTableFilterColumn[] = [{ key: 'name', label: 'name' }]; - totalCount = 0; - filteredCount = 0; - - multiSort: MultiSort = { - active: this._initialSortColumn, - direction: this._initialSortDirection, - sortValueIndex: 0, - totalValues: 2 - }; - displayedColumns: string[] = []; - dataSource: MatTableDataSource = new MatTableDataSource(); - - @ViewChild(MatPaginator) paginator!: MatPaginator; - - constructor(private nifiCommon: NiFiCommon) {} - - ngAfterViewInit(): void { - this.dataSource.paginator = this.paginator; + constructor(private nifiCommon: NiFiCommon) { + super(); } @Input() set portType(type: 'input' | 'output') { if (type === 'input') { - this.displayedColumns = ['moreDetails', 'name', 'runStatus', 'in', 'actions']; - } else { this.displayedColumns = ['moreDetails', 'name', 'runStatus', 'out', 'actions']; + } else { + this.displayedColumns = ['moreDetails', 'name', 'runStatus', 'in', 'actions']; } this._portType = type; } @@ -82,74 +61,21 @@ export class PortStatusTable implements AfterViewInit { return this._portType; } - @Input() selectedPortId!: string; + override filterPredicate(data: PortStatusSnapshotEntity, filter: string): boolean { + const { filterTerm, filterColumn, filterStatus } = JSON.parse(filter); + const matchOnStatus: boolean = filterStatus !== 'All'; - @Input() set initialSortColumn(initialSortColumn: SupportedColumns) { - this._initialSortColumn = initialSortColumn; - this.multiSort = { ...this.multiSort, active: initialSortColumn }; - } - - get initialSortColumn() { - return this._initialSortColumn; - } - - @Input() set initialSortDirection(initialSortDirection: SortDirection) { - this._initialSortDirection = initialSortDirection; - this.multiSort = { ...this.multiSort, direction: initialSortDirection }; - } - - get initialSortDirection() { - return this._initialSortDirection; - } - - @Input() set ports(ports: PortStatusSnapshotEntity[]) { - if (ports) { - this.dataSource.data = this.sortEntities(ports, this.multiSort); - this.dataSource.filterPredicate = (data: PortStatusSnapshotEntity, filter: string) => { - const { filterTerm, filterColumn, filterStatus } = JSON.parse(filter); - const matchOnStatus: boolean = filterStatus !== 'All'; - - if (matchOnStatus) { - if (data.portStatusSnapshot.runStatus !== filterStatus) { - return false; - } - } - if (filterTerm === '') { - return true; - } - - const field: string = data.portStatusSnapshot[filterColumn as keyof PortStatusSnapshot] as string; - return this.nifiCommon.stringContains(field, filterTerm, true); - }; - - this.totalCount = ports.length; - this.filteredCount = ports.length; + if (matchOnStatus) { + if (data.portStatusSnapshot.runStatus !== filterStatus) { + return false; + } } - } - - @Input() summaryListingStatus: string | null = null; - @Input() loadedTimestamp: string | null = null; - - @Output() refresh: EventEmitter = new EventEmitter(); - @Output() selectPort: EventEmitter = new EventEmitter(); - @Output() clearSelection: EventEmitter = new EventEmitter(); - - applyFilter(filter: SummaryTableFilterArgs) { - this.dataSource.filter = JSON.stringify(filter); - this.filteredCount = this.dataSource.filteredData.length; - this.resetPaginator(); - this.selectNone(); - } - - resetPaginator(): void { - if (this.dataSource.paginator) { - this.dataSource.paginator.firstPage(); + if (filterTerm === '') { + return true; } - } - paginationChanged(): void { - // clear out any selection - this.selectNone(); + const field: string = data.portStatusSnapshot[filterColumn as keyof PortStatusSnapshot] as string; + return this.nifiCommon.stringContains(field, filterTerm, true); } formatName(port: PortStatusSnapshotEntity): string { @@ -193,31 +119,11 @@ export class PortStatusTable implements AfterViewInit { return ['/process-groups', port.portStatusSnapshot.groupId, componentType, port.id]; } - select(port: PortStatusSnapshotEntity): void { - this.selectPort.next(port); - } - - private selectNone() { - this.clearSelection.next(); - } - - isSelected(port: PortStatusSnapshotEntity): boolean { - if (this.selectedPortId) { - return port.id === this.selectedPortId; - } - return false; - } - - sortData(sort: Sort) { - this.setMultiSort(sort); - this.dataSource.data = this.sortEntities(this.dataSource.data, sort); - } - canRead(port: PortStatusSnapshotEntity) { return port.canRead; } - private supportsMultiValuedSort(sort: Sort): boolean { + override supportsMultiValuedSort(sort: Sort): boolean { switch (sort.active) { case 'in': case 'out': @@ -227,29 +133,7 @@ export class PortStatusTable implements AfterViewInit { } } - private setMultiSort(sort: Sort) { - const { active, direction, sortValueIndex, totalValues } = this.multiSort; - - if (this.supportsMultiValuedSort(sort)) { - if (active === sort.active) { - // previous sort was of the same column - if (direction === 'desc' && sort.direction === 'asc') { - // change from previous index to the next - const newIndex = sortValueIndex + 1 >= totalValues ? 0 : sortValueIndex + 1; - this.multiSort = { ...sort, sortValueIndex: newIndex, totalValues }; - } else { - this.multiSort = { ...sort, sortValueIndex, totalValues }; - } - } else { - // sorting a different column, just reset - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } else { - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } - - private sortEntities(data: PortStatusSnapshotEntity[], sort: Sort): PortStatusSnapshotEntity[] { + override sortEntities(data: PortStatusSnapshotEntity[], sort: Sort): PortStatusSnapshotEntity[] { if (!data) { return []; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.html index e2d5da3aa91c..9fc10276efb9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.html @@ -32,7 +32,7 @@ Filter By @for (option of filterableColumns; track option) { - {{ option.label }} + {{ option.label }} } @@ -42,12 +42,12 @@ Status - All Statuses - Running - Stopped - Validating - Disabled - Invalid + All Statuses + Running + Stopped + Validating + Disabled + Invalid
@@ -55,7 +55,20 @@ @if (includePrimaryNodeOnlyFilter) {
- Primary Node + Primary Node +
+ } + + @if (clusterNodes && clusterNodes.length > 0) { +
+ + Cluster Node + + @for (node of clusterNodes; track node) { + {{ node.address }} + } + +
} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.scss index 3651a8ab7069..2d8164458736 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.scss @@ -14,3 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +.summary-table-filter-container { + .cluster-node-selection { + max-width: 100%; + + .mat-mdc-form-field { + width: 300px; + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.ts index 7f46f7a538f0..f09115e6ad5a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/summary-table-filter/summary-table-filter.component.ts @@ -19,6 +19,7 @@ import { AfterViewInit, Component, DestroyRef, EventEmitter, inject, Input, Outp import { FormBuilder, FormGroup } from '@angular/forms'; import { debounceTime } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { NodeSearchResult } from '../../../../../state/cluster-summary'; export interface SummaryTableFilterColumn { key: string; @@ -30,6 +31,11 @@ export interface SummaryTableFilterArgs { filterColumn: string; filterStatus?: string; primaryOnly?: boolean; + clusterNode?: NodeSearchResult; +} + +export interface SummaryTableFilterContext extends SummaryTableFilterArgs { + changedField: string; } @Component({ @@ -42,13 +48,71 @@ export class SummaryTableFilter implements AfterViewInit { private _filteredCount = 0; private _totalCount = 0; private _initialFilterColumn = 'name'; + private _filterableColumns: SummaryTableFilterColumn[] = []; private destroyRef: DestroyRef = inject(DestroyRef); + showFilterMatchedLabel = false; + allNodes: NodeSearchResult = { + id: 'All', + address: 'All Nodes' + }; + + private _clusterNodes: NodeSearchResult[] = []; + private _selectedNode: NodeSearchResult | null = this.allNodes; + + @Input() set filterableColumns(filterableColumns: SummaryTableFilterColumn[]) { + this._filterableColumns = filterableColumns; + } + get filterableColumns(): SummaryTableFilterColumn[] { + return this._filterableColumns; + } - @Input() filterableColumns: SummaryTableFilterColumn[] = []; @Input() includeStatusFilter = false; @Input() includePrimaryNodeOnlyFilter = false; - @Output() filterChanged: EventEmitter = new EventEmitter(); + + @Input() set selectedNode(node: NodeSearchResult | null) { + const n: NodeSearchResult = node ? (node.id !== 'All' ? node : this.allNodes) : this.allNodes; + // find it in the available nodes + const found = this._clusterNodes.find((node) => node.id === n.id); + if (found) { + this.filterForm.get('clusterNode')?.setValue(found); + this._selectedNode = found; + } + } + + @Input() set clusterNodes(nodes: NodeSearchResult[] | null) { + if (!nodes) { + this._clusterNodes = []; + } else { + // test if the nodes have changed + if (!this.areSame(this._clusterNodes, nodes)) { + this._clusterNodes = [this.allNodes, ...nodes]; + if (this._selectedNode) { + this.selectedNode = this._selectedNode; + } + } + } + } + get clusterNodes(): NodeSearchResult[] { + return this._clusterNodes; + } + + private areSame(a: NodeSearchResult[], b: NodeSearchResult[]) { + if (a.length !== b.length) { + return false; + } + + const noMatch = a.filter((node) => b.findIndex((n) => n.id === node.id) < 0); + return noMatch.length === 0 || (noMatch.length === 1 && noMatch[0].id === 'All'); + } + + private areEqual(a: NodeSearchResult, b: NodeSearchResult) { + return a.id === b.id; + } + + @Output() filterChanged: EventEmitter = new EventEmitter(); + @Output() clusterFilterChanged: EventEmitter = + new EventEmitter(); @Input() set filterTerm(term: string) { this.filterForm.get('filterTerm')?.value(term); @@ -94,7 +158,8 @@ export class SummaryTableFilter implements AfterViewInit { filterTerm: '', filterColumn: this._initialFilterColumn || 'name', filterStatus: 'All', - primaryOnly: false + primaryOnly: false, + clusterNode: this.allNodes }); } @@ -106,7 +171,8 @@ export class SummaryTableFilter implements AfterViewInit { const filterColumn = this.filterForm.get('filterColumn')?.value; const filterStatus = this.filterForm.get('filterStatus')?.value; const primaryOnly = this.filterForm.get('primaryOnly')?.value; - this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly); + const clusterNode = this.filterForm.get('clusterNode')?.value; + this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly, clusterNode, 'filterTerm'); }); this.filterForm @@ -116,7 +182,8 @@ export class SummaryTableFilter implements AfterViewInit { const filterTerm = this.filterForm.get('filterTerm')?.value; const filterStatus = this.filterForm.get('filterStatus')?.value; const primaryOnly = this.filterForm.get('primaryOnly')?.value; - this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly); + const clusterNode = this.filterForm.get('clusterNode')?.value; + this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly, clusterNode, 'filterColumn'); }); this.filterForm @@ -126,7 +193,8 @@ export class SummaryTableFilter implements AfterViewInit { const filterTerm = this.filterForm.get('filterTerm')?.value; const filterColumn = this.filterForm.get('filterColumn')?.value; const primaryOnly = this.filterForm.get('primaryOnly')?.value; - this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly); + const clusterNode = this.filterForm.get('clusterNode')?.value; + this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly, clusterNode, 'filterStatus'); }); this.filterForm @@ -136,17 +204,45 @@ export class SummaryTableFilter implements AfterViewInit { const filterTerm = this.filterForm.get('filterTerm')?.value; const filterColumn = this.filterForm.get('filterColumn')?.value; const filterStatus = this.filterForm.get('filterStatus')?.value; - this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly); + const clusterNode = this.filterForm.get('clusterNode')?.value; + this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly, clusterNode, 'primaryOnly'); + }); + + this.filterForm + .get('clusterNode') + ?.valueChanges.pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe((clusterNode) => { + if (this._selectedNode?.id !== clusterNode.id) { + this._selectedNode = clusterNode; + const filterTerm = this.filterForm.get('filterTerm')?.value; + const filterColumn = this.filterForm.get('filterColumn')?.value; + const filterStatus = this.filterForm.get('filterStatus')?.value; + const primaryOnly = this.filterForm.get('primaryOnly')?.value; + this.applyFilter(filterTerm, filterColumn, filterStatus, primaryOnly, clusterNode, 'clusterNode'); + } }); } - applyFilter(filterTerm: string, filterColumn: string, filterStatus: string, primaryOnly: boolean) { + applyFilter( + filterTerm: string, + filterColumn: string, + filterStatus: string, + primaryOnly: boolean, + clusterNode: NodeSearchResult, + changedField: string + ) { this.filterChanged.next({ filterColumn, filterStatus, filterTerm, - primaryOnly + primaryOnly, + clusterNode, + changedField }); - this.showFilterMatchedLabel = filterTerm?.length > 0 || filterStatus !== 'All' || primaryOnly; + this.showFilterMatchedLabel = + filterTerm?.length > 0 || + filterStatus !== 'All' || + primaryOnly || + (clusterNode ? clusterNode.id !== 'All' : false); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.html index 6fd0e42bd888..f72ce3afd3ed 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.html @@ -22,14 +22,19 @@ } @else { } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.ts index e337cc5c50e9..bb8c89a26527 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-listing.component.ts @@ -17,22 +17,31 @@ import { Component } from '@angular/core'; import { Store } from '@ngrx/store'; -import { ConnectionStatusSnapshotEntity, SummaryListingState } from '../../state/summary-listing'; +import { SummaryListingState } from '../../state/summary-listing'; import { initialState } from '../../state/summary-listing/summary-listing.reducer'; import * as SummaryListingActions from '../../state/summary-listing/summary-listing.actions'; import { selectConnectionIdFromRoute, selectConnectionStatus, selectConnectionStatusSnapshots, + selectSelectedClusterNode, selectSummaryListingLoadedTimestamp, selectSummaryListingStatus, selectViewStatusHistory } from '../../state/summary-listing/summary-listing.selectors'; import { selectCurrentUser } from '../../../../state/current-user/current-user.selectors'; -import { filter, switchMap, take } from 'rxjs'; +import { filter, map, switchMap, take } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { getStatusHistoryAndOpenDialog } from '../../../../state/status-history/status-history.actions'; -import { ComponentType } from '../../../../state/shared'; +import { ComponentType, isDefinedAndNotNull } from '../../../../state/shared'; +import { loadClusterSummary } from '../../../../state/cluster-summary/cluster-summary.actions'; +import { ConnectionStatusSnapshotEntity } from '../../state'; +import { + selectClusterSearchResults, + selectClusterSummary +} from '../../../../state/cluster-summary/cluster-summary.selectors'; +import * as ClusterStatusActions from '../../state/component-cluster-status/component-cluster-status.actions'; +import { NodeSearchResult } from '../../../../state/cluster-summary'; @Component({ selector: 'connection-status-listing', @@ -45,6 +54,15 @@ export class ConnectionStatusListing { currentUser$ = this.store.select(selectCurrentUser); connectionStatusSnapshots$ = this.store.select(selectConnectionStatusSnapshots); selectedConnectionId$ = this.store.select(selectConnectionIdFromRoute); + connectedToCluster$ = this.store.select(selectClusterSummary).pipe( + isDefinedAndNotNull(), + map((cluster) => cluster.connectedToCluster) + ); + clusterNodes$ = this.store.select(selectClusterSearchResults).pipe( + isDefinedAndNotNull(), + map((results) => results.nodeResults) + ); + selectedClusterNode$ = this.store.select(selectSelectedClusterNode); constructor(private store: Store) { this.store @@ -80,6 +98,7 @@ export class ConnectionStatusListing { refreshSummaryListing() { this.store.dispatch(SummaryListingActions.loadSummaryListing({ recursive: true })); + this.store.dispatch(loadClusterSummary()); } selectConnection(connection: ConnectionStatusSnapshotEntity): void { @@ -103,4 +122,19 @@ export class ConnectionStatusListing { }) ); } + + viewClusteredDetails(processor: ConnectionStatusSnapshotEntity): void { + this.store.dispatch( + ClusterStatusActions.loadComponentClusterStatusAndOpenDialog({ + request: { + id: processor.id, + componentType: ComponentType.Connection + } + }) + ); + } + + clusterNodeSelected(clusterNode: NodeSearchResult) { + this.store.dispatch(SummaryListingActions.selectClusterNode({ clusterNode })); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html index 31cfc53bddd9..7706656ae83c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html @@ -26,6 +26,8 @@ [includeStatusFilter]="false" [includePrimaryNodeOnlyFilter]="false" filterColumn="sourceName" + [clusterNodes]="clusterNodes" + [selectedNode]="selectedClusterNode" (filterChanged)="applyFilter($event)">
@@ -213,6 +215,13 @@ class="pointer fa fa-area-chart" title="View Status History" (click)="viewStatusHistoryClicked($event, item)">
+ + @if (connectedToCluster) { +
+ } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.scss index 5214b0d2125a..f6ac47ab8695 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.scss @@ -23,8 +23,8 @@ } .mat-column-actions { - width: 72px; - min-width: 72px; + width: 80px; + min-width: 80px; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts index 7202d0b9eaa3..df0e1f83abbb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts @@ -15,21 +15,18 @@ * limitations under the License. */ -import { AfterViewInit, Component, EventEmitter, Input, Output, ViewChild } from '@angular/core'; +import { Component } from '@angular/core'; import { CommonModule } from '@angular/common'; import { SummaryTableFilterModule } from '../../common/summary-table-filter/summary-table-filter.module'; -import { MatSortModule, Sort, SortDirection } from '@angular/material/sort'; -import { MultiSort } from '../../common'; +import { MatSortModule, Sort } from '@angular/material/sort'; import { NiFiCommon } from '../../../../../service/nifi-common.service'; -import { - SummaryTableFilterArgs, - SummaryTableFilterColumn -} from '../../common/summary-table-filter/summary-table-filter.component'; -import { ConnectionStatusSnapshot, ConnectionStatusSnapshotEntity } from '../../../state/summary-listing'; -import { MatTableDataSource, MatTableModule } from '@angular/material/table'; +import { SummaryTableFilterColumn } from '../../common/summary-table-filter/summary-table-filter.component'; +import { MatTableModule } from '@angular/material/table'; import { ComponentType } from '../../../../../state/shared'; import { RouterLink } from '@angular/router'; -import { MatPaginator, MatPaginatorModule } from '@angular/material/paginator'; +import { MatPaginatorModule } from '@angular/material/paginator'; +import { ConnectionStatusSnapshot, ConnectionStatusSnapshotEntity } from '../../../state'; +import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; export type SupportedColumns = 'name' | 'queue' | 'in' | 'out' | 'threshold' | 'sourceName' | 'destinationName'; @@ -40,26 +37,13 @@ export type SupportedColumns = 'name' | 'queue' | 'in' | 'out' | 'threshold' | ' templateUrl: './connection-status-table.component.html', styleUrls: ['./connection-status-table.component.scss'] }) -export class ConnectionStatusTable implements AfterViewInit { - private _initialSortColumn: SupportedColumns = 'sourceName'; - private _initialSortDirection: SortDirection = 'asc'; - +export class ConnectionStatusTable extends ComponentStatusTable { filterableColumns: SummaryTableFilterColumn[] = [ { key: 'sourceName', label: 'source' }, { key: 'name', label: 'name' }, { key: 'destinationName', label: 'destination' } ]; - totalCount = 0; - filteredCount = 0; - - multiSort: MultiSort = { - active: this._initialSortColumn, - direction: this._initialSortDirection, - sortValueIndex: 0, - totalValues: 2 - }; - displayedColumns: string[] = [ 'moreDetails', 'name', @@ -72,88 +56,19 @@ export class ConnectionStatusTable implements AfterViewInit { 'actions' ]; - dataSource: MatTableDataSource = - new MatTableDataSource(); - - @ViewChild(MatPaginator) paginator!: MatPaginator; - - constructor(private nifiCommon: NiFiCommon) {} - - ngAfterViewInit(): void { - this.dataSource.paginator = this.paginator; - } - - @Input() set initialSortColumn(initialSortColumn: SupportedColumns) { - this._initialSortColumn = initialSortColumn; - this.multiSort = { ...this.multiSort, active: initialSortColumn }; - } - - get initialSortColumn() { - return this._initialSortColumn; - } - - @Input() set initialSortDirection(initialSortDirection: SortDirection) { - this._initialSortDirection = initialSortDirection; - this.multiSort = { ...this.multiSort, direction: initialSortDirection }; + constructor(private nifiCommon: NiFiCommon) { + super(); } - get initialSortDirection() { - return this._initialSortDirection; - } - - @Input() selectedConnectionId!: string; - - @Input() set connections(connections: ConnectionStatusSnapshotEntity[]) { - if (connections) { - this.dataSource.data = this.sortEntities(connections, this.multiSort); - this.dataSource.filterPredicate = (data: ConnectionStatusSnapshotEntity, filter: string) => { - const { filterTerm, filterColumn } = JSON.parse(filter); - - if (filterTerm === '') { - return true; - } - - const field: string = data.connectionStatusSnapshot[ - filterColumn as keyof ConnectionStatusSnapshot - ] as string; - return this.nifiCommon.stringContains(field, filterTerm, true); - }; - - this.totalCount = connections.length; - this.filteredCount = connections.length; - } - } - - @Input() summaryListingStatus: string | null = null; - @Input() loadedTimestamp: string | null = null; + override filterPredicate(data: ConnectionStatusSnapshotEntity, filter: string): boolean { + const { filterTerm, filterColumn } = JSON.parse(filter); - @Output() refresh: EventEmitter = new EventEmitter(); - @Output() viewStatusHistory: EventEmitter = - new EventEmitter(); - @Output() selectConnection: EventEmitter = - new EventEmitter(); - @Output() clearSelection: EventEmitter = new EventEmitter(); - - resetPaginator(): void { - if (this.dataSource.paginator) { - this.dataSource.paginator.firstPage(); + if (filterTerm === '') { + return true; } - } - applyFilter(filter: SummaryTableFilterArgs) { - this.dataSource.filter = JSON.stringify(filter); - this.filteredCount = this.dataSource.filteredData.length; - this.resetPaginator(); - this.selectNone(); - } - - paginationChanged(): void { - // clear out any selection - this.selectNone(); - } - - private selectNone() { - this.clearSelection.next(); + const field: string = data.connectionStatusSnapshot[filterColumn as keyof ConnectionStatusSnapshot] as string; + return this.nifiCommon.stringContains(field, filterTerm, true); } getConnectionLink(connection: ConnectionStatusSnapshotEntity): string[] { @@ -165,54 +80,39 @@ export class ConnectionStatusTable implements AfterViewInit { ]; } - select(connection: ConnectionStatusSnapshotEntity): void { - this.selectConnection.next(connection); - } - - isSelected(connection: ConnectionStatusSnapshotEntity): boolean { - if (this.selectedConnectionId) { - return connection.id === this.selectedConnectionId; - } - return false; - } - canRead(connection: ConnectionStatusSnapshotEntity): boolean { return connection.canRead; } - sortData(sort: Sort) { - this.setMultiSort(sort); - this.dataSource.data = this.sortEntities(this.dataSource.data, sort); - } - formatName(connection: ConnectionStatusSnapshotEntity): string { return connection.connectionStatusSnapshot.name; } + formatSource(connection: ConnectionStatusSnapshotEntity): string { return connection.connectionStatusSnapshot.sourceName; } + formatDestination(connection: ConnectionStatusSnapshotEntity): string { return connection.connectionStatusSnapshot.destinationName; } + formatIn(connection: ConnectionStatusSnapshotEntity): string { return connection.connectionStatusSnapshot.input; } + formatOut(connection: ConnectionStatusSnapshotEntity): string { return connection.connectionStatusSnapshot.output; } + formatQueue(connection: ConnectionStatusSnapshotEntity): string { return connection.connectionStatusSnapshot.queued; } + formatThreshold(connection: ConnectionStatusSnapshotEntity): string { return `${connection.connectionStatusSnapshot.percentUseCount}% | ${connection.connectionStatusSnapshot.percentUseBytes}%`; } - viewStatusHistoryClicked(event: MouseEvent, connection: ConnectionStatusSnapshotEntity): void { - event.stopPropagation(); - this.viewStatusHistory.next(connection); - } - - private supportsMultiValuedSort(sort: Sort): boolean { + override supportsMultiValuedSort(sort: Sort): boolean { switch (sort.active) { case 'in': case 'out': @@ -224,29 +124,7 @@ export class ConnectionStatusTable implements AfterViewInit { } } - private setMultiSort(sort: Sort) { - const { active, direction, sortValueIndex, totalValues } = this.multiSort; - - if (this.supportsMultiValuedSort(sort)) { - if (active === sort.active) { - // previous sort was of the same column - if (direction === 'desc' && sort.direction === 'asc') { - // change from previous index to the next - const newIndex = sortValueIndex + 1 >= totalValues ? 0 : sortValueIndex + 1; - this.multiSort = { ...sort, sortValueIndex: newIndex, totalValues }; - } else { - this.multiSort = { ...sort, sortValueIndex, totalValues }; - } - } else { - // sorting a different column, just reset - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } else { - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } - - private sortEntities(data: ConnectionStatusSnapshotEntity[], sort: Sort): ConnectionStatusSnapshotEntity[] { + override sortEntities(data: ConnectionStatusSnapshotEntity[], sort: Sort): ConnectionStatusSnapshotEntity[] { if (!data) { return []; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.html index 901b3b6f6371..ee0233a0315a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.html @@ -22,14 +22,19 @@ } @else { } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.ts index 8c22deaf96a8..adae9ce68a40 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/input-port-status-listing/input-port-status-listing.component.ts @@ -19,14 +19,25 @@ import { Component } from '@angular/core'; import { selectInputPortIdFromRoute, selectInputPortStatusSnapshots, + selectSelectedClusterNode, selectSummaryListingLoadedTimestamp, selectSummaryListingStatus } from '../../state/summary-listing/summary-listing.selectors'; import { selectCurrentUser } from '../../../../state/current-user/current-user.selectors'; -import { PortStatusSnapshotEntity, SummaryListingState } from '../../state/summary-listing'; +import { SummaryListingState } from '../../state/summary-listing'; import { Store } from '@ngrx/store'; import { initialState } from '../../state/summary-listing/summary-listing.reducer'; import * as SummaryListingActions from '../../state/summary-listing/summary-listing.actions'; +import { loadClusterSummary } from '../../../../state/cluster-summary/cluster-summary.actions'; +import { PortStatusSnapshotEntity } from '../../state'; +import { + selectClusterSearchResults, + selectClusterSummary +} from '../../../../state/cluster-summary/cluster-summary.selectors'; +import { ComponentType, isDefinedAndNotNull } from '../../../../state/shared'; +import { map } from 'rxjs'; +import { NodeSearchResult } from '../../../../state/cluster-summary'; +import * as ClusterStatusActions from '../../state/component-cluster-status/component-cluster-status.actions'; @Component({ selector: 'input-port-status-listing', @@ -39,6 +50,15 @@ export class InputPortStatusListing { summaryListingStatus$ = this.store.select(selectSummaryListingStatus); currentUser$ = this.store.select(selectCurrentUser); selectedPortId$ = this.store.select(selectInputPortIdFromRoute); + connectedToCluster$ = this.store.select(selectClusterSummary).pipe( + isDefinedAndNotNull(), + map((cluster) => cluster.connectedToCluster) + ); + clusterNodes$ = this.store.select(selectClusterSearchResults).pipe( + isDefinedAndNotNull(), + map((results) => results.nodeResults) + ); + selectedClusterNode$ = this.store.select(selectSelectedClusterNode); constructor(private store: Store) {} @@ -48,6 +68,7 @@ export class InputPortStatusListing { refreshSummaryListing() { this.store.dispatch(SummaryListingActions.loadSummaryListing({ recursive: true })); + this.store.dispatch(loadClusterSummary()); } selectPort(port: PortStatusSnapshotEntity): void { @@ -63,4 +84,19 @@ export class InputPortStatusListing { clearSelection() { this.store.dispatch(SummaryListingActions.clearInputPortStatusSelection()); } + + clusterNodeSelected(clusterNode: NodeSearchResult) { + this.store.dispatch(SummaryListingActions.selectClusterNode({ clusterNode })); + } + + viewClusteredDetails(port: PortStatusSnapshotEntity): void { + this.store.dispatch( + ClusterStatusActions.loadComponentClusterStatusAndOpenDialog({ + request: { + id: port.id, + componentType: ComponentType.InputPort + } + }) + ); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.html index 558c8178c04d..df72786f27d9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.html @@ -22,14 +22,19 @@ } @else { } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.ts index d069aa5e66ff..90482a40717d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/output-port-status-listing/output-port-status-listing.component.ts @@ -19,14 +19,25 @@ import { Component } from '@angular/core'; import { selectOutputPortIdFromRoute, selectOutputPortStatusSnapshots, + selectSelectedClusterNode, selectSummaryListingLoadedTimestamp, selectSummaryListingStatus } from '../../state/summary-listing/summary-listing.selectors'; import { selectCurrentUser } from '../../../../state/current-user/current-user.selectors'; import { Store } from '@ngrx/store'; -import { PortStatusSnapshotEntity, SummaryListingState } from '../../state/summary-listing'; +import { SummaryListingState } from '../../state/summary-listing'; import { initialState } from '../../state/summary-listing/summary-listing.reducer'; import * as SummaryListingActions from '../../state/summary-listing/summary-listing.actions'; +import { loadClusterSummary } from '../../../../state/cluster-summary/cluster-summary.actions'; +import { PortStatusSnapshotEntity } from '../../state'; +import { + selectClusterSearchResults, + selectClusterSummary +} from '../../../../state/cluster-summary/cluster-summary.selectors'; +import { ComponentType, isDefinedAndNotNull } from '../../../../state/shared'; +import { map } from 'rxjs'; +import { NodeSearchResult } from '../../../../state/cluster-summary'; +import * as ClusterStatusActions from '../../state/component-cluster-status/component-cluster-status.actions'; @Component({ selector: 'output-port-status-listing', @@ -39,6 +50,15 @@ export class OutputPortStatusListing { summaryListingStatus$ = this.store.select(selectSummaryListingStatus); currentUser$ = this.store.select(selectCurrentUser); selectedPortId$ = this.store.select(selectOutputPortIdFromRoute); + connectedToCluster$ = this.store.select(selectClusterSummary).pipe( + isDefinedAndNotNull(), + map((cluster) => cluster.connectedToCluster) + ); + clusterNodes$ = this.store.select(selectClusterSearchResults).pipe( + isDefinedAndNotNull(), + map((results) => results.nodeResults) + ); + selectedClusterNode$ = this.store.select(selectSelectedClusterNode); constructor(private store: Store) {} @@ -48,6 +68,7 @@ export class OutputPortStatusListing { refreshSummaryListing() { this.store.dispatch(SummaryListingActions.loadSummaryListing({ recursive: true })); + this.store.dispatch(loadClusterSummary()); } selectPort(port: PortStatusSnapshotEntity): void { @@ -63,4 +84,19 @@ export class OutputPortStatusListing { clearSelection() { this.store.dispatch(SummaryListingActions.clearOutputPortStatusSelection()); } + + clusterNodeSelected(clusterNode: NodeSearchResult) { + this.store.dispatch(SummaryListingActions.selectClusterNode({ clusterNode })); + } + + viewClusteredDetails(port: PortStatusSnapshotEntity): void { + this.store.dispatch( + ClusterStatusActions.loadComponentClusterStatusAndOpenDialog({ + request: { + id: port.id, + componentType: ComponentType.OutputPort + } + }) + ); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.html index a38af8cf7091..7d0af4cc0d50 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.html @@ -22,15 +22,20 @@ } @else { } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.ts index bbb7cc083059..468f63480b62 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.component.ts @@ -18,22 +18,31 @@ import { Component } from '@angular/core'; import { initialState } from '../../state/summary-listing/summary-listing.reducer'; import * as SummaryListingActions from '../../state/summary-listing/summary-listing.actions'; -import { ProcessGroupStatusSnapshotEntity, SummaryListingState } from '../../state/summary-listing'; +import { SummaryListingState } from '../../state/summary-listing'; import { Store } from '@ngrx/store'; import { selectProcessGroupIdFromRoute, selectProcessGroupStatus, selectProcessGroupStatusItem, selectProcessGroupStatusSnapshots, + selectSelectedClusterNode, selectSummaryListingLoadedTimestamp, selectSummaryListingStatus, selectViewStatusHistory } from '../../state/summary-listing/summary-listing.selectors'; -import { filter, switchMap, take } from 'rxjs'; +import { filter, map, switchMap, take } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { getStatusHistoryAndOpenDialog } from '../../../../state/status-history/status-history.actions'; -import { ComponentType } from '../../../../state/shared'; +import { ComponentType, isDefinedAndNotNull } from '../../../../state/shared'; import { selectCurrentUser } from '../../../../state/current-user/current-user.selectors'; +import { loadClusterSummary } from '../../../../state/cluster-summary/cluster-summary.actions'; +import { ProcessGroupStatusSnapshotEntity } from '../../state'; +import * as ClusterStatusActions from '../../state/component-cluster-status/component-cluster-status.actions'; +import { NodeSearchResult } from '../../../../state/cluster-summary'; +import { + selectClusterSearchResults, + selectClusterSummary +} from '../../../../state/cluster-summary/cluster-summary.selectors'; @Component({ selector: 'process-group-status-listing', @@ -47,6 +56,15 @@ export class ProcessGroupStatusListing { currentUser$ = this.store.select(selectCurrentUser); selectedProcessGroupId$ = this.store.select(selectProcessGroupIdFromRoute); processGroupStatus$ = this.store.select(selectProcessGroupStatus); + connectedToCluster$ = this.store.select(selectClusterSummary).pipe( + isDefinedAndNotNull(), + map((cluster) => cluster.connectedToCluster) + ); + clusterNodes$ = this.store.select(selectClusterSearchResults).pipe( + isDefinedAndNotNull(), + map((results) => results.nodeResults) + ); + selectedClusterNode$ = this.store.select(selectSelectedClusterNode); constructor(private store: Store) { this.store @@ -82,6 +100,7 @@ export class ProcessGroupStatusListing { refreshSummaryListing() { this.store.dispatch(SummaryListingActions.loadSummaryListing({ recursive: true })); + this.store.dispatch(loadClusterSummary()); } viewStatusHistory(pg: ProcessGroupStatusSnapshotEntity): void { @@ -105,4 +124,19 @@ export class ProcessGroupStatusListing { clearSelection() { this.store.dispatch(SummaryListingActions.clearProcessGroupStatusSelection()); } + + viewClusteredDetails(pg: ProcessGroupStatusSnapshotEntity): void { + this.store.dispatch( + ClusterStatusActions.loadComponentClusterStatusAndOpenDialog({ + request: { + id: pg.id, + componentType: ComponentType.ProcessGroup + } + }) + ); + } + + clusterNodeSelected(clusterNode: NodeSearchResult) { + this.store.dispatch(SummaryListingActions.selectClusterNode({ clusterNode })); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.module.ts index 59a02e71dfda..45e72898981b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-listing.module.ts @@ -20,10 +20,11 @@ import { ProcessGroupStatusListing } from './process-group-status-listing.compon import { CommonModule } from '@angular/common'; import { NgxSkeletonLoaderModule } from 'ngx-skeleton-loader'; import { ProcessGroupStatusTable } from './process-group-status-table/process-group-status-table.component'; +import { ProcessorStatusTable } from '../processor-status-listing/processor-status-table/processor-status-table.component'; @NgModule({ declarations: [ProcessGroupStatusListing], exports: [ProcessGroupStatusListing], - imports: [CommonModule, NgxSkeletonLoaderModule, ProcessGroupStatusTable] + imports: [CommonModule, NgxSkeletonLoaderModule, ProcessGroupStatusTable, ProcessorStatusTable] }) export class ProcessGroupStatusListingModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html index 212bffb5ac81..362e65487ce4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html @@ -25,6 +25,8 @@ [filterableColumns]="filterableColumns" [includeStatusFilter]="false" [includePrimaryNodeOnlyFilter]="false" + [clusterNodes]="clusterNodes" + [selectedNode]="selectedClusterNode" (filterChanged)="applyFilter($event)">
@@ -341,6 +343,13 @@ class="pointer fa fa-area-chart" title="View Status History" (click)="viewStatusHistoryClicked($event, item)">
+ + @if (connectedToCluster) { +
+ } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.scss index 802193673e24..ece5fddef076 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.scss @@ -23,8 +23,8 @@ } .mat-column-actions { - width: 72px; - min-width: 72px; + width: 80px; + min-width: 80px; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts index dbe93501a899..99fc53ed288a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts @@ -15,20 +15,17 @@ * limitations under the License. */ -import { AfterViewInit, Component, EventEmitter, Input, Output, ViewChild } from '@angular/core'; +import { Component, Input } from '@angular/core'; import { CommonModule } from '@angular/common'; -import { MatSortModule, Sort, SortDirection } from '@angular/material/sort'; -import { MultiSort } from '../../common'; -import { MatTableDataSource, MatTableModule } from '@angular/material/table'; +import { MatSortModule, Sort } from '@angular/material/sort'; +import { MatTableModule } from '@angular/material/table'; import { SummaryTableFilterModule } from '../../common/summary-table-filter/summary-table-filter.module'; -import { ProcessGroupStatusSnapshot, ProcessGroupStatusSnapshotEntity } from '../../../state/summary-listing'; -import { - SummaryTableFilterArgs, - SummaryTableFilterColumn -} from '../../common/summary-table-filter/summary-table-filter.component'; +import { SummaryTableFilterColumn } from '../../common/summary-table-filter/summary-table-filter.component'; import { NiFiCommon } from '../../../../../service/nifi-common.service'; import { RouterLink } from '@angular/router'; -import { MatPaginator, MatPaginatorModule } from '@angular/material/paginator'; +import { MatPaginatorModule } from '@angular/material/paginator'; +import { ProcessGroupStatusSnapshot, ProcessGroupStatusSnapshotEntity } from '../../../state'; +import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; export type SupportedColumns = | 'name' @@ -49,20 +46,8 @@ export type SupportedColumns = templateUrl: './process-group-status-table.component.html', styleUrls: ['./process-group-status-table.component.scss'] }) -export class ProcessGroupStatusTable implements AfterViewInit { - private _initialSortColumn: SupportedColumns = 'name'; - private _initialSortDirection: SortDirection = 'asc'; - +export class ProcessGroupStatusTable extends ComponentStatusTable { filterableColumns: SummaryTableFilterColumn[] = [{ key: 'name', label: 'name' }]; - totalCount = 0; - filteredCount = 0; - - multiSort: MultiSort = { - active: this._initialSortColumn, - direction: this._initialSortDirection, - sortValueIndex: 0, - totalValues: 2 - }; displayedColumns: string[] = [ 'moreDetails', @@ -79,87 +64,23 @@ export class ProcessGroupStatusTable implements AfterViewInit { 'actions' ]; - dataSource: MatTableDataSource = - new MatTableDataSource(); - - constructor(private nifiCommon: NiFiCommon) {} - - applyFilter(filter: SummaryTableFilterArgs) { - this.dataSource.filter = JSON.stringify(filter); - this.filteredCount = this.dataSource.filteredData.length; - this.resetPaginator(); - this.selectNone(); - } - - @Input() selectedProcessGroupId!: string; - - @Input() set initialSortColumn(initialSortColumn: SupportedColumns) { - this._initialSortColumn = initialSortColumn; - this.multiSort = { ...this.multiSort, active: initialSortColumn }; - } - - get initialSortColumn() { - return this._initialSortColumn; - } - - @Input() set initialSortDirection(initialSortDirection: SortDirection) { - this._initialSortDirection = initialSortDirection; - this.multiSort = { ...this.multiSort, direction: initialSortDirection }; - } - - get initialSortDirection() { - return this._initialSortDirection; + constructor(private nifiCommon: NiFiCommon) { + super(); } @Input() rootProcessGroup!: ProcessGroupStatusSnapshot; - @Input() set processGroups(processGroups: ProcessGroupStatusSnapshotEntity[]) { - if (processGroups) { - this.dataSource.data = this.sortEntities(processGroups, this.multiSort); - - this.dataSource.filterPredicate = (data: ProcessGroupStatusSnapshotEntity, filter: string): boolean => { - const { filterTerm, filterColumn } = JSON.parse(filter); - - if (filterTerm === '') { - return true; - } - - const field: string = data.processGroupStatusSnapshot[ - filterColumn as keyof ProcessGroupStatusSnapshot - ] as string; - return this.nifiCommon.stringContains(field, filterTerm, true); - }; - - this.totalCount = processGroups.length; - this.filteredCount = processGroups.length; - } - } - - @Input() summaryListingStatus: string | null = null; - @Input() loadedTimestamp: string | null = null; + override filterPredicate(data: ProcessGroupStatusSnapshotEntity, filter: string): boolean { + const { filterTerm, filterColumn } = JSON.parse(filter); - @Output() viewStatusHistory: EventEmitter = - new EventEmitter(); - @Output() selectProcessGroup: EventEmitter = - new EventEmitter(); - @Output() clearSelection: EventEmitter = new EventEmitter(); - @Output() refresh: EventEmitter = new EventEmitter(); - - @ViewChild(MatPaginator) paginator!: MatPaginator; - - ngAfterViewInit(): void { - this.dataSource.paginator = this.paginator; - } - - resetPaginator(): void { - if (this.dataSource.paginator) { - this.dataSource.paginator.firstPage(); + if (filterTerm === '') { + return true; } - } - paginationChanged(): void { - // clear out any selection - this.selectNone(); + const field: string = data.processGroupStatusSnapshot[ + filterColumn as keyof ProcessGroupStatusSnapshot + ] as string; + return this.nifiCommon.stringContains(field, filterTerm, true); } formatName(pg: ProcessGroupStatusSnapshotEntity): string { @@ -252,7 +173,7 @@ export class ProcessGroupStatusTable implements AfterViewInit { return 0; } - private supportsMultiValuedSort(sort: Sort): boolean { + override supportsMultiValuedSort(sort: Sort): boolean { switch (sort.active) { case 'transferred': case 'in': @@ -268,34 +189,7 @@ export class ProcessGroupStatusTable implements AfterViewInit { } } - private setMultiSort(sort: Sort) { - const { active, direction, sortValueIndex, totalValues } = this.multiSort; - - if (this.supportsMultiValuedSort(sort)) { - if (active === sort.active) { - // previous sort was of the same column - if (direction === 'desc' && sort.direction === 'asc') { - // change from previous index to the next - const newIndex = sortValueIndex + 1 >= totalValues ? 0 : sortValueIndex + 1; - this.multiSort = { ...sort, sortValueIndex: newIndex, totalValues }; - } else { - this.multiSort = { ...sort, sortValueIndex, totalValues }; - } - } else { - // sorting a different column, just reset - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } else { - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } - - sortData(sort: Sort) { - this.setMultiSort(sort); - this.dataSource.data = this.sortEntities(this.dataSource.data, sort); - } - - private sortEntities(data: ProcessGroupStatusSnapshotEntity[], sort: Sort): ProcessGroupStatusSnapshotEntity[] { + override sortEntities(data: ProcessGroupStatusSnapshotEntity[], sort: Sort): ProcessGroupStatusSnapshotEntity[] { if (!data) { return []; } @@ -438,24 +332,4 @@ export class ProcessGroupStatusTable implements AfterViewInit { getProcessGroupLink(pg: ProcessGroupStatusSnapshotEntity): string[] { return ['/process-groups', pg.id]; } - - select(pg: ProcessGroupStatusSnapshotEntity): void { - this.selectProcessGroup.next(pg); - } - - isSelected(pg: ProcessGroupStatusSnapshotEntity): boolean { - if (this.selectedProcessGroupId) { - return pg.id === this.selectedProcessGroupId; - } - return false; - } - - viewStatusHistoryClicked(event: MouseEvent, pg: ProcessGroupStatusSnapshotEntity): void { - event.stopPropagation(); - this.viewStatusHistory.next(pg); - } - - private selectNone() { - this.clearSelection.next(); - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.html index 2916eac3d09b..b09f131ac199 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.html @@ -21,14 +21,19 @@ } @else { } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.ts index cc56c69e246f..546f6fc2fdea 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-listing.component.ts @@ -21,20 +21,29 @@ import { selectProcessorIdFromRoute, selectProcessorStatus, selectProcessorStatusSnapshots, + selectSelectedClusterNode, selectSummaryListingLoadedTimestamp, selectSummaryListingStatus, selectViewStatusHistory } from '../../state/summary-listing/summary-listing.selectors'; -import { ProcessorStatusSnapshotEntity, SummaryListingState } from '../../state/summary-listing'; +import { SummaryListingState } from '../../state/summary-listing'; import { selectCurrentUser } from '../../../../state/current-user/current-user.selectors'; import { initialState } from '../../state/summary-listing/summary-listing.reducer'; import { getStatusHistoryAndOpenDialog } from '../../../../state/status-history/status-history.actions'; -import { ComponentType } from '../../../../state/shared'; -import { combineLatest, delay, filter, Subject, switchMap, take } from 'rxjs'; +import { ComponentType, isDefinedAndNotNull } from '../../../../state/shared'; +import { combineLatest, delay, filter, map, Subject, switchMap, take } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import * as SummaryListingActions from '../../state/summary-listing/summary-listing.actions'; +import * as ClusterStatusActions from '../../state/component-cluster-status/component-cluster-status.actions'; import { MatPaginator } from '@angular/material/paginator'; import { ProcessorStatusTable } from './processor-status-table/processor-status-table.component'; +import { + selectClusterSearchResults, + selectClusterSummary +} from '../../../../state/cluster-summary/cluster-summary.selectors'; +import { loadClusterSummary } from '../../../../state/cluster-summary/cluster-summary.actions'; +import { ProcessorStatusSnapshotEntity } from '../../state'; +import { NodeSearchResult } from '../../../../state/cluster-summary'; @Component({ selector: 'processor-status-listing', @@ -46,6 +55,15 @@ export class ProcessorStatusListing implements AfterViewInit { loadedTimestamp$ = this.store.select(selectSummaryListingLoadedTimestamp); summaryListingStatus$ = this.store.select(selectSummaryListingStatus); selectedProcessorId$ = this.store.select(selectProcessorIdFromRoute); + connectedToCluster$ = this.store.select(selectClusterSummary).pipe( + isDefinedAndNotNull(), + map((cluster) => cluster.connectedToCluster) + ); + clusterNodes$ = this.store.select(selectClusterSearchResults).pipe( + isDefinedAndNotNull(), + map((results) => results.nodeResults) + ); + selectedClusterNode$ = this.store.select(selectSelectedClusterNode); currentUser$ = this.store.select(selectCurrentUser); @@ -104,6 +122,7 @@ export class ProcessorStatusListing implements AfterViewInit { refreshSummaryListing() { this.store.dispatch(SummaryListingActions.loadSummaryListing({ recursive: true })); + this.store.dispatch(loadClusterSummary()); } viewStatusHistory(processor: ProcessorStatusSnapshotEntity): void { @@ -127,4 +146,19 @@ export class ProcessorStatusListing implements AfterViewInit { clearSelection() { this.store.dispatch(SummaryListingActions.clearProcessorStatusSelection()); } + + viewClusteredDetails(processor: ProcessorStatusSnapshotEntity): void { + this.store.dispatch( + ClusterStatusActions.loadComponentClusterStatusAndOpenDialog({ + request: { + id: processor.id, + componentType: ComponentType.Processor + } + }) + ); + } + + clusterNodeSelected(clusterNode: NodeSearchResult) { + this.store.dispatch(SummaryListingActions.selectClusterNode({ clusterNode })); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/_processor-status-table.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/_processor-status-table.component-theme.scss new file mode 100644 index 000000000000..0ec9510c2571 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/_processor-status-table.component-theme.scss @@ -0,0 +1,42 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@use 'sass:map'; +@use '@angular/material' as mat; + +@mixin nifi-theme($material-theme) { + // Get the color config from the theme. + $color-config: mat.get-color-config($material-theme); + + // Get the color palette from the color-config. + $accent-palette: map.get($color-config, 'accent'); + $primary-palette: map.get($color-config, 'primary'); + + // Get hues from palette + $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); + $primary-palette-contrast-300: mat.get-color-from-palette($primary-palette, '300-contrast'); + $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); + $primary-palette-900: mat.get-color-from-palette($primary-palette, 900); + + .processor-status-table { + .primary-node-only { + color: $accent-palette-A400; + background-color: $primary-palette-contrast-300; + border: 1px solid $primary-palette-300; + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html index 54da6350b76b..833c487863b8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html @@ -25,6 +25,8 @@ [filterableColumns]="filterableColumns" [includeStatusFilter]="true" [includePrimaryNodeOnlyFilter]="true" + [clusterNodes]="clusterNodes" + [selectedNode]="selectedClusterNode" (filterChanged)="applyFilter($event)">
@@ -60,7 +62,16 @@
Name
- {{ formatName(item) }} +
+ @if (item.processorStatusSnapshot.executionNode === 'PRIMARY') { +
+ P +
+ } +
{{ formatName(item) }}
+
@@ -248,6 +259,13 @@ class="pointer fa fa-area-chart" title="View Status History" (click)="viewStatusHistoryClicked($event, item)">
+ + @if (connectedToCluster) { +
+ } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.scss index fa1fa87a6616..e4f2e7fc8df4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.scss @@ -22,8 +22,19 @@ } .mat-column-actions { - width: 72px; - min-width: 72px; + width: 80px; + min-width: 80px; } } + .primary-node-only { + font-family: Roboto; + font-size: 10px; + font-weight: bold; + line-height: 14px; + width: 16px; + height: 16px; + border-radius: 8px; + float: left; + text-align: center; + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts index afab58678790..f83e659039fb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts @@ -15,21 +15,18 @@ * limitations under the License. */ -import { AfterViewInit, Component, EventEmitter, Input, Output, ViewChild } from '@angular/core'; -import { MatTableDataSource, MatTableModule } from '@angular/material/table'; -import { ProcessorStatusSnapshot, ProcessorStatusSnapshotEntity } from '../../../state/summary-listing'; -import { MatSortModule, Sort, SortDirection } from '@angular/material/sort'; -import { - SummaryTableFilterArgs, - SummaryTableFilterColumn -} from '../../common/summary-table-filter/summary-table-filter.component'; +import { Component } from '@angular/core'; +import { MatTableModule } from '@angular/material/table'; +import { MatSortModule, Sort } from '@angular/material/sort'; +import { SummaryTableFilterColumn } from '../../common/summary-table-filter/summary-table-filter.component'; import { RouterLink } from '@angular/router'; import { SummaryTableFilterModule } from '../../common/summary-table-filter/summary-table-filter.module'; import { NgClass } from '@angular/common'; import { ComponentType } from '../../../../../state/shared'; -import { MultiSort } from '../../common'; import { NiFiCommon } from '../../../../../service/nifi-common.service'; -import { MatPaginator, MatPaginatorModule } from '@angular/material/paginator'; +import { MatPaginatorModule } from '@angular/material/paginator'; +import { ProcessorStatusSnapshot, ProcessorStatusSnapshotEntity } from '../../../state'; +import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; export type SupportedColumns = 'name' | 'type' | 'processGroup' | 'runStatus' | 'in' | 'out' | 'readWrite' | 'tasks'; @@ -40,23 +37,11 @@ export type SupportedColumns = 'name' | 'type' | 'processGroup' | 'runStatus' | standalone: true, imports: [RouterLink, SummaryTableFilterModule, MatTableModule, MatSortModule, NgClass, MatPaginatorModule] }) -export class ProcessorStatusTable implements AfterViewInit { - private _initialSortColumn: SupportedColumns = 'name'; - private _initialSortDirection: SortDirection = 'asc'; - +export class ProcessorStatusTable extends ComponentStatusTable { filterableColumns: SummaryTableFilterColumn[] = [ { key: 'name', label: 'name' }, { key: 'type', label: 'type' } ]; - totalCount = 0; - filteredCount = 0; - - multiSort: MultiSort = { - active: this._initialSortColumn, - direction: this._initialSortDirection, - sortValueIndex: 0, - totalValues: 2 - }; displayedColumns: string[] = [ 'moreDetails', @@ -70,95 +55,9 @@ export class ProcessorStatusTable implements AfterViewInit { 'tasks', 'actions' ]; - dataSource: MatTableDataSource = - new MatTableDataSource(); - - @ViewChild(MatPaginator) paginator!: MatPaginator; - - constructor(private nifiCommon: NiFiCommon) {} - - ngAfterViewInit(): void { - this.dataSource.paginator = this.paginator; - } - applyFilter(filter: SummaryTableFilterArgs) { - this.dataSource.filter = JSON.stringify(filter); - this.filteredCount = this.dataSource.filteredData.length; - this.resetPaginator(); - this.selectNone(); - } - - @Input() selectedProcessorId!: string; - - @Input() set initialSortColumn(initialSortColumn: SupportedColumns) { - this._initialSortColumn = initialSortColumn; - this.multiSort = { ...this.multiSort, active: initialSortColumn }; - } - - get initialSortColumn() { - return this._initialSortColumn; - } - - @Input() set initialSortDirection(initialSortDirection: SortDirection) { - this._initialSortDirection = initialSortDirection; - this.multiSort = { ...this.multiSort, direction: initialSortDirection }; - } - - get initialSortDirection() { - return this._initialSortDirection; - } - - @Input() set processors(processors: ProcessorStatusSnapshotEntity[]) { - if (processors) { - this.dataSource.data = this.sortEntities(processors, this.multiSort); - this.dataSource.filterPredicate = (data: ProcessorStatusSnapshotEntity, filter: string): boolean => { - const { filterTerm, filterColumn, filterStatus, primaryOnly } = JSON.parse(filter); - const matchOnStatus: boolean = filterStatus !== 'All'; - - if (primaryOnly) { - if (data.processorStatusSnapshot.executionNode !== 'PRIMARY') { - return false; - } - } - if (matchOnStatus) { - if (data.processorStatusSnapshot.runStatus !== filterStatus) { - return false; - } - } - if (filterTerm === '') { - return true; - } - - const field: string = data.processorStatusSnapshot[ - filterColumn as keyof ProcessorStatusSnapshot - ] as string; - return this.nifiCommon.stringContains(field, filterTerm, true); - }; - - this.totalCount = processors.length; - this.filteredCount = processors.length; - } - } - - @Input() summaryListingStatus: string | null = null; - @Input() loadedTimestamp: string | null = null; - - @Output() refresh: EventEmitter = new EventEmitter(); - @Output() viewStatusHistory: EventEmitter = - new EventEmitter(); - @Output() selectProcessor: EventEmitter = - new EventEmitter(); - @Output() clearSelection: EventEmitter = new EventEmitter(); - - resetPaginator(): void { - if (this.dataSource.paginator) { - this.dataSource.paginator.firstPage(); - } - } - - paginationChanged(): void { - // clear out any selection - this.selectNone(); + constructor(private nifiCommon: NiFiCommon) { + super(); } formatName(processor: ProcessorStatusSnapshotEntity): string { @@ -220,12 +119,7 @@ export class ProcessorStatusTable implements AfterViewInit { } } - sortData(sort: Sort) { - this.setMultiSort(sort); - this.dataSource.data = this.sortEntities(this.dataSource.data, sort); - } - - private sortEntities(data: ProcessorStatusSnapshotEntity[], sort: Sort): ProcessorStatusSnapshotEntity[] { + override sortEntities(data: ProcessorStatusSnapshotEntity[], sort: Sort): ProcessorStatusSnapshotEntity[] { if (!data) { return []; } @@ -302,11 +196,7 @@ export class ProcessorStatusTable implements AfterViewInit { }); } - private compare(a: number | string, b: number | string, isAsc: boolean) { - return (a < b ? -1 : a > b ? 1 : 0) * (isAsc ? 1 : -1); - } - - private supportsMultiValuedSort(sort: Sort): boolean { + override supportsMultiValuedSort(sort: Sort): boolean { switch (sort.active) { case 'in': case 'out': @@ -318,45 +208,25 @@ export class ProcessorStatusTable implements AfterViewInit { } } - private setMultiSort(sort: Sort) { - const { active, direction, sortValueIndex, totalValues } = this.multiSort; + override filterPredicate(data: ProcessorStatusSnapshotEntity, filter: string): boolean { + const { filterTerm, filterColumn, filterStatus, primaryOnly } = JSON.parse(filter); + const matchOnStatus: boolean = filterStatus !== 'All'; - if (this.supportsMultiValuedSort(sort)) { - if (active === sort.active) { - // previous sort was of the same column - if (direction === 'desc' && sort.direction === 'asc') { - // change from previous index to the next - const newIndex = sortValueIndex + 1 >= totalValues ? 0 : sortValueIndex + 1; - this.multiSort = { ...sort, sortValueIndex: newIndex, totalValues }; - } else { - this.multiSort = { ...sort, sortValueIndex, totalValues }; - } - } else { - // sorting a different column, just reset - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; + if (primaryOnly) { + if (data.processorStatusSnapshot.executionNode !== 'PRIMARY') { + return false; } - } else { - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; } - } - - select(processor: ProcessorStatusSnapshotEntity): void { - this.selectProcessor.next(processor); - } - - isSelected(processor: ProcessorStatusSnapshotEntity): boolean { - if (this.selectedProcessorId) { - return processor.id === this.selectedProcessorId; + if (matchOnStatus) { + if (data.processorStatusSnapshot.runStatus !== filterStatus) { + return false; + } + } + if (filterTerm === '') { + return true; } - return false; - } - - viewStatusHistoryClicked(event: MouseEvent, processor: ProcessorStatusSnapshotEntity): void { - event.stopPropagation(); - this.viewStatusHistory.next(processor); - } - private selectNone() { - this.clearSelection.next(); + const field: string = data.processorStatusSnapshot[filterColumn as keyof ProcessorStatusSnapshot] as string; + return this.nifiCommon.stringContains(field, filterTerm, true); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.html index d5e655c08dad..40e77d95d1d8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.html @@ -22,14 +22,19 @@ } @else { } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.ts index b1981e02ea32..76872ff18589 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-listing.component.ts @@ -20,19 +20,28 @@ import { selectRemoteProcessGroupIdFromRoute, selectRemoteProcessGroupStatus, selectRemoteProcessGroupStatusSnapshots, + selectSelectedClusterNode, selectSummaryListingLoadedTimestamp, selectSummaryListingStatus, selectViewStatusHistory } from '../../state/summary-listing/summary-listing.selectors'; import { selectCurrentUser } from '../../../../state/current-user/current-user.selectors'; import { Store } from '@ngrx/store'; -import { RemoteProcessGroupStatusSnapshotEntity, SummaryListingState } from '../../state/summary-listing'; -import { filter, switchMap, take } from 'rxjs'; +import { SummaryListingState } from '../../state/summary-listing'; +import { filter, map, switchMap, take } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { getStatusHistoryAndOpenDialog } from '../../../../state/status-history/status-history.actions'; -import { ComponentType } from '../../../../state/shared'; +import { ComponentType, isDefinedAndNotNull } from '../../../../state/shared'; import { initialState } from '../../state/summary-listing/summary-listing.reducer'; import * as SummaryListingActions from '../../state/summary-listing/summary-listing.actions'; +import { loadClusterSummary } from '../../../../state/cluster-summary/cluster-summary.actions'; +import { RemoteProcessGroupStatusSnapshotEntity } from '../../state'; +import { + selectClusterSearchResults, + selectClusterSummary +} from '../../../../state/cluster-summary/cluster-summary.selectors'; +import * as ClusterStatusActions from '../../state/component-cluster-status/component-cluster-status.actions'; +import { NodeSearchResult } from '../../../../state/cluster-summary'; @Component({ selector: 'remote-process-group-status-listing', @@ -45,6 +54,15 @@ export class RemoteProcessGroupStatusListing { currentUser$ = this.store.select(selectCurrentUser); rpgStatusSnapshots$ = this.store.select(selectRemoteProcessGroupStatusSnapshots); selectedRpgId$ = this.store.select(selectRemoteProcessGroupIdFromRoute); + connectedToCluster$ = this.store.select(selectClusterSummary).pipe( + isDefinedAndNotNull(), + map((cluster) => cluster.connectedToCluster) + ); + clusterNodes$ = this.store.select(selectClusterSearchResults).pipe( + isDefinedAndNotNull(), + map((results) => results.nodeResults) + ); + selectedClusterNode$ = this.store.select(selectSelectedClusterNode); constructor(private store: Store) { this.store @@ -80,6 +98,7 @@ export class RemoteProcessGroupStatusListing { refreshSummaryListing() { this.store.dispatch(SummaryListingActions.loadSummaryListing({ recursive: true })); + this.store.dispatch(loadClusterSummary()); } selectRemoteProcessGroup(rpg: RemoteProcessGroupStatusSnapshotEntity): void { @@ -103,4 +122,19 @@ export class RemoteProcessGroupStatusListing { }) ); } + + viewClusteredDetails(processor: RemoteProcessGroupStatusSnapshotEntity): void { + this.store.dispatch( + ClusterStatusActions.loadComponentClusterStatusAndOpenDialog({ + request: { + id: processor.id, + componentType: ComponentType.RemoteProcessGroup + } + }) + ); + } + + clusterNodeSelected(clusterNode: NodeSearchResult) { + this.store.dispatch(SummaryListingActions.selectClusterNode({ clusterNode })); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html index 72cbfe80708e..a2754c9d7cf7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html @@ -26,6 +26,8 @@ [filterableColumns]="filterableColumns" [includeStatusFilter]="false" [includePrimaryNodeOnlyFilter]="false" + [clusterNodes]="clusterNodes" + [selectedNode]="selectedClusterNode" (filterChanged)="applyFilter($event)">
@@ -155,6 +157,13 @@ class="pointer fa fa-area-chart" title="View Status History" (click)="viewStatusHistoryClicked($event, item)">
+ + @if (connectedToCluster) { +
+ } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.scss index 6be7cb541dc1..80086e7ca0ff 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.scss @@ -23,8 +23,8 @@ } .mat-column-actions { - width: 72px; - min-width: 72px; + width: 80px; + min-width: 80px; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts index f75a8ea727fa..554c564bcb77 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts @@ -15,24 +15,18 @@ * limitations under the License. */ -import { AfterViewInit, Component, EventEmitter, Input, Output, ViewChild } from '@angular/core'; +import { Component } from '@angular/core'; import { CommonModule } from '@angular/common'; import { SummaryTableFilterModule } from '../../common/summary-table-filter/summary-table-filter.module'; -import { MatSortModule, Sort, SortDirection } from '@angular/material/sort'; -import { - SummaryTableFilterArgs, - SummaryTableFilterColumn -} from '../../common/summary-table-filter/summary-table-filter.component'; -import { MultiSort } from '../../common'; -import { MatTableDataSource, MatTableModule } from '@angular/material/table'; -import { - RemoteProcessGroupStatusSnapshot, - RemoteProcessGroupStatusSnapshotEntity -} from '../../../state/summary-listing'; +import { MatSortModule, Sort } from '@angular/material/sort'; +import { SummaryTableFilterColumn } from '../../common/summary-table-filter/summary-table-filter.component'; +import { MatTableModule } from '@angular/material/table'; import { NiFiCommon } from '../../../../../service/nifi-common.service'; import { ComponentType } from '../../../../../state/shared'; import { RouterLink } from '@angular/router'; -import { MatPaginator, MatPaginatorModule } from '@angular/material/paginator'; +import { MatPaginatorModule } from '@angular/material/paginator'; +import { RemoteProcessGroupStatusSnapshot, RemoteProcessGroupStatusSnapshotEntity } from '../../../state'; +import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; export type SupportedColumns = 'name' | 'uri' | 'transmitting' | 'sent' | 'received'; @@ -43,105 +37,28 @@ export type SupportedColumns = 'name' | 'uri' | 'transmitting' | 'sent' | 'recei templateUrl: './remote-process-group-status-table.component.html', styleUrls: ['./remote-process-group-status-table.component.scss'] }) -export class RemoteProcessGroupStatusTable implements AfterViewInit { - private _initialSortColumn: SupportedColumns = 'name'; - private _initialSortDirection: SortDirection = 'asc'; - +export class RemoteProcessGroupStatusTable extends ComponentStatusTable { filterableColumns: SummaryTableFilterColumn[] = [ { key: 'name', label: 'name' }, { key: 'targetUri', label: 'uri' } ]; - - totalCount = 0; - filteredCount = 0; - - multiSort: MultiSort = { - active: this._initialSortColumn, - direction: this._initialSortDirection, - sortValueIndex: 0, - totalValues: 2 - }; - displayedColumns: string[] = ['moreDetails', 'name', 'uri', 'transmitting', 'sent', 'received', 'actions']; - dataSource: MatTableDataSource = - new MatTableDataSource(); - - @ViewChild(MatPaginator) paginator!: MatPaginator; - - ngAfterViewInit(): void { - this.dataSource.paginator = this.paginator; - } - - constructor(private nifiCommon: NiFiCommon) {} - - @Input() set initialSortColumn(initialSortColumn: SupportedColumns) { - this._initialSortColumn = initialSortColumn; - this.multiSort = { ...this.multiSort, active: initialSortColumn }; - } - - get initialSortColumn() { - return this._initialSortColumn; + constructor(private nifiCommon: NiFiCommon) { + super(); } - @Input() set initialSortDirection(initialSortDirection: SortDirection) { - this._initialSortDirection = initialSortDirection; - this.multiSort = { ...this.multiSort, direction: initialSortDirection }; - } - - get initialSortDirection() { - return this._initialSortDirection; - } - - @Input() selectedRemoteProcessGroupId!: string; - - @Input() set remoteProcessGroups(rpgs: RemoteProcessGroupStatusSnapshotEntity[]) { - if (rpgs) { - this.dataSource.data = this.sortEntities(rpgs, this.multiSort); - this.dataSource.filterPredicate = (data: RemoteProcessGroupStatusSnapshotEntity, filter: string) => { - const { filterTerm, filterColumn } = JSON.parse(filter); - - if (filterTerm === '') { - return true; - } + override filterPredicate(data: RemoteProcessGroupStatusSnapshotEntity, filter: string): boolean { + const { filterTerm, filterColumn } = JSON.parse(filter); - const field: string = data.remoteProcessGroupStatusSnapshot[ - filterColumn as keyof RemoteProcessGroupStatusSnapshot - ] as string; - return this.nifiCommon.stringContains(field, filterTerm, true); - }; - - this.totalCount = rpgs.length; - this.filteredCount = rpgs.length; + if (filterTerm === '') { + return true; } - } - - @Input() summaryListingStatus: string | null = null; - @Input() loadedTimestamp: string | null = null; - @Output() refresh: EventEmitter = new EventEmitter(); - @Output() viewStatusHistory: EventEmitter = - new EventEmitter(); - @Output() selectRemoteProcessGroup: EventEmitter = - new EventEmitter(); - @Output() clearSelection: EventEmitter = new EventEmitter(); - - applyFilter(filter: SummaryTableFilterArgs) { - this.dataSource.filter = JSON.stringify(filter); - this.filteredCount = this.dataSource.filteredData.length; - this.resetPaginator(); - this.selectNone(); - } - - resetPaginator(): void { - if (this.dataSource.paginator) { - this.dataSource.paginator.firstPage(); - } - } - - paginationChanged(): void { - // clear out any selection - this.selectNone(); + const field: string = data.remoteProcessGroupStatusSnapshot[ + filterColumn as keyof RemoteProcessGroupStatusSnapshot + ] as string; + return this.nifiCommon.stringContains(field, filterTerm, true); } getRemoteProcessGroupLink(rpg: RemoteProcessGroupStatusSnapshotEntity): string[] { @@ -153,35 +70,10 @@ export class RemoteProcessGroupStatusTable implements AfterViewInit { ]; } - select(rpg: RemoteProcessGroupStatusSnapshotEntity) { - this.selectRemoteProcessGroup.next(rpg); - } - - private selectNone() { - this.clearSelection.next(); - } - - isSelected(rpg: RemoteProcessGroupStatusSnapshotEntity): boolean { - if (this.selectedRemoteProcessGroupId) { - return rpg.id === this.selectedRemoteProcessGroupId; - } - return false; - } - canRead(rpg: RemoteProcessGroupStatusSnapshotEntity): boolean { return rpg.canRead; } - sortData(sort: Sort) { - this.setMultiSort(sort); - this.dataSource.data = this.sortEntities(this.dataSource.data, sort); - } - - viewStatusHistoryClicked(event: MouseEvent, rpg: RemoteProcessGroupStatusSnapshotEntity): void { - event.stopPropagation(); - this.viewStatusHistory.next(rpg); - } - formatName(rpg: RemoteProcessGroupStatusSnapshotEntity): string { return rpg.remoteProcessGroupStatusSnapshot.name; } @@ -214,7 +106,7 @@ export class RemoteProcessGroupStatusTable implements AfterViewInit { } } - private supportsMultiValuedSort(sort: Sort): boolean { + override supportsMultiValuedSort(sort: Sort): boolean { switch (sort.active) { case 'sent': case 'received': @@ -224,29 +116,7 @@ export class RemoteProcessGroupStatusTable implements AfterViewInit { } } - private setMultiSort(sort: Sort) { - const { active, direction, sortValueIndex, totalValues } = this.multiSort; - - if (this.supportsMultiValuedSort(sort)) { - if (active === sort.active) { - // previous sort was of the same column - if (direction === 'desc' && sort.direction === 'asc') { - // change from previous index to the next - const newIndex = sortValueIndex + 1 >= totalValues ? 0 : sortValueIndex + 1; - this.multiSort = { ...sort, sortValueIndex: newIndex, totalValues }; - } else { - this.multiSort = { ...sort, sortValueIndex, totalValues }; - } - } else { - // sorting a different column, just reset - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } else { - this.multiSort = { ...sort, sortValueIndex: 0, totalValues }; - } - } - - private sortEntities( + override sortEntities( data: RemoteProcessGroupStatusSnapshotEntity[], sort: Sort ): RemoteProcessGroupStatusSnapshotEntity[] { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.spec.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.spec.ts new file mode 100644 index 000000000000..fbcac00d14e8 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.spec.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentTypeNamePipe } from './component-type-name.pipe'; + +describe('ComponentTypeNamePipe', () => { + it('create an instance', () => { + const pipe = new ComponentTypeNamePipe(); + expect(pipe).toBeTruthy(); + }); +}); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.ts new file mode 100644 index 000000000000..e76c950ae320 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pipes/component-type-name.pipe.ts @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Pipe, PipeTransform } from '@angular/core'; +import { ComponentType } from '../state/shared'; + +@Pipe({ + name: 'componentTypeName', + standalone: true +}) +export class ComponentTypeNamePipe implements PipeTransform { + transform(type: ComponentType): string { + switch (type) { + case ComponentType.Connection: + return 'Connection'; + case ComponentType.Processor: + return 'Processor'; + case ComponentType.OutputPort: + return 'Output Port'; + case ComponentType.InputPort: + return 'Input Port'; + case ComponentType.ProcessGroup: + return 'Process Group'; + case ComponentType.ControllerService: + return 'Controller Service'; + case ComponentType.Flow: + return 'Flow'; + case ComponentType.FlowAnalysisRule: + return 'Flow Analysis Rule'; + case ComponentType.FlowRegistryClient: + return 'Flow Registry Client'; + case ComponentType.Funnel: + return 'Funnel'; + case ComponentType.Label: + return 'Label'; + case ComponentType.ParameterProvider: + return 'Parameter Provider'; + case ComponentType.RemoteProcessGroup: + return 'Remote Process Group'; + case ComponentType.ReportingTask: + return 'Reporting Task'; + default: + return ''; + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.actions.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.actions.ts index 5adea058db44..92256d16b7ba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.actions.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.actions.ts @@ -16,7 +16,7 @@ */ import { createAction, props } from '@ngrx/store'; -import { LoadClusterSummaryResponse } from './index'; +import { ClusterSearchRequest, ClusterSearchResults, LoadClusterSummaryResponse } from './index'; const CLUSTER_SUMMARY_STATE_PREFIX = '[Cluster Summary State]'; @@ -37,3 +37,13 @@ export const clusterSummaryApiError = createAction( ); export const clearClusterSummaryApiError = createAction(`${CLUSTER_SUMMARY_STATE_PREFIX} Clear About Api Error`); + +export const searchCluster = createAction( + `${CLUSTER_SUMMARY_STATE_PREFIX} Search Cluster`, + props<{ request: ClusterSearchRequest }>() +); + +export const searchClusterSuccess = createAction( + `${CLUSTER_SUMMARY_STATE_PREFIX} Search Cluster Success`, + props<{ response: ClusterSearchResults }>() +); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.effects.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.effects.ts index cd6c4e8fbd35..4256ca868b08 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.effects.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.effects.ts @@ -16,16 +16,23 @@ */ import { Injectable } from '@angular/core'; -import { Actions, createEffect, ofType } from '@ngrx/effects'; +import { Actions, concatLatestFrom, createEffect, ofType } from '@ngrx/effects'; import * as ClusterSummaryActions from './cluster-summary.actions'; -import { asyncScheduler, catchError, from, interval, map, of, switchMap, takeUntil } from 'rxjs'; +import { asyncScheduler, catchError, filter, from, interval, map, of, switchMap, takeUntil } from 'rxjs'; import { ClusterService } from '../../service/cluster.service'; +import { selectClusterSummary } from './cluster-summary.selectors'; +import { isDefinedAndNotNull } from '../shared'; +import { Store } from '@ngrx/store'; +import { ClusterSummaryState } from './index'; +import { HttpErrorResponse } from '@angular/common/http'; +import * as ErrorActions from '../error/error.actions'; @Injectable() export class ClusterSummaryEffects { constructor( private actions$: Actions, - private clusterService: ClusterService + private clusterService: ClusterService, + private store: Store ) {} loadClusterSummary$ = createEffect(() => @@ -57,4 +64,29 @@ export class ClusterSummaryEffects { switchMap(() => of(ClusterSummaryActions.loadClusterSummary())) ) ); + + searchCluster$ = createEffect(() => + this.actions$.pipe( + ofType(ClusterSummaryActions.searchCluster), + map((action) => action.request), + concatLatestFrom(() => + this.store.select(selectClusterSummary).pipe( + isDefinedAndNotNull(), + filter((clusterSummary) => clusterSummary.connectedToCluster) + ) + ), + switchMap(([request]) => { + return from(this.clusterService.searchCluster(request.q)).pipe( + map((response) => + ClusterSummaryActions.searchClusterSuccess({ + response: response + }) + ), + catchError((errorResponse: HttpErrorResponse) => + of(ErrorActions.snackBarError({ error: errorResponse.error })) + ) + ); + }) + ) + ); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.reducer.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.reducer.ts index a7c66b97123e..d6fa003f09f8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.reducer.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.reducer.ts @@ -18,14 +18,16 @@ import { createReducer, on } from '@ngrx/store'; import { ClusterSummaryState } from './index'; import { - clusterSummaryApiError, clearClusterSummaryApiError, + clusterSummaryApiError, loadClusterSummary, - loadClusterSummarySuccess + loadClusterSummarySuccess, + searchClusterSuccess } from './cluster-summary.actions'; export const initialState: ClusterSummaryState = { clusterSummary: null, + searchResults: null, error: null, status: 'pending' }; @@ -51,5 +53,9 @@ export const clusterSummaryReducer = createReducer( ...state, error: null, status: 'pending' as const + })), + on(searchClusterSuccess, (state, { response }) => ({ + ...state, + searchResults: response })) ); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.selectors.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.selectors.ts index 634593f253e3..58afb3a3d739 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.selectors.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/cluster-summary.selectors.ts @@ -24,3 +24,8 @@ export const selectClusterSummary = createSelector( selectClusterSummaryState, (state: ClusterSummaryState) => state.clusterSummary ); + +export const selectClusterSearchResults = createSelector( + selectClusterSummaryState, + (state: ClusterSummaryState) => state.searchResults +); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/index.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/index.ts index 09ee6b0bb293..7e90e99ae263 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/index.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/cluster-summary/index.ts @@ -40,6 +40,11 @@ export interface ClusterSearchResults { export interface ClusterSummaryState { clusterSummary: ClusterSummary | null; + searchResults: ClusterSearchResults | null; error: string | null; status: 'pending' | 'loading' | 'error' | 'success'; } + +export interface ClusterSearchRequest { + q?: string; +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss new file mode 100644 index 000000000000..10194ed036e4 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss @@ -0,0 +1,62 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@use 'sass:map'; +@use '@angular/material' as mat; + +@mixin nifi-theme($material-theme, $canvas-theme) { + // Get the color config from the theme. + $color-config: mat.get-color-config($material-theme); + $canvas-color-config: mat.get-color-config($canvas-theme); + + // Get the color palette from the color-config. + $primary-palette: map.get($color-config, 'primary'); + $accent-palette: map.get($color-config, 'accent'); + $warn-palette: map.get($color-config, 'warn'); + $canvas-primary-palette: map.get($canvas-color-config, 'primary'); + + // Get hues from palette + $primary-palette-700: mat.get-color-from-palette($primary-palette, 700); + $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); + $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); + $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); + + .component-context { + .fa, + .icon { + color: $accent-palette-A400; + } + + .component-context-logo { + .icon { + color: $warn-palette-A200; + } + } + + .component-context-name { + color: $canvas-primary-palette-A200; + } + + .component-context-type { + color: $primary-palette-700; + } + + .component-context-id { + color: $warn-palette-A200; + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.html new file mode 100644 index 000000000000..439e668617fd --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.html @@ -0,0 +1,31 @@ + + +
+
+ +
+
{{ name }}
+
{{ type | componentTypeName }}
+
+
+ @if (id) { +
{{ id }}
+ } +
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.scss new file mode 100644 index 000000000000..5b9f87bf5983 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.scss @@ -0,0 +1,41 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.component-context { + .component-context-logo { + text-align: start; + + .icon { + font-size: 32px; + } + } + + .component-context-name { + font-size: 15px; + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + + .component-context-type { + font-size: 12px; + line-height: 16px; + } + + .component-context-id { + font-size: 12px; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.spec.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.spec.ts new file mode 100644 index 000000000000..51ae4b3490f5 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.spec.ts @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ComponentContext } from './component-context.component'; + +describe('ComponentContext', () => { + let component: ComponentContext; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [ComponentContext] + }).compileComponents(); + + fixture = TestBed.createComponent(ComponentContext); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.ts new file mode 100644 index 000000000000..fb327f774d31 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/component-context.component.ts @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, Input } from '@angular/core'; +import { ComponentType } from '../../../state/shared'; +import { ComponentTypeNamePipe } from '../../../pipes/component-type-name.pipe'; + +@Component({ + selector: 'component-context', + standalone: true, + imports: [ComponentTypeNamePipe], + templateUrl: './component-context.component.html', + styleUrl: './component-context.component.scss' +}) +export class ComponentContext { + private _componentType: ComponentType = ComponentType.Processor; + componentIconClass: string = ''; + + @Input() set type(type: ComponentType) { + this._componentType = type; + this.componentIconClass = this.getIconClassName(type); + } + + get type(): ComponentType { + return this._componentType; + } + + @Input() id: string | null = null; + @Input() name: string = ''; + + private getIconClassName(type: ComponentType) { + switch (type) { + case ComponentType.Connection: + return 'icon-connect'; + case ComponentType.Processor: + return 'icon-processor'; + case ComponentType.OutputPort: + return 'icon-port-out'; + case ComponentType.InputPort: + return 'icon-port-in'; + case ComponentType.ProcessGroup: + return 'icon-group'; + case ComponentType.Funnel: + return 'icon-funnel'; + case ComponentType.Label: + return 'icon-label'; + case ComponentType.RemoteProcessGroup: + return 'icon-group-remote'; + default: + return 'icon-connect'; + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss index 5f2586295721..bcc5f6a9f679 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss @@ -54,6 +54,7 @@ th { background-color: $primary-palette-500 !important; color: $canvas-primary-palette-900; + user-select: none; } tr:hover { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss index 524caae52fe3..63e2591decba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss @@ -62,6 +62,8 @@ @use 'app/ui/common/status-history/status-history.component-theme' as status-history; @use 'app/ui/common/tooltips/property-hint-tip/property-hint-tip.component-theme' as property-hint-tip; @use 'app/ui/common/provenance-event-dialog/provenance-event-dialog.component-theme' as provenance-event-dialog; +@use 'app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component-theme' as processor-status-table; +@use 'app/ui/common/component-context/component-context.component-theme' as component-context; // Plus imports for other components in your app. @use 'roboto-fontface/css/roboto/roboto-fontface.css'; @@ -513,6 +515,8 @@ $appFontPath: '~roboto-fontface/fonts'; @include status-history.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include property-hint-tip.nifi-theme($material-theme-light); @include provenance-event-dialog.nifi-theme($material-theme-light); +@include processor-status-table.nifi-theme($material-theme-light); +@include component-context.nifi-theme($material-theme-light, $nifi-canvas-theme-light); .dark-theme { // Include the dark theme color styles. @@ -563,4 +567,6 @@ $appFontPath: '~roboto-fontface/fonts'; @include status-history.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include property-hint-tip.nifi-theme($material-theme-dark); @include provenance-event-dialog.nifi-theme($material-theme-dark); + @include processor-status-table.nifi-theme($material-theme-dark); + @include component-context.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); } From f9c1c3f04291fc7de8add93ce4d485a38fed6b9d Mon Sep 17 00:00:00 2001 From: Scott Aslan Date: Thu, 29 Feb 2024 16:09:32 -0500 Subject: [PATCH 11/73] [NIFI-12778] manage remote ports (#8433) * [NIFI-12778] manage remote ports * update last refreshed timestamp and loadedTimestamp * address review feedback * final touches * address addition review comments * formatDuration check isDurationBlank This closes #8433 --- .../feature/flow-designer-routing.module.ts | 13 + .../service/canvas-context-menu.service.ts | 15 +- .../service/manage-remote-port.service.ts | 72 ++++ .../flow-designer/state/flow/flow.actions.ts | 8 +- .../flow-designer/state/flow/flow.effects.ts | 12 + .../pages/flow-designer/state/flow/index.ts | 8 + .../state/manage-remote-ports/index.ts | 100 +++++ .../manage-remote-ports.actions.ts | 77 ++++ .../manage-remote-ports.effects.ts | 277 ++++++++++++++ .../manage-remote-ports.reducer.ts | 75 ++++ .../manage-remote-ports.selectors.ts | 55 +++ .../controller-services.component.spec.ts | 3 +- .../_manage-remote-ports.component-theme.scss | 45 +++ .../edit-remote-port.component.html | 72 ++++ .../edit-remote-port.component.scss | 27 ++ .../edit-remote-port.component.spec.ts | 63 +++ .../edit-remote-port.component.ts | 107 ++++++ .../manage-remote-ports-routing.module.ts | 40 ++ .../manage-remote-ports.component.html | 240 ++++++++++++ .../manage-remote-ports.component.scss | 38 ++ .../manage-remote-ports.component.spec.ts | 56 +++ .../manage-remote-ports.component.ts | 358 ++++++++++++++++++ .../manage-remote-ports.module.ts | 52 +++ .../src/app/service/nifi-common.service.ts | 27 ++ .../src/main/nifi/src/assets/themes/nifi.scss | 2 +- .../main/nifi/src/assets/themes/purple.scss | 2 +- .../src/main/nifi/src/styles.scss | 7 + 27 files changed, 1843 insertions(+), 8 deletions(-) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manage-remote-port.service.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/index.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.actions.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.effects.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.reducer.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.selectors.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports-routing.module.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.spec.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.ts create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/feature/flow-designer-routing.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/feature/flow-designer-routing.module.ts index b19a2fa7937d..5e1c862606eb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/feature/flow-designer-routing.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/feature/flow-designer-routing.module.ts @@ -39,6 +39,19 @@ const routes: Routes = [ } ] }, + { + path: 'remote-process-group/:rpgId', + component: FlowDesigner, + children: [ + { + path: 'manage-remote-ports', + loadChildren: () => + import('../ui/manage-remote-ports/manage-remote-ports.module').then( + (m) => m.ManageRemotePortsModule + ) + } + ] + }, { path: '', component: RootGroupRedirector, canActivate: [rootGroupGuard] } ]; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts index 01e3f8e44059..d7e091bd0c7d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts @@ -32,6 +32,7 @@ import { navigateToEditComponent, navigateToEditCurrentProcessGroup, navigateToManageComponentPolicies, + navigateToManageRemotePorts, navigateToProvenanceForComponent, navigateToQueueListing, navigateToViewStatusHistoryForComponent, @@ -771,12 +772,20 @@ export class CanvasContextMenu implements ContextMenuDefinitionProvider { }, { condition: (selection: any) => { - return this.canvasUtils.isRemoteProcessGroup(selection); + return this.canvasUtils.canRead(selection) && this.canvasUtils.isRemoteProcessGroup(selection); }, clazz: 'fa fa-cloud', text: 'Manage remote ports', - action: () => { - // TODO - remotePorts + action: (selection: any) => { + const selectionData = selection.datum(); + + this.store.dispatch( + navigateToManageRemotePorts({ + request: { + id: selectionData.id + } + }) + ); } }, { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manage-remote-port.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manage-remote-port.service.ts new file mode 100644 index 000000000000..f923cdb8b259 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manage-remote-port.service.ts @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Injectable } from '@angular/core'; +import { Observable } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { NiFiCommon } from '../../../service/nifi-common.service'; +import { ConfigureRemotePortRequest, ToggleRemotePortTransmissionRequest } from '../state/manage-remote-ports'; +import { Client } from '../../../service/client.service'; +import { ComponentType } from '../../../state/shared'; + +@Injectable({ providedIn: 'root' }) +export class ManageRemotePortService { + private static readonly API: string = '../nifi-api'; + + constructor( + private httpClient: HttpClient, + private client: Client, + private nifiCommon: NiFiCommon + ) {} + + getRemotePorts(rpgId: string): Observable { + return this.httpClient.get(`${ManageRemotePortService.API}/remote-process-groups/${rpgId}`); + } + + updateRemotePort(configureRemotePortRequest: ConfigureRemotePortRequest): Observable { + const type = + configureRemotePortRequest.payload.type === ComponentType.InputPort ? 'input-ports' : 'output-ports'; + return this.httpClient.put( + `${this.nifiCommon.stripProtocol(configureRemotePortRequest.uri)}/${type}/${ + configureRemotePortRequest.payload.remoteProcessGroupPort.id + }`, + { + revision: configureRemotePortRequest.payload.revision, + remoteProcessGroupPort: configureRemotePortRequest.payload.remoteProcessGroupPort, + disconnectedNodeAcknowledged: configureRemotePortRequest.payload.disconnectedNodeAcknowledged + } + ); + } + + updateRemotePortTransmission( + toggleRemotePortTransmissionRequest: ToggleRemotePortTransmissionRequest + ): Observable { + const payload: any = { + revision: this.client.getRevision(toggleRemotePortTransmissionRequest.rpg), + disconnectedNodeAcknowledged: toggleRemotePortTransmissionRequest.disconnectedNodeAcknowledged, + state: toggleRemotePortTransmissionRequest.state + }; + + const type = + toggleRemotePortTransmissionRequest.type === ComponentType.InputPort ? 'input-ports' : 'output-ports'; + + return this.httpClient.put( + `${ManageRemotePortService.API}/remote-process-groups/${toggleRemotePortTransmissionRequest.rpg.id}/${type}/${toggleRemotePortTransmissionRequest.portId}/run-status`, + payload + ); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts index 7dfb10a63485..f2276dca012b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts @@ -76,7 +76,8 @@ import { ImportFromRegistryDialogRequest, ImportFromRegistryRequest, GoToRemoteProcessGroupRequest, - RefreshRemoteProcessGroupRequest + RefreshRemoteProcessGroupRequest, + RpgManageRemotePortsRequest } from './index'; import { StatusHistoryRequest } from '../../../../state/status-history'; @@ -400,6 +401,11 @@ export const openEditRemoteProcessGroupDialog = createAction( props<{ request: EditComponentDialogRequest }>() ); +export const navigateToManageRemotePorts = createAction( + `${CANVAS_PREFIX} Open Remote Process Group Manage Remote Ports`, + props<{ request: RpgManageRemotePortsRequest }>() +); + export const updateComponent = createAction( `${CANVAS_PREFIX} Update Component`, props<{ request: UpdateComponentRequest }>() diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts index cd0015d8dfa2..8c0716c826c1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts @@ -1319,6 +1319,18 @@ export class FlowEffects { { dispatch: false } ); + navigateToManageRemotePorts$ = createEffect( + () => + this.actions$.pipe( + ofType(FlowActions.navigateToManageRemotePorts), + map((action) => action.request), + tap((request) => { + this.router.navigate(['/remote-process-group', request.id, 'manage-remote-ports']); + }) + ), + { dispatch: false } + ); + openEditRemoteProcessGroupDialog$ = createEffect( () => this.actions$.pipe( diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/index.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/index.ts index 384451a8bc38..2d929ced3b50 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/index.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/flow/index.ts @@ -270,6 +270,14 @@ export interface EditComponentDialogRequest { entity: any; } +export interface EditRemotePortDialogRequest extends EditComponentDialogRequest { + rpg?: any; +} + +export interface RpgManageRemotePortsRequest { + id: string; +} + export interface NavigateToControllerServicesRequest { id: string; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/index.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/index.ts new file mode 100644 index 000000000000..4f547e56a965 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/index.ts @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentType } from '../../../../state/shared'; + +export const remotePortsFeatureKey = 'remotePortListing'; + +export interface PortSummary { + batchSettings: { + count?: number; + size?: string; + duration?: string; + }; + comments: string; + concurrentlySchedulableTaskCount: number; + connected: boolean; + exists: boolean; + groupId: string; + id: string; + name: string; + targetId: string; + targetRunning: boolean; + transmitting: boolean; + useCompression: boolean; + versionedComponentId: string; + type?: ComponentType.InputPort | ComponentType.OutputPort; +} + +export interface EditRemotePortDialogRequest { + id: string; + port: PortSummary; + rpg: any; +} + +export interface ToggleRemotePortTransmissionRequest { + rpg: any; + portId: string; + disconnectedNodeAcknowledged: boolean; + state: string; + type: ComponentType.InputPort | ComponentType.OutputPort | undefined; +} + +export interface StartRemotePortTransmissionRequest { + rpg: any; + port: PortSummary; +} + +export interface StopRemotePortTransmissionRequest { + rpg: any; + port: PortSummary; +} + +export interface LoadRemotePortsRequest { + rpgId: string; +} + +export interface LoadRemotePortsResponse { + ports: PortSummary[]; + rpg: any; + loadedTimestamp: string; +} + +export interface ConfigureRemotePortRequest { + id: string; + uri: string; + payload: any; + postUpdateNavigation?: string[]; +} + +export interface ConfigureRemotePortSuccess { + id: string; + port: any; +} + +export interface SelectRemotePortRequest { + rpgId: string; + id: string; +} + +export interface RemotePortsState { + ports: PortSummary[]; + saving: boolean; + rpg: any; + loadedTimestamp: string; + status: 'pending' | 'loading' | 'success'; +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.actions.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.actions.ts new file mode 100644 index 000000000000..a084dc5b2d9b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.actions.ts @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createAction, props } from '@ngrx/store'; +import { + ConfigureRemotePortRequest, + ConfigureRemotePortSuccess, + EditRemotePortDialogRequest, + LoadRemotePortsRequest, + LoadRemotePortsResponse, + SelectRemotePortRequest, + StartRemotePortTransmissionRequest, + StopRemotePortTransmissionRequest +} from './index'; + +export const resetRemotePortsState = createAction('[Manage Remote Ports] Reset Remote Ports State'); + +export const loadRemotePorts = createAction( + '[Manage Remote Ports] Load Remote Ports', + props<{ request: LoadRemotePortsRequest }>() +); + +export const loadRemotePortsSuccess = createAction( + '[Manage Remote Ports] Load Remote Ports Success', + props<{ response: LoadRemotePortsResponse }>() +); + +export const remotePortsBannerApiError = createAction( + '[Manage Remote Ports] Remote Ports Banner Api Error', + props<{ error: string }>() +); + +export const navigateToEditPort = createAction('[Manage Remote Ports] Navigate To Edit Port', props<{ id: string }>()); + +export const openConfigureRemotePortDialog = createAction( + '[Manage Remote Ports] Open Configure Port Dialog', + props<{ request: EditRemotePortDialogRequest }>() +); + +export const configureRemotePort = createAction( + '[Manage Remote Ports] Configure Port', + props<{ request: ConfigureRemotePortRequest }>() +); + +export const configureRemotePortSuccess = createAction( + '[Manage Remote Ports] Configure Port Success', + props<{ response: ConfigureRemotePortSuccess }>() +); + +export const startRemotePortTransmission = createAction( + '[Manage Remote Ports] Start Port Transmission', + props<{ request: StartRemotePortTransmissionRequest }>() +); + +export const stopRemotePortTransmission = createAction( + '[Manage Remote Ports] Stop Port Transmission', + props<{ request: StopRemotePortTransmissionRequest }>() +); + +export const selectRemotePort = createAction( + '[Manage Remote Ports] Select Port Summary', + props<{ request: SelectRemotePortRequest }>() +); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.effects.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.effects.ts new file mode 100644 index 000000000000..786635068806 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.effects.ts @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Injectable } from '@angular/core'; +import { Actions, concatLatestFrom, createEffect, ofType } from '@ngrx/effects'; +import * as ManageRemotePortsActions from './manage-remote-ports.actions'; +import { catchError, from, map, of, switchMap, tap } from 'rxjs'; +import { MatDialog } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { NiFiState } from '../../../../state'; +import { Router } from '@angular/router'; +import { selectRpg, selectRpgIdFromRoute, selectStatus } from './manage-remote-ports.selectors'; +import * as ErrorActions from '../../../../state/error/error.actions'; +import { ErrorHelper } from '../../../../service/error-helper.service'; +import { HttpErrorResponse } from '@angular/common/http'; +import { ManageRemotePortService } from '../../service/manage-remote-port.service'; +import { PortSummary } from './index'; +import { EditRemotePortComponent } from '../../ui/manage-remote-ports/edit-remote-port/edit-remote-port.component'; +import { EditRemotePortDialogRequest } from '../flow'; +import { ComponentType, isDefinedAndNotNull } from '../../../../state/shared'; +import { selectTimeOffset } from '../../../../state/flow-configuration/flow-configuration.selectors'; +import { selectAbout } from '../../../../state/about/about.selectors'; + +@Injectable() +export class ManageRemotePortsEffects { + constructor( + private actions$: Actions, + private store: Store, + private manageRemotePortService: ManageRemotePortService, + private errorHelper: ErrorHelper, + private dialog: MatDialog, + private router: Router + ) {} + + loadRemotePorts$ = createEffect(() => + this.actions$.pipe( + ofType(ManageRemotePortsActions.loadRemotePorts), + map((action) => action.request), + concatLatestFrom(() => [ + this.store.select(selectStatus), + this.store.select(selectTimeOffset).pipe(isDefinedAndNotNull()), + this.store.select(selectAbout).pipe(isDefinedAndNotNull()) + ]), + switchMap(([request, status, timeOffset, about]) => { + return this.manageRemotePortService.getRemotePorts(request.rpgId).pipe( + map((response) => { + // get the current user time to properly convert the server time + const now: Date = new Date(); + + // convert the user offset to millis + const userTimeOffset: number = now.getTimezoneOffset() * 60 * 1000; + + // create the proper date by adjusting by the offsets + const date: Date = new Date(Date.now() + userTimeOffset + timeOffset); + + const ports: PortSummary[] = []; + + response.component.contents.inputPorts.forEach((inputPort: PortSummary) => { + const port = { + ...inputPort, + type: ComponentType.InputPort + } as PortSummary; + + ports.push(port); + }); + + response.component.contents.outputPorts.forEach((outputPort: PortSummary) => { + const port = { + ...outputPort, + type: ComponentType.OutputPort + } as PortSummary; + + ports.push(port); + }); + + return ManageRemotePortsActions.loadRemotePortsSuccess({ + response: { + ports, + rpg: response, + loadedTimestamp: `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()} ${ + about.timezone + }` + } + }); + }), + catchError((errorResponse: HttpErrorResponse) => + of(this.errorHelper.handleLoadingError(status, errorResponse)) + ) + ); + }) + ) + ); + + navigateToEditPort$ = createEffect( + () => + this.actions$.pipe( + ofType(ManageRemotePortsActions.navigateToEditPort), + map((action) => action.id), + concatLatestFrom(() => this.store.select(selectRpgIdFromRoute)), + tap(([id, rpgId]) => { + this.router.navigate(['/remote-process-group', rpgId, 'manage-remote-ports', id, 'edit']); + }) + ), + { dispatch: false } + ); + + remotePortsBannerApiError$ = createEffect(() => + this.actions$.pipe( + ofType(ManageRemotePortsActions.remotePortsBannerApiError), + map((action) => action.error), + switchMap((error) => of(ErrorActions.addBannerError({ error }))) + ) + ); + + startRemotePortTransmission$ = createEffect(() => + this.actions$.pipe( + ofType(ManageRemotePortsActions.startRemotePortTransmission), + map((action) => action.request), + switchMap((request) => { + return this.manageRemotePortService + .updateRemotePortTransmission({ + portId: request.port.id, + rpg: request.rpg, + disconnectedNodeAcknowledged: false, + type: request.port.type, + state: 'TRANSMITTING' + }) + .pipe( + map((response) => { + return ManageRemotePortsActions.loadRemotePorts({ + request: { + rpgId: response.remoteProcessGroupPort.groupId + } + }); + }), + catchError((errorResponse: HttpErrorResponse) => + of(ErrorActions.snackBarError({ error: errorResponse.error })) + ) + ); + }) + ) + ); + + stopRemotePortTransmission$ = createEffect(() => + this.actions$.pipe( + ofType(ManageRemotePortsActions.stopRemotePortTransmission), + map((action) => action.request), + switchMap((request) => { + return this.manageRemotePortService + .updateRemotePortTransmission({ + portId: request.port.id, + rpg: request.rpg, + disconnectedNodeAcknowledged: false, + type: request.port.type, + state: 'STOPPED' + }) + .pipe( + map((response) => { + return ManageRemotePortsActions.loadRemotePorts({ + request: { + rpgId: response.remoteProcessGroupPort.groupId + } + }); + }), + catchError((errorResponse: HttpErrorResponse) => + of(ErrorActions.snackBarError({ error: errorResponse.error })) + ) + ); + }) + ) + ); + + selectRemotePort$ = createEffect( + () => + this.actions$.pipe( + ofType(ManageRemotePortsActions.selectRemotePort), + map((action) => action.request), + tap((request) => { + this.router.navigate(['/remote-process-group', request.rpgId, 'manage-remote-ports', request.id]); + }) + ), + { dispatch: false } + ); + + openConfigureRemotePortDialog$ = createEffect( + () => + this.actions$.pipe( + ofType(ManageRemotePortsActions.openConfigureRemotePortDialog), + map((action) => action.request), + concatLatestFrom(() => [this.store.select(selectRpg).pipe(isDefinedAndNotNull())]), + tap(([request, rpg]) => { + const portId: string = request.id; + + const editDialogReference = this.dialog.open(EditRemotePortComponent, { + data: { + type: request.port.type, + entity: request.port, + rpg + } as EditRemotePortDialogRequest, + id: portId + }); + + editDialogReference.afterClosed().subscribe((response) => { + this.store.dispatch(ErrorActions.clearBannerErrors()); + if (response != 'ROUTED') { + this.store.dispatch( + ManageRemotePortsActions.selectRemotePort({ + request: { + rpgId: rpg.id, + id: portId + } + }) + ); + } + }); + }) + ), + { dispatch: false } + ); + + configureRemotePort$ = createEffect(() => + this.actions$.pipe( + ofType(ManageRemotePortsActions.configureRemotePort), + map((action) => action.request), + switchMap((request) => + from(this.manageRemotePortService.updateRemotePort(request)).pipe( + map((response) => + ManageRemotePortsActions.configureRemotePortSuccess({ + response: { + id: request.id, + port: response.remoteProcessGroupPort + } + }) + ), + catchError((errorResponse: HttpErrorResponse) => { + if (this.errorHelper.showErrorInContext(errorResponse.status)) { + return of( + ManageRemotePortsActions.remotePortsBannerApiError({ + error: errorResponse.error + }) + ); + } else { + this.dialog.getDialogById(request.id)?.close('ROUTED'); + return of(this.errorHelper.fullScreenError(errorResponse)); + } + }) + ) + ) + ) + ); + + configureRemotePortSuccess$ = createEffect( + () => + this.actions$.pipe( + ofType(ManageRemotePortsActions.configureRemotePortSuccess), + map((action) => action.response), + tap(() => { + this.dialog.closeAll(); + }) + ), + { dispatch: false } + ); +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.reducer.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.reducer.ts new file mode 100644 index 000000000000..98b5f244d688 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.reducer.ts @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createReducer, on } from '@ngrx/store'; +import { + configureRemotePort, + configureRemotePortSuccess, + loadRemotePorts, + loadRemotePortsSuccess, + remotePortsBannerApiError, + resetRemotePortsState +} from './manage-remote-ports.actions'; +import { produce } from 'immer'; +import { RemotePortsState } from './index'; + +export const initialState: RemotePortsState = { + ports: [], + saving: false, + loadedTimestamp: '', + rpg: null, + status: 'pending' +}; + +export const manageRemotePortsReducer = createReducer( + initialState, + on(resetRemotePortsState, () => ({ + ...initialState + })), + on(loadRemotePorts, (state) => ({ + ...state, + status: 'loading' as const + })), + on(loadRemotePortsSuccess, (state, { response }) => ({ + ...state, + ports: response.ports, + loadedTimestamp: response.loadedTimestamp, + rpg: response.rpg, + status: 'success' as const + })), + on(remotePortsBannerApiError, (state) => ({ + ...state, + saving: false + })), + on(configureRemotePort, (state) => ({ + ...state, + saving: true + })), + on(configureRemotePortSuccess, (state, { response }) => { + return produce(state, (draftState) => { + const componentIndex: number = draftState.ports.findIndex((f: any) => response.id === f.id); + const port = { + ...response.port, + type: state.ports[componentIndex].type + }; + if (componentIndex > -1) { + draftState.ports[componentIndex] = port; + } + draftState.saving = false; + }); + }) +); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.selectors.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.selectors.ts new file mode 100644 index 000000000000..045f6c70c463 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/state/manage-remote-ports/manage-remote-ports.selectors.ts @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createFeatureSelector, createSelector } from '@ngrx/store'; +import { selectCurrentRoute } from '../../../../state/router/router.selectors'; +import { remotePortsFeatureKey, RemotePortsState } from './index'; + +export const selectRemotePortsState = createFeatureSelector(remotePortsFeatureKey); + +export const selectSaving = createSelector(selectRemotePortsState, (state: RemotePortsState) => state.saving); + +export const selectStatus = createSelector(selectRemotePortsState, (state: RemotePortsState) => state.status); + +export const selectRpgIdFromRoute = createSelector(selectCurrentRoute, (route) => { + if (route) { + // always select the rpg id from the route + return route.params.rpgId; + } + return null; +}); + +export const selectPortIdFromRoute = createSelector(selectCurrentRoute, (route) => { + if (route) { + // always select the port id from the route + return route.params.id; + } + return null; +}); + +export const selectSingleEditedPort = createSelector(selectCurrentRoute, (route) => { + if (route?.routeConfig?.path == 'edit') { + return route.params.id; + } + return null; +}); + +export const selectPorts = createSelector(selectRemotePortsState, (state: RemotePortsState) => state.ports); +export const selectRpg = createSelector(selectRemotePortsState, (state: RemotePortsState) => state.rpg); + +export const selectPort = (id: string) => + createSelector(selectPorts, (port: any[]) => port.find((port) => id == port.id)); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.component.spec.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.component.spec.ts index fccb8a666123..c384ba515ff3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.component.spec.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.component.spec.ts @@ -22,7 +22,6 @@ import { provideMockStore } from '@ngrx/store/testing'; import { initialState } from '../../state/controller-services/controller-services.reducer'; import { RouterTestingModule } from '@angular/router/testing'; import { Component } from '@angular/core'; -import { ControllerServicesModule } from './controller-services.module'; import { HttpClientTestingModule } from '@angular/common/http/testing'; describe('ControllerServices', () => { @@ -39,7 +38,7 @@ describe('ControllerServices', () => { beforeEach(() => { TestBed.configureTestingModule({ declarations: [ControllerServices], - imports: [RouterTestingModule, MockNavigation, ControllerServicesModule, HttpClientTestingModule], + imports: [RouterTestingModule, MockNavigation, HttpClientTestingModule], providers: [ provideMockStore({ initialState diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss new file mode 100644 index 000000000000..30dd724e1685 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@use 'sass:map'; +@use '@angular/material' as mat; + +@mixin nifi-theme($theme, $canvas-theme) { + // Get the color config from the theme. + $color-config: mat.get-color-config($theme); + $canvas-color-config: mat.get-color-config($canvas-theme); + + // Get the color palette from the color-config. + $primary-palette: map.get($color-config, 'primary'); + $canvas-accent-palette: map.get($canvas-color-config, 'accent'); + + // Get hues from palette + $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $canvas-accent-palette-A200: mat.get-color-from-palette($canvas-accent-palette, 'A200'); + + .manage-remote-ports-header { + color: $primary-palette-500; + } + + .manage-remote-ports-table { + .listing-table { + .fa.fa-warning { + color: $canvas-accent-palette-A200; + } + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html new file mode 100644 index 000000000000..d2cf80e4d60a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html @@ -0,0 +1,72 @@ + + +

Edit Remote {{ portTypeLabel }}

+
+ + +
+
+
Name
+
+ {{ request.entity.name }} +
+
+
+
+ + Concurrent Tasks + + +
+
+ Compressed + +
+
+ + Batch Count + + +
+
+ + Batch Size + + +
+
+ + Batch Duration + + +
+ + @if ({ value: (saving$ | async)! }; as saving) { + + + + + } +
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.scss new file mode 100644 index 000000000000..b4a795bb966a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.scss @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@use '@angular/material' as mat; + +.edit-remote-port-form { + @include mat.button-density(-1); + + width: 500px; + .mat-mdc-form-field { + width: 100%; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.spec.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.spec.ts new file mode 100644 index 000000000000..078fb0e334d6 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.spec.ts @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { EditRemotePortComponent } from './edit-remote-port.component'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { provideMockStore } from '@ngrx/store/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { EditComponentDialogRequest } from '../../../state/flow'; +import { ComponentType } from '../../../../../state/shared'; +import { initialState } from '../../../state/manage-remote-ports/manage-remote-ports.reducer'; + +describe('EditRemotePortComponent', () => { + let component: EditRemotePortComponent; + let fixture: ComponentFixture; + + const data: EditComponentDialogRequest = { + type: ComponentType.OutputPort, + uri: 'https://localhost:4200/nifi-api/remote-process-groups/95a4b210-018b-1000-772a-5a9ebfa03287', + entity: { + id: 'a687e30e-018b-1000-f904-849a9f8e6bdb', + groupId: '95a4b210-018b-1000-772a-5a9ebfa03287', + name: 'out', + transmitting: false, + concurrentlySchedulableTaskCount: 1, + useCompression: true, + batchSettings: { + count: '', + size: '', + duration: '' + } + } + }; + + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [EditRemotePortComponent, NoopAnimationsModule], + providers: [{ provide: MAT_DIALOG_DATA, useValue: data }, provideMockStore({ initialState })] + }); + fixture = TestBed.createComponent(EditRemotePortComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.ts new file mode 100644 index 000000000000..d07f664d0d4f --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.ts @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogModule } from '@angular/material/dialog'; +import { FormBuilder, FormControl, FormGroup, ReactiveFormsModule, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { MatInputModule } from '@angular/material/input'; +import { MatCheckboxModule } from '@angular/material/checkbox'; +import { MatButtonModule } from '@angular/material/button'; +import { AsyncPipe } from '@angular/common'; +import { ErrorBanner } from '../../../../../ui/common/error-banner/error-banner.component'; +import { NifiSpinnerDirective } from '../../../../../ui/common/spinner/nifi-spinner.directive'; +import { selectSaving } from '../../../state/manage-remote-ports/manage-remote-ports.selectors'; +import { EditRemotePortDialogRequest } from '../../../state/flow'; +import { Client } from '../../../../../service/client.service'; +import { ComponentType } from '../../../../../state/shared'; +import { PortSummary } from '../../../state/manage-remote-ports'; +import { configureRemotePort } from '../../../state/manage-remote-ports/manage-remote-ports.actions'; + +@Component({ + standalone: true, + templateUrl: './edit-remote-port.component.html', + imports: [ + ReactiveFormsModule, + ErrorBanner, + MatDialogModule, + MatInputModule, + MatCheckboxModule, + MatButtonModule, + AsyncPipe, + NifiSpinnerDirective + ], + styleUrls: ['./edit-remote-port.component.scss'] +}) +export class EditRemotePortComponent { + saving$ = this.store.select(selectSaving); + + editPortForm: FormGroup; + portTypeLabel: string; + + constructor( + @Inject(MAT_DIALOG_DATA) public request: EditRemotePortDialogRequest, + private formBuilder: FormBuilder, + private store: Store, + private client: Client + ) { + // set the port type name + if (ComponentType.InputPort == this.request.type) { + this.portTypeLabel = 'Input Port'; + } else { + this.portTypeLabel = 'Output Port'; + } + + // build the form + this.editPortForm = this.formBuilder.group({ + concurrentTasks: new FormControl(request.entity.concurrentlySchedulableTaskCount, Validators.required), + compressed: new FormControl(request.entity.useCompression), + count: new FormControl(request.entity.batchSettings.count), + size: new FormControl(request.entity.batchSettings.size), + duration: new FormControl(request.entity.batchSettings.duration) + }); + } + + editRemotePort() { + const payload: any = { + revision: this.client.getRevision(this.request.rpg), + disconnectedNodeAcknowledged: false, + type: this.request.type, + remoteProcessGroupPort: { + concurrentlySchedulableTaskCount: this.editPortForm.get('concurrentTasks')?.value, + useCompression: this.editPortForm.get('compressed')?.value, + batchSettings: { + count: this.editPortForm.get('count')?.value, + size: this.editPortForm.get('size')?.value, + duration: this.editPortForm.get('duration')?.value + }, + id: this.request.entity.id, + groupId: this.request.entity.groupId + } as PortSummary + }; + + this.store.dispatch( + configureRemotePort({ + request: { + id: this.request.entity.id, + uri: this.request.rpg.uri, + payload + } + }) + ); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports-routing.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports-routing.module.ts new file mode 100644 index 000000000000..db63639f5c22 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports-routing.module.ts @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; +import { ManageRemotePorts } from './manage-remote-ports.component'; + +const routes: Routes = [ + { + path: '', + component: ManageRemotePorts, + children: [ + { + path: ':id', + component: ManageRemotePorts, + children: [{ path: 'edit', component: ManageRemotePorts }] + } + ] + } +]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule] +}) +export class ManageRemotePortsRoutingModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html new file mode 100644 index 000000000000..a4647849b599 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html @@ -0,0 +1,240 @@ + + +
+
+ +
+
+

Manage Remote Ports

+ @if (portsState$ | async; as portsState) { +
+
+
+
Name
+
+ {{ portsState.rpg?.id }} +
+
+
+
+
+
Urls
+
+ {{ portsState.rpg?.component?.targetUri }} +
+
+
+
+ @if (isInitialLoading(portsState)) { +
+ +
+ } @else { +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ @if (hasComments(item)) { +
+
+
+ } + @if (portExists(item)) { +
+
+
+ } +
+ 		+
Name
+ 		+ {{ formatName(item) }} + +
Type
+ 		+ {{ formatType(item) }} + +
+ Concurrent Tasks +
+ 		+ + {{ formatTasks(item) }} + + +
+ Compressed +
+ 		+ {{ formatCompression(item) }} + +
+ Batch Count +
+ 		+ + {{ formatCount(item) }} + + +
+ Batch Size +
+ 		+ + {{ formatSize(item) }} + + +
+ Batch Duration +
+ 		+ + {{ formatDuration(item) }} + + +
+ @if ( + port.exists === true && + port.connected === true && + port.transmitting === false + ) { +
+ } + + @if (currentRpg) { + @if (port.transmitting) { +
+ } @else { + @if (port.connected && port.exists) { +
+ } + } + } +
+
+
+
+
+
+
+ +
Last updated:
+
{{ portsState.loadedTimestamp }}
+
+
+
+ } + } +
+
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.scss new file mode 100644 index 000000000000..c998c7780177 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.scss @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@use '@angular/material' as mat; + +.refresh-container { + line-height: normal; +} + +.manage-remote-ports-table.listing-table { + @include mat.table-density(-4); + + table { + .mat-column-moreDetails { + width: 32px; + min-width: 32px; + } + + .mat-column-actions { + width: 75px; + min-width: 75px; + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.spec.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.spec.ts new file mode 100644 index 000000000000..987754a9009a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.spec.ts @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ManageRemotePorts } from './manage-remote-ports.component'; +import { provideMockStore } from '@ngrx/store/testing'; +import { RouterTestingModule } from '@angular/router/testing'; +import { Component } from '@angular/core'; +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { initialState } from '../../state/manage-remote-ports/manage-remote-ports.reducer'; + +describe('ManageRemotePorts', () => { + let component: ManageRemotePorts; + let fixture: ComponentFixture; + + @Component({ + selector: 'navigation', + standalone: true, + template: '' + }) + class MockNavigation {} + + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [ManageRemotePorts], + imports: [RouterTestingModule, MockNavigation, HttpClientTestingModule], + providers: [ + provideMockStore({ + initialState + }) + ] + }); + fixture = TestBed.createComponent(ManageRemotePorts); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.ts new file mode 100644 index 000000000000..6e7388e30de2 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.ts @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { Store } from '@ngrx/store'; +import { filter, switchMap, take, tap } from 'rxjs'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { + selectRemotePortsState, + selectPort, + selectPortIdFromRoute, + selectPorts, + selectRpg, + selectRpgIdFromRoute, + selectSingleEditedPort +} from '../../state/manage-remote-ports/manage-remote-ports.selectors'; +import { RemotePortsState, PortSummary } from '../../state/manage-remote-ports'; +import { + loadRemotePorts, + navigateToEditPort, + openConfigureRemotePortDialog, + resetRemotePortsState, + selectRemotePort, + startRemotePortTransmission, + stopRemotePortTransmission +} from '../../state/manage-remote-ports/manage-remote-ports.actions'; +import { initialState } from '../../state/manage-remote-ports/manage-remote-ports.reducer'; +import { isDefinedAndNotNull, TextTipInput } from '../../../../state/shared'; +import { selectCurrentUser } from '../../../../state/current-user/current-user.selectors'; +import { NiFiState } from '../../../../state'; +import { NiFiCommon } from '../../../../service/nifi-common.service'; +import { MatTableDataSource } from '@angular/material/table'; +import { Sort } from '@angular/material/sort'; +import { TextTip } from '../../../../ui/common/tooltips/text-tip/text-tip.component'; +import { concatLatestFrom } from '@ngrx/effects'; +import { loadFlowConfiguration } from '../../../../state/flow-configuration/flow-configuration.actions'; +import { + selectFlowConfiguration, + selectTimeOffset +} from '../../../../state/flow-configuration/flow-configuration.selectors'; +import { selectAbout } from '../../../../state/about/about.selectors'; +import { loadAbout } from '../../../../state/about/about.actions'; + +@Component({ + templateUrl: './manage-remote-ports.component.html', + styleUrls: ['./manage-remote-ports.component.scss'] +}) +export class ManageRemotePorts implements OnInit, OnDestroy { + initialSortColumn: 'name' | 'type' | 'tasks' | 'count' | 'size' | 'duration' | 'compression' | 'actions' = 'name'; + initialSortDirection: 'asc' | 'desc' = 'asc'; + activeSort: Sort = { + active: this.initialSortColumn, + direction: this.initialSortDirection + }; + portsState$ = this.store.select(selectRemotePortsState); + selectedRpgId$ = this.store.select(selectRpgIdFromRoute); + selectedPortId!: string; + currentUser$ = this.store.select(selectCurrentUser); + flowConfiguration$ = this.store.select(selectFlowConfiguration).pipe(isDefinedAndNotNull()); + displayedColumns: string[] = [ + 'moreDetails', + 'name', + 'type', + 'tasks', + 'count', + 'size', + 'duration', + 'compression', + 'actions' + ]; + dataSource: MatTableDataSource = new MatTableDataSource([]); + protected readonly TextTip = TextTip; + + private currentRpgId!: string; + protected currentRpg: any | null = null; + + constructor( + private store: Store, + private nifiCommon: NiFiCommon + ) { + // load the ports after the flow configuration `timeOffset` and about `timezone` are loaded into the store + this.store + .select(selectTimeOffset) + .pipe( + isDefinedAndNotNull(), + switchMap(() => this.store.select(selectAbout)), + isDefinedAndNotNull(), + switchMap(() => this.store.select(selectRpgIdFromRoute)), + tap((rpgId) => (this.currentRpgId = rpgId)), + takeUntilDestroyed() + ) + .subscribe((rpgId) => { + this.store.dispatch( + loadRemotePorts({ + request: { + rpgId + } + }) + ); + }); + + // track selection using the port id from the route + this.store + .select(selectPortIdFromRoute) + .pipe(isDefinedAndNotNull(), takeUntilDestroyed()) + .subscribe((portId) => { + this.selectedPortId = portId; + }); + + // data for table + this.store + .select(selectPorts) + .pipe(isDefinedAndNotNull(), takeUntilDestroyed()) + .subscribe((ports) => { + this.dataSource = new MatTableDataSource(this.sortEntities(ports, this.activeSort)); + }); + + // the current RPG Entity + this.store + .select(selectRpg) + .pipe( + isDefinedAndNotNull(), + tap((rpg) => (this.currentRpg = rpg)), + takeUntilDestroyed() + ) + .subscribe(); + + // handle editing remote port deep link + this.store + .select(selectSingleEditedPort) + .pipe( + isDefinedAndNotNull(), + switchMap((id: string) => + this.store.select(selectPort(id)).pipe( + filter((entity) => entity != null), + take(1) + ) + ), + concatLatestFrom(() => [this.store.select(selectRpg).pipe(isDefinedAndNotNull())]), + takeUntilDestroyed() + ) + .subscribe(([entity, rpg]) => { + if (entity) { + this.store.dispatch( + openConfigureRemotePortDialog({ + request: { + id: entity.id, + port: entity, + rpg + } + }) + ); + } + }); + } + + ngOnInit(): void { + this.store.dispatch(loadFlowConfiguration()); + this.store.dispatch(loadAbout()); + } + + isInitialLoading(state: RemotePortsState): boolean { + // using the current timestamp to detect the initial load event + return state.loadedTimestamp == initialState.loadedTimestamp; + } + + refreshManageRemotePortsListing(): void { + this.store.dispatch( + loadRemotePorts({ + request: { + rpgId: this.currentRpgId + } + }) + ); + } + + formatName(entity: PortSummary): string { + return entity.name; + } + + formatTasks(entity: PortSummary): string { + return entity.concurrentlySchedulableTaskCount ? `${entity.concurrentlySchedulableTaskCount}` : 'No value set'; + } + + formatCount(entity: PortSummary): string { + if (!this.isCountBlank(entity)) { + return `${entity.batchSettings.count}`; + } + return 'No value set'; + } + + isCountBlank(entity: PortSummary): boolean { + return this.nifiCommon.isUndefined(entity.batchSettings.count); + } + + formatSize(entity: PortSummary): string { + if (!this.isSizeBlank(entity)) { + return `${entity.batchSettings.size}`; + } + return 'No value set'; + } + + isSizeBlank(entity: PortSummary): boolean { + return this.nifiCommon.isBlank(entity.batchSettings.size); + } + + formatDuration(entity: PortSummary): string { + if (!this.isDurationBlank(entity)) { + return `${entity.batchSettings.duration}`; + } + return 'No value set'; + } + + isDurationBlank(entity: PortSummary): boolean { + return this.nifiCommon.isBlank(entity.batchSettings.duration); + } + + formatCompression(entity: PortSummary): string { + return entity.useCompression ? 'Yes' : 'No'; + } + + formatType(entity: PortSummary): string { + return entity.type || ''; + } + + configureClicked(port: PortSummary, event: MouseEvent): void { + event.stopPropagation(); + this.store.dispatch( + navigateToEditPort({ + id: port.id + }) + ); + } + + hasComments(entity: PortSummary): boolean { + return !this.nifiCommon.isBlank(entity.comments); + } + + portExists(entity: PortSummary): boolean { + return !entity.exists; + } + + getCommentsTipData(entity: PortSummary): TextTipInput { + return { + text: entity.comments + }; + } + + getDisconnectedTipData(): TextTipInput { + return { + text: 'This port has been removed.' + }; + } + + toggleTransmission(port: PortSummary): void { + if (this.currentRpg) { + if (port.transmitting) { + this.store.dispatch( + stopRemotePortTransmission({ + request: { + rpg: this.currentRpg, + port + } + }) + ); + } else { + if (port.connected && port.exists) { + this.store.dispatch( + startRemotePortTransmission({ + request: { + rpg: this.currentRpg, + port + } + }) + ); + } + } + } + } + + select(entity: PortSummary): void { + this.store.dispatch( + selectRemotePort({ + request: { + rpgId: this.currentRpgId, + id: entity.id + } + }) + ); + } + + isSelected(entity: any): boolean { + if (this.selectedPortId) { + return entity.id == this.selectedPortId; + } + return false; + } + + sortData(sort: Sort) { + this.activeSort = sort; + this.dataSource.data = this.sortEntities(this.dataSource.data, sort); + } + + private sortEntities(data: PortSummary[], sort: Sort): PortSummary[] { + if (!data) { + return []; + } + return data.slice().sort((a, b) => { + const isAsc = sort.direction === 'asc'; + let retVal = 0; + + switch (sort.active) { + case 'name': + retVal = this.nifiCommon.compareString(this.formatName(a), this.formatName(b)); + break; + case 'type': + retVal = this.nifiCommon.compareString(this.formatType(a), this.formatType(b)); + break; + case 'tasks': + retVal = this.nifiCommon.compareString(this.formatTasks(a), this.formatTasks(b)); + break; + case 'compression': + retVal = this.nifiCommon.compareString(this.formatCompression(a), this.formatCompression(b)); + break; + case 'count': + retVal = this.nifiCommon.compareString(this.formatCount(a), this.formatCount(b)); + break; + case 'size': + retVal = this.nifiCommon.compareString(this.formatSize(a), this.formatSize(b)); + break; + case 'duration': + retVal = this.nifiCommon.compareString(this.formatDuration(a), this.formatDuration(b)); + break; + default: + return 0; + } + return retVal * (isAsc ? 1 : -1); + }); + } + + ngOnDestroy(): void { + this.store.dispatch(resetRemotePortsState()); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts new file mode 100644 index 000000000000..9578ec9ce8ee --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { ManageRemotePorts } from './manage-remote-ports.component'; +import { NgxSkeletonLoaderModule } from 'ngx-skeleton-loader'; +import { ControllerServiceTable } from '../../../../ui/common/controller-service/controller-service-table/controller-service-table.component'; +import { ManageRemotePortsRoutingModule } from './manage-remote-ports-routing.module'; +import { Breadcrumbs } from '../common/breadcrumbs/breadcrumbs.component'; +import { Navigation } from '../../../../ui/common/navigation/navigation.component'; +import { MatTableModule } from '@angular/material/table'; +import { MatSortModule } from '@angular/material/sort'; +import { NifiTooltipDirective } from '../../../../ui/common/tooltips/nifi-tooltip.directive'; +import { StoreModule } from '@ngrx/store'; +import { EffectsModule } from '@ngrx/effects'; +import { ManageRemotePortsEffects } from '../../state/manage-remote-ports/manage-remote-ports.effects'; +import { remotePortsFeatureKey } from '../../state/manage-remote-ports'; +import { manageRemotePortsReducer } from '../../state/manage-remote-ports/manage-remote-ports.reducer'; + +@NgModule({ + declarations: [ManageRemotePorts], + exports: [ManageRemotePorts], + imports: [ + CommonModule, + NgxSkeletonLoaderModule, + ManageRemotePortsRoutingModule, + StoreModule.forFeature(remotePortsFeatureKey, manageRemotePortsReducer), + EffectsModule.forFeature(ManageRemotePortsEffects), + ControllerServiceTable, + Breadcrumbs, + Navigation, + MatTableModule, + MatSortModule, + NifiTooltipDirective + ] +}) +export class ManageRemotePortsModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/service/nifi-common.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/service/nifi-common.service.ts index bbe69cfdcd1d..be8e1b4c12f5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/service/nifi-common.service.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/service/nifi-common.service.ts @@ -184,6 +184,33 @@ export class NiFiCommon { return true; } + /** + * Determines if the specified object is defined and not null. + * + * @argument {object} obj The object to test + */ + public isDefinedAndNotNull(obj: any) { + return !this.isUndefined(obj) && !this.isNull(obj); + } + + /** + * Determines if the specified object is undefined. + * + * @argument {object} obj The object to test + */ + public isUndefined(obj: any) { + return typeof obj === 'undefined'; + } + + /** + * Determines if the specified object is null. + * + * @argument {object} obj The object to test + */ + public isNull(obj: any) { + return obj === null; + } + /** * Determines if the specified array is empty. If the specified arg is not an * array, then true is returned. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss index 34ada14216ce..b8ff67946d4c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss @@ -170,7 +170,7 @@ $nifi-canvas-dark-palette: ( 500: #acacac, // g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .cdk-drag-disabled, .resizable-triangle 600: #545454, // .canvas-background, .navigation-control, .operation-control, .lineage 700: #696060, // .canvas-background, g.component rect.body.unauthorized, g.component rect.processor-icon-container.unauthorized, g.connection rect.body.unauthorized, #birdseye, .lineage - 800: rgba(#6b6464, 0.5), // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats + 800: rgba(#6b6464, 1), // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats 900: rgba(#252424, 0.97), // circle.flowfile-link, .processor-read-write-stats, .process-group-stats-in-out, .tooltip, .property-editor, .disabled, .enabled, .stopped, .running, .has-errors, .invalid, .validating, .transmitting, .not-transmitting, .up-to-date, .locally-modified, .sync-failure, .stale, .locally-modified-and-stale, g.component rect.body, text.bulletin-icon, rect.processor-icon-container, circle.restricted-background, circle.is-primary-background, g.connection rect.body, text.connection-to-run-status, text.expiration-icon, text.load-balance-icon, text.penalized-icon, g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down, text.version-control, .breadcrumb-container, #birdseye, .controller-bulletins .fa, .search-container:hover, .search-container.open, .login-background, table th, .mat-sort-header-arrow, .CodeMirror, #status-history-chart-container, #status-history-chart-control-container, #status-history-chart-control-container, // some analog colors for headers and hover states, inputs, stats, etc diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss index fe59f6521071..40eb88e23a12 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss @@ -171,7 +171,7 @@ $nifi-canvas-dark-palette: ( 500: #acacac, // g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .cdk-drag-disabled, .resizable-triangle 600: #545454, // .canvas-background, .navigation-control, .operation-control, .lineage 700: #696060, // .canvas-background, g.component rect.body.unauthorized, g.component rect.processor-icon-container.unauthorized, g.connection rect.body.unauthorized, #birdseye, .lineage - 800: rgba(#6b6464, 0.5), // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats + 800: rgba(#6b6464, 1), // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats 900: rgba(#252424, 0.97), // circle.flowfile-link, .processor-read-write-stats, .process-group-stats-in-out, .tooltip, .property-editor, .disabled, .enabled, .stopped, .running, .has-errors, .invalid, .validating, .transmitting, .not-transmitting, .up-to-date, .locally-modified, .sync-failure, .stale, .locally-modified-and-stale, g.component rect.body, text.bulletin-icon, rect.processor-icon-container, circle.restricted-background, circle.is-primary-background, g.connection rect.body, text.connection-to-run-status, text.expiration-icon, text.load-balance-icon, text.penalized-icon, g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down, text.version-control, .breadcrumb-container, #birdseye, .controller-bulletins .fa, .search-container:hover, .search-container.open, .login-background, table th, .mat-sort-header-arrow, .CodeMirror, #status-history-chart-container, #status-history-chart-control-container, #status-history-chart-control-container, // some analog colors for headers and hover states, inputs, stats, etc diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss index 63e2591decba..145863cc7105 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss @@ -41,6 +41,7 @@ @use 'app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/create-remote-process-group.component-theme' as create-remote-process-group; @use 'app/pages/flow-designer/ui/common/banner/banner.component-theme' as banner; @use 'app/pages/flow-designer/ui/controller-service/controller-services.component-theme' as controller-service; +@use 'app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component-theme' as manage-remote-ports; @use 'app/pages/login/feature/login.component-theme' as login; @use 'app/pages/login/ui/login-form/login-form.component-theme' as login-form; @use 'app/pages/provenance/feature/provenance.component-theme' as provenance; @@ -460,6 +461,10 @@ $appFontPath: '~roboto-fontface/fonts'; cursor: pointer; } + .disabled { + cursor: not-allowed; + } + .value, .refresh-timestamp { font-weight: 500; @@ -483,6 +488,7 @@ $appFontPath: '~roboto-fontface/fonts'; @include canvas.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include banner.nifi-theme($material-theme-light); @include controller-service.nifi-theme($material-theme-light); +@include manage-remote-ports.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include footer.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include navigation-control.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include birdseye-control.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @@ -535,6 +541,7 @@ $appFontPath: '~roboto-fontface/fonts'; @include canvas.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include banner.nifi-theme($material-theme-dark); @include controller-service.nifi-theme($material-theme-dark); + @include manage-remote-ports.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include footer.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include navigation-control.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include birdseye-control.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); From 55738f85221f9408f52878af28e21b2e79145868 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Fri, 23 Feb 2024 17:09:48 -0500 Subject: [PATCH 12/73] NIFI-12740 Fixed Python to Java Object Binding Fixed issue in NiFiPythonGateway that stems from the fact that the thread adding an object to the JavaObjectBindings was not necessarily the thread removing them. The algorithm that was in place assumed that the same thread would be used, in order to ensure that an object could be unbound before being accessed. The new algorithm binds each new object to all active method invocations and only unbinds the objects after all method invocations complete, regardless of thread. Additionally, found that many method calls could create new proxies on the Python side, just for getter methods whose values don't change. This is very expensive, so introduced a new @Idempotent annotation that can be added to interface methods such that we can cache the value and avoid the expensive overhead. This closes #8456 Signed-off-by: David Handermann --- .../main/asciidoc/python-developer-guide.adoc | 2 +- .../py4j/CachingPythonProcessorDetails.java | 133 ---------- .../org/apache/nifi/py4j/PythonProcess.java | 37 +-- .../py4j/StandardPythonProcessorBridge.java | 30 ++- .../nifi/py4j/client/CommandBuilder.java | 2 +- .../nifi/py4j/client/JavaObjectBindings.java | 73 ++++-- .../nifi/py4j/client/NiFiPythonGateway.java | 227 +++++++++--------- .../client/PythonProxyInvocationHandler.java | 52 ++-- .../py4j/server/NiFiGatewayConnection.java | 46 +--- .../processor/PythonProcessorProxy.java | 42 ++-- .../py4j/client/TestNiFiPythonGateway.java | 153 ++++++++++++ .../nifi/python/processor/Idempotent.java | 37 +++ .../python/processor/PythonProcessor.java | 3 +- .../processor/PythonProcessorAdapter.java | 3 + 14 files changed, 473 insertions(+), 367 deletions(-) delete mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/CachingPythonProcessorDetails.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/client/TestNiFiPythonGateway.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/Idempotent.java diff --git a/nifi-docs/src/main/asciidoc/python-developer-guide.adoc b/nifi-docs/src/main/asciidoc/python-developer-guide.adoc index d610c482a673..8e52375e9988 100644 --- a/nifi-docs/src/main/asciidoc/python-developer-guide.adoc +++ b/nifi-docs/src/main/asciidoc/python-developer-guide.adoc @@ -712,7 +712,7 @@ class MyProcessor(FlowFileTransform): version = '0.0.1-SNAPSHOT' dependencies = ['debugpy'] - def onScheduled(context): + def onScheduled(self, context): try: import debugpy debugpy.connect(6688) diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/CachingPythonProcessorDetails.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/CachingPythonProcessorDetails.java deleted file mode 100644 index 6a9b36a53202..000000000000 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/CachingPythonProcessorDetails.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.py4j; - -import org.apache.nifi.python.PythonProcessorDetails; -import org.apache.nifi.python.processor.documentation.MultiProcessorUseCaseDetails; -import org.apache.nifi.python.processor.documentation.PropertyDescription; -import org.apache.nifi.python.processor.documentation.UseCaseDetails; - -import java.util.List; - -/** - * A wrapper around a PythonProcessorDetails that caches the results of the delegate's methods. - * Making calls to the Python side is relatively expensive, and we make many calls to {@link #getProcessorType()}, - * {@link #getProcessorVersion()}, etc. This simple wrapper allows us to make the calls only once. - */ -public class CachingPythonProcessorDetails implements PythonProcessorDetails { - private final PythonProcessorDetails delegate; - private volatile String processorType; - private volatile String processorVersion; - private volatile String capabilityDescription; - private volatile String sourceLocation; - private volatile List tags; - private volatile List dependencies; - private volatile String processorInterface; - private volatile List useCases; - private volatile List multiProcessorUseCases; - private volatile List propertyDescriptions; - - - public CachingPythonProcessorDetails(final PythonProcessorDetails delegate) { - this.delegate = delegate; - } - - @Override - public String getProcessorType() { - if (processorType == null) { - processorType = delegate.getProcessorType(); - } - return processorType; - } - - @Override - public String getProcessorVersion() { - if (processorVersion == null) { - processorVersion = delegate.getProcessorVersion(); - } - return processorVersion; - } - - @Override - public String getSourceLocation() { - if (sourceLocation == null) { - sourceLocation = delegate.getSourceLocation(); - } - return sourceLocation; - } - - @Override - public List getTags() { - if (tags == null) { - tags = delegate.getTags(); - } - return tags; - } - - @Override - public List getDependencies() { - if (dependencies == null) { - dependencies = delegate.getDependencies(); - } - return dependencies; - } - - @Override - public String getCapabilityDescription() { - if (capabilityDescription == null) { - capabilityDescription = delegate.getCapabilityDescription(); - } - return capabilityDescription; - } - - @Override - public String getInterface() { - if (processorInterface == null) { - processorInterface = delegate.getInterface(); - } - return processorInterface; - } - - @Override - public List getUseCases() { - if (useCases == null) { - useCases = delegate.getUseCases(); - } - return useCases; - } - - @Override - public List getMultiProcessorUseCases() { - if (multiProcessorUseCases == null) { - multiProcessorUseCases = delegate.getMultiProcessorUseCases(); - } - return multiProcessorUseCases; - } - - @Override - public List getPropertyDescriptions() { - if (propertyDescriptions == null) { - propertyDescriptions = delegate.getPropertyDescriptions(); - } - return propertyDescriptions; - } - - @Override - public void free() { - } -} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java index 10ca2325160e..2180681c0dc6 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java @@ -380,22 +380,27 @@ public PythonProcessorAdapter createProcessor() { }; // Create a PythonProcessorDetails and then call getProcessorType and getProcessorVersion to ensure that the details are cached - final PythonProcessorDetails processorDetails = new CachingPythonProcessorDetails(controller.getProcessorDetails(type, version)); - processorDetails.getProcessorType(); - processorDetails.getProcessorVersion(); - - final PythonProcessorBridge processorBridge = new StandardPythonProcessorBridge.Builder() - .controller(controller) - .creationWorkflow(creationWorkflow) - .processorDetails(processorDetails) - .workingDirectory(processConfig.getPythonWorkingDirectory()) - .moduleFile(new File(controller.getModuleFile(type, version))) - .build(); - - final CreatedProcessor createdProcessor = new CreatedProcessor(identifier, type, processorBridge); - createdProcessors.add(createdProcessor); - - return processorBridge; + final PythonProcessorDetails processorDetails = controller.getProcessorDetails(type, version); + try { + final String processorType = processorDetails.getProcessorType(); + final String processorVersion = processorDetails.getProcessorVersion(); + + final PythonProcessorBridge processorBridge = new StandardPythonProcessorBridge.Builder() + .controller(controller) + .creationWorkflow(creationWorkflow) + .processorType(processorType) + .processorVersion(processorVersion) + .workingDirectory(processConfig.getPythonWorkingDirectory()) + .moduleFile(new File(controller.getModuleFile(type, version))) + .build(); + + final CreatedProcessor createdProcessor = new CreatedProcessor(identifier, type, processorBridge); + createdProcessors.add(createdProcessor); + + return processorBridge; + } finally { + processorDetails.free(); + } } /** diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java index 9546b1e0e208..22aa56585eba 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonProcessorBridge.java @@ -18,7 +18,6 @@ package org.apache.nifi.py4j; import org.apache.nifi.python.PythonController; -import org.apache.nifi.python.PythonProcessorDetails; import org.apache.nifi.python.processor.PythonProcessorAdapter; import org.apache.nifi.python.processor.PythonProcessorBridge; import org.apache.nifi.python.processor.PythonProcessorInitializationContext; @@ -36,7 +35,8 @@ public class StandardPythonProcessorBridge implements PythonProcessorBridge { private static final Logger logger = LoggerFactory.getLogger(StandardPythonProcessorBridge.class); private final ProcessorCreationWorkflow creationWorkflow; - private final PythonProcessorDetails processorDetails; + private final String processorType; + private final String processorVersion; private volatile PythonProcessorAdapter adapter; private final File workingDir; private final File moduleFile; @@ -51,7 +51,8 @@ public class StandardPythonProcessorBridge implements PythonProcessorBridge { private StandardPythonProcessorBridge(final Builder builder) { this.controller = builder.controller; this.creationWorkflow = builder.creationWorkflow; - this.processorDetails = builder.processorDetails; + this.processorType = builder.processorType; + this.processorVersion = builder.processorVersion; this.workingDir = builder.workDir; this.moduleFile = builder.moduleFile; this.lastModified = this.moduleFile.lastModified(); @@ -165,7 +166,7 @@ private void initializePythonSide(final boolean continualRetry, final Completabl @Override public String getProcessorType() { - return processorDetails.getProcessorType(); + return processorType; } @Override @@ -175,7 +176,7 @@ public boolean reload() { return false; } - controller.reloadProcessor(getProcessorType(), processorDetails.getProcessorVersion(), workingDir.getAbsolutePath()); + controller.reloadProcessor(getProcessorType(), processorVersion, workingDir.getAbsolutePath()); initializePythonSide(false, new CompletableFuture<>()); lastModified = moduleFile.lastModified(); @@ -188,7 +189,8 @@ public static class Builder { private ProcessorCreationWorkflow creationWorkflow; private File workDir; private File moduleFile; - private PythonProcessorDetails processorDetails; + private String processorType; + private String processorVersion; public Builder controller(final PythonController controller) { this.controller = controller; @@ -200,8 +202,13 @@ public Builder creationWorkflow(final ProcessorCreationWorkflow creationWorkflow return this; } - public Builder processorDetails(final PythonProcessorDetails details) { - this.processorDetails = details; + public Builder processorType(final String processorType) { + this.processorType = processorType; + return this; + } + + public Builder processorVersion(final String processorVersion) { + this.processorVersion = processorVersion; return this; } @@ -222,8 +229,11 @@ public StandardPythonProcessorBridge build() { if (creationWorkflow == null) { throw new IllegalStateException("Must specify the Processor Creation Workflow"); } - if (processorDetails == null) { - throw new IllegalStateException("Must specify the Processor Details"); + if (processorType == null) { + throw new IllegalStateException("Must specify the Processor Type"); + } + if (processorVersion == null) { + throw new IllegalStateException("Must specify the Processor Version"); } if (workDir == null) { throw new IllegalStateException("Must specify the Working Directory"); diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/CommandBuilder.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/CommandBuilder.java index 142fe0d2d795..fa048b3d1a41 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/CommandBuilder.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/CommandBuilder.java @@ -104,7 +104,7 @@ private String getArgumentLine(final Object arg) { } private String bind(final Object value) { - final String objectId = bindings.bind(value); + final String objectId = bindings.bind(value, 1); boundIds.add(objectId); return objectId; } diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/JavaObjectBindings.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/JavaObjectBindings.java index aca0564199df..ee1e9b2166dd 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/JavaObjectBindings.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/JavaObjectBindings.java @@ -24,49 +24,90 @@ import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class JavaObjectBindings { private static final Logger logger = LoggerFactory.getLogger(JavaObjectBindings.class); private final AtomicLong idGenerator = new AtomicLong(0L); - private final Map bindings = new ConcurrentHashMap<>(); + private final Map bindings = new HashMap<>(); + private final Map bindingCounts = new HashMap<>(); + private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); + private final Lock readLock = rwLock.readLock(); + private final Lock writeLock = rwLock.writeLock(); public JavaObjectBindings() { - bind(Protocol.DEFAULT_JVM_OBJECT_ID, new JVMView("default", Protocol.DEFAULT_JVM_OBJECT_ID)); + bind(Protocol.DEFAULT_JVM_OBJECT_ID, new JVMView("default", Protocol.DEFAULT_JVM_OBJECT_ID), 1); } - public String bind(final Object object) { + public String bind(final Object object, final int count) { final String id = "o" + idGenerator.getAndIncrement(); - return bind(id, object); + return bind(id, object, count); } - public String bind(final String objectId, final Object object) { - bindings.put(objectId, object); + public String bind(final String objectId, final Object object, final int count) { + if (count == 0) { + logger.debug("Will not bind {} to ID {} because count is 0", object, objectId); + return objectId; + } - logger.debug("Bound {} to ID {}", object, objectId); + writeLock.lock(); + try { + bindings.put(objectId, object); + bindingCounts.put(objectId, count); + } finally { + writeLock.unlock(); + } + + logger.debug("Bound {} to ID {} with count {}", object, objectId, count); return objectId; } public Object getBoundObject(final String objectId) { - return bindings.get(objectId); + readLock.lock(); + try { + return bindings.get(objectId); + } finally { + readLock.unlock(); + } } public Object unbind(final String objectId) { - final Object unbound = bindings.remove(objectId); - logger.debug("Unbound {} from ID {}", unbound, objectId); + writeLock.lock(); + try { + final Integer currentValue = bindingCounts.remove(objectId); + final int updatedValue = (currentValue == null) ? 0 : currentValue - 1; + if (updatedValue < 1) { + final Object unbound = bindings.remove(objectId); + logger.debug("Unbound {} from ID {}", unbound, objectId); + return unbound; + } - return unbound; + bindingCounts.put(objectId, updatedValue); + logger.debug("Decremented binding count for ID {} to {}", objectId, updatedValue); + return bindings.get(objectId); + } finally { + writeLock.unlock(); + } } + public Map getCountsPerClass() { final Map counts = new HashMap<>(); - bindings.values().forEach(object -> { - final String className = (object == null) ? "" : object.getClass().getName(); - counts.merge(className, 1, Integer::sum); - }); + + readLock.lock(); + try { + bindings.values().forEach(object -> { + final String className = (object == null) ? "" : object.getClass().getName(); + counts.merge(className, 1, Integer::sum); + }); + } finally { + readLock.unlock(); + } return counts; } diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/NiFiPythonGateway.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/NiFiPythonGateway.java index 4051fac85385..5397b8b4cce1 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/NiFiPythonGateway.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/NiFiPythonGateway.java @@ -17,15 +17,6 @@ package org.apache.nifi.py4j.client; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Stack; -import java.util.concurrent.ConcurrentHashMap; - import org.apache.nifi.python.PythonController; import org.apache.nifi.python.PythonObjectProxy; import org.apache.nifi.python.processor.PreserveJavaBinding; @@ -34,8 +25,17 @@ import org.slf4j.LoggerFactory; import py4j.CallbackClient; import py4j.Gateway; +import py4j.ReturnObject; import py4j.reflection.PythonProxyHandler; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; + /** *

@@ -64,13 +64,29 @@ public class NiFiPythonGateway extends Gateway { private static final Logger logger = LoggerFactory.getLogger(NiFiPythonGateway.class); private final JavaObjectBindings objectBindings; - private final Map> invocationBindingsById = new ConcurrentHashMap<>(); + + // Guarded by synchronized methods + private final List activeInvocations = new ArrayList<>(); + + private final ReturnObject END_OF_ITERATOR_OBJECT = ReturnObject.getErrorReturnObject(new NoSuchElementException()); + private final Method freeMethod; + private final Method pingMethod; public NiFiPythonGateway(final JavaObjectBindings bindings, final Object entryPoint, final CallbackClient callbackClient) { super(entryPoint, callbackClient); this.objectBindings = bindings; + + freeMethod = getMethod(PythonObjectProxy.class, "free"); + pingMethod = getMethod(PythonController.class, "ping"); } + private Method getMethod(final Class clazz, final String methodName) { + try { + return clazz.getMethod(methodName); + } catch (final NoSuchMethodException ignored) { + return null; + } + } public JavaObjectBindings getObjectBindings() { return objectBindings; @@ -82,20 +98,55 @@ public Object getObject(final String objectId) { } @Override - public String putNewObject(final Object object) { - final String objectId = objectBindings.bind(object); + public ReturnObject invoke(final String methodName, final String targetObjectId, final List args) { + // Py4J detects the end of an iterator on the Python side by catching a Py4JError. It makes the assumption that + // if it encounters Py4JError then it was due to a NoSuchElementException being thrown on the Java side. In this case, + // it throws a StopIteration exception on the Python side. This has a couple of problems: + // 1. It's not clear that the exception was thrown because the iterator was exhausted. It could have been thrown for + // any other reason, such as an IOException, etc. + // 2. It relies on Exception handling for flow control, which is generally considered bad practice. + // While we aren't going to go so far as to override the Python code, we can at least prevent Java from constantly + // throwing the Exception, catching it, logging it, and re-throwing it. + // Instead, we just create the ReturnObject once and return it. + final boolean intervene = isNextOnExhaustedIterator(methodName, targetObjectId, args); + if (!intervene) { + return super.invoke(methodName, targetObjectId, args); + } + + return END_OF_ITERATOR_OBJECT; + } + + private boolean isNextOnExhaustedIterator(final String methodName, final String targetObjectId, final List args) { + if (!"next".equals(methodName)) { + return false; + } + if (args != null && !args.isEmpty()) { + return false; + } + + final Object targetObject = getObjectFromId(targetObjectId); + if (!(targetObject instanceof final Iterator itr)) { + return false; + } + + return !itr.hasNext(); + } + + @Override + public synchronized String putNewObject(final Object object) { + final String objectId = objectBindings.bind(object, activeInvocations.size() + 1); - final InvocationBindings bindings = getInvocationBindings(); - if (bindings != null) { - bindings.add(objectId); + for (final InvocationBindings activeInvocation : activeInvocations) { + activeInvocation.add(objectId); } - logger.debug("Binding {}: {} ({}) for {}", objectId, object, object == null ? "null" : object.getClass().getName(), bindings); + + logger.debug("Binding {}: {} ({})", objectId, object, object == null ? "null" : object.getClass().getName()); return objectId; } @Override - public Object putObject(final String id, final Object object) { - objectBindings.bind(id, object); + public synchronized Object putObject(final String id, final Object object) { + objectBindings.bind(id, object, activeInvocations.size() + 1); logger.debug("Binding {}: {} ({})", id, object, object == null ? "null" : object.getClass().getName()); return super.putObject(id, object); @@ -103,55 +154,16 @@ public Object putObject(final String id, final Object object) { @Override public void deleteObject(final String objectId) { - // When the python side no longer needs an object, its finalizer will notify the Java side that it's no longer needed and can be removed - // from the accessible objects on the Java heap. However, if we are making a call to the Python side, it's possible that even though the Python - // side no longer needs the object, the Java side still needs it bound. For instance, consider the following case: - // - // Java side calls PythonController.getProcessorTypes() - // Python side needs to return an ArrayList, so it calls back to the Java side (in a separate thread) to create this ArrayList with ID o54 - // Python side adds several Processors to the ArrayList. - // Python side returns the ArrayList to the Java side. - // Python side no longer needs the ArrayList, so while the Java side is processing the response from the Python side, the Python side notifies the Java side that it's no longer needed. - // Java side unbinds the ArrayList. - // Java side parses the response from Python, indicating that the return value is the object with ID o54. - // Java side cannot access object with ID o54 because it was already removed. - // - // To avoid this, we check if there is an Invocation Binding (indicating that we are in the middle of a method invocation) and if so, - // we add the object to a list of objects to delete on completion. - // If there is no Invocation Binding, we unbind the object immediately. - final InvocationBindings invocationBindings = getInvocationBindings(); - if (invocationBindings == null) { - final Object unbound = objectBindings.unbind(objectId); - logger.debug("Unbound {}: {} because it was explicitly requested from Python side", objectId, unbound); - } else { - invocationBindings.deleteOnCompletion(objectId); - logger.debug("Unbound {} because it was requested from Python side but in active method invocation so will not remove until invocation completed", objectId); - } + logger.debug("Unbinding {} because it was explicitly requested from Python side", objectId); + objectBindings.unbind(objectId); } - private InvocationBindings getInvocationBindings() { - final long threadId = Thread.currentThread().threadId(); - final Stack stack = invocationBindingsById.get(threadId); - if (stack == null || stack.isEmpty()) { - return null; - } - - return stack.peek(); - } @Override protected PythonProxyHandler createPythonProxyHandler(final String id) { logger.debug("Creating Python Proxy Handler for ID {}", id); final PythonProxyInvocationHandler createdHandler = new PythonProxyInvocationHandler(this, id); - Method proxyFreeMethod; - try { - proxyFreeMethod = PythonObjectProxy.class.getMethod("free"); - } catch (final NoSuchMethodException ignored) { - proxyFreeMethod = null; - } - - final Method freeMethod = proxyFreeMethod; return new PythonProxyHandler(id, this) { @Override public Object invoke(final Object proxy, final Method method, final Object[] args) throws Throwable { @@ -163,54 +175,41 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg return createdHandler.invoke(proxy, method, args); } - @Override - protected void finalize() { - // Do nothing. Prevent super.finalize() from being called. - } }; } - public void beginInvocation(final String objectId, final Method method, final Object[] args) { - final long threadId = Thread.currentThread().threadId(); - final InvocationBindings bindings = new InvocationBindings(objectId, method, args); - final Stack stack = invocationBindingsById.computeIfAbsent(threadId, id -> new Stack<>()); - stack.push(bindings); - - logger.debug("Beginning method invocation {} on {} with args {}", method, objectId, Arrays.toString(args)); - } - - public void endInvocation(final String objectId, final Method method, final Object[] args) { + public synchronized InvocationBindings beginInvocation(final String objectId, final Method method, final Object[] args) { final boolean unbind = isUnbind(method); - final long threadId = Thread.currentThread().threadId(); - final Stack stack = invocationBindingsById.get(threadId); - if (stack == null) { - return; + final InvocationBindings bindings = new InvocationBindings(objectId, method, args, unbind); + + // We don't want to keep track of invocations of the Ping method. + // The Ping method has no arguments or bound objects to free, and can occasionally + // even hang, if the timing is right because of the startup sequence of the Python side and how the + // communications are established. + if (!pingMethod.equals(method)) { + activeInvocations.add(bindings); } - while (!stack.isEmpty()) { - final InvocationBindings invocationBindings = stack.pop(); - final String methodName = invocationBindings.getMethod().getName(); - - invocationBindings.getObjectIds().forEach(id -> { - if (unbind) { - final Object unbound = objectBindings.unbind(id); - logger.debug("Unbinding {}: {} because invocation of {} on {} with args {} has completed", id, unbound, methodName, objectId, Arrays.toString(args)); - } else { - logger.debug("Will not unbind {} even though invocation of {} on {} with args {} has completed because of the method being completed", - id, methodName, objectId, Arrays.toString(args)); - } - }); + logger.debug("Beginning method invocation {}", bindings); - invocationBindings.getObjectsToDeleteOnCompletion().forEach(id -> { - final Object unbound = objectBindings.unbind(id); - logger.debug("Unbinding {}: {} because invocation of {} on {} with args {} has completed", id, unbound, methodName, objectId, Arrays.toString(args)); - }); + return bindings; + } + + public synchronized void endInvocation(final InvocationBindings invocationBindings) { + final String methodName = invocationBindings.getMethod().getName(); + logger.debug("Ending method invocation {}", invocationBindings); + + final String objectId = invocationBindings.getTargetObjectId(); + + activeInvocations.remove(invocationBindings); + invocationBindings.getObjectIds().forEach(id -> { + final Object unbound = objectBindings.unbind(id); - if (Objects.equals(invocationBindings.getTargetObjectId(), objectId) && Objects.equals(invocationBindings.getMethod(), method) && Arrays.equals(invocationBindings.getArgs(), args)) { - break; + if (logger.isDebugEnabled()) { + logger.debug("Unbinding {}: {} because invocation of {} on {} with args {} has completed", id, unbound, methodName, objectId, Arrays.toString(invocationBindings.getArgs())); } - } + }); } protected boolean isUnbind(final Method method) { @@ -222,42 +221,52 @@ protected boolean isUnbind(final Method method) { // that it is no longer needed. We can do this by annotating the method with the PreserveJavaBinding annotation. final boolean relevantClass = PythonController.class.isAssignableFrom(declaringClass) || PythonProcessor.class.isAssignableFrom(declaringClass); if (relevantClass && method.getAnnotation(PreserveJavaBinding.class) == null) { - return true; + // No need for binding / unbinding if types are primitives, because the values themselves are sent across and not references to objects. + boolean bindNecessary = isBindNecessary(method.getReturnType()); + for (final Class parameterType : method.getParameterTypes()) { + if (isBindNecessary(parameterType)) { + bindNecessary = true; + break; + } + } + + return bindNecessary; } return false; } + private boolean isBindNecessary(final Class type) { + return !type.isPrimitive() && type != String.class && type != byte[].class; + } + - private static class InvocationBindings { + public static class InvocationBindings { private final String targetObjectId; private final Method method; private final Object[] args; + private final boolean unbind; private final List objectIds = new ArrayList<>(); - private final List deleteOnCompletion = new ArrayList<>(); - public InvocationBindings(final String targetObjectId, final Method method, final Object[] args) { + public InvocationBindings(final String targetObjectId, final Method method, final Object[] args, final boolean unbind) { this.targetObjectId = targetObjectId; this.method = method; this.args = args; + this.unbind = unbind; } - public void add(final String objectId) { - objectIds.add(objectId); + public boolean isUnbind() { + return unbind; } - public void deleteOnCompletion(final String objectId) { - deleteOnCompletion.add(objectId); + public void add(final String objectId) { + objectIds.add(objectId); } public List getObjectIds() { return objectIds; } - public List getObjectsToDeleteOnCompletion() { - return deleteOnCompletion; - } - public String getTargetObjectId() { return targetObjectId; } diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/PythonProxyInvocationHandler.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/PythonProxyInvocationHandler.java index edba8c60cac5..c753083e90e5 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/PythonProxyInvocationHandler.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/client/PythonProxyInvocationHandler.java @@ -17,6 +17,8 @@ package org.apache.nifi.py4j.client; +import org.apache.nifi.py4j.client.NiFiPythonGateway.InvocationBindings; +import org.apache.nifi.python.processor.Idempotent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import py4j.Protocol; @@ -30,6 +32,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; public class PythonProxyInvocationHandler implements InvocationHandler { private static final Logger logger = LoggerFactory.getLogger(PythonProxyInvocationHandler.class); @@ -38,6 +42,7 @@ public class PythonProxyInvocationHandler implements InvocationHandler { private final NiFiPythonGateway gateway; private final JavaObjectBindings bindings; private final String gcCommand; + private final ConcurrentMap cachedValues = new ConcurrentHashMap<>(); public PythonProxyInvocationHandler(final NiFiPythonGateway gateway, final String objectId) { this.objectId = objectId; @@ -59,6 +64,15 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg return "PythonProxy[targetObjectId=" + objectId + "]"; } + // Only support caching for 0-arg methods currently + final boolean idempotent = (args == null || args.length == 0) && method.getAnnotation(Idempotent.class) != null; + if (idempotent) { + final Object cachedValue = cachedValues.get(method); + if (cachedValue != null) { + return cachedValue; + } + } + final CommandBuilder commandBuilder = new CommandBuilder(bindings, objectId, method.getName()); final String command = commandBuilder.buildCommand(args); @@ -67,23 +81,27 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg logger.debug("Invoking {} on {} with args {} using command {}", method, proxy, argList, command); } - gateway.beginInvocation(this.objectId, method, args); - - final String response = gateway.getCallbackClient().sendCommand(command); - final Object output = Protocol.getReturnValue(response, gateway); - final Object convertedOutput = convertOutput(method, output); - - if (gateway.isUnbind(method)) { - commandBuilder.getBoundIds().forEach(bindings::unbind); - commandBuilder.getBoundIds().forEach(i -> logger.debug("For method invocation {} unbound {} (from command builder)", method.getName(), i)); - } else { - commandBuilder.getBoundIds().forEach(i -> logger.debug("For method invocation {} will not unbind {} (from command builder) because arguments of this method are not to be unbound", - method.getName(), i)); + final InvocationBindings invocationBindings = gateway.beginInvocation(this.objectId, method, args); + try { + final String response = gateway.getCallbackClient().sendCommand(command); + final Object output = Protocol.getReturnValue(response, gateway); + final Object result = convertOutput(method, output); + if (idempotent) { + cachedValues.putIfAbsent(method, result); + } + + return result; + } finally { + if (invocationBindings.isUnbind()) { + commandBuilder.getBoundIds().forEach(bindings::unbind); + commandBuilder.getBoundIds().forEach(i -> logger.debug("For method invocation {} unbound {} (from command builder)", method.getName(), i)); + } else { + commandBuilder.getBoundIds().forEach(i -> logger.debug("For method invocation {} will not unbind {} (from command builder) because arguments of this method are not to be unbound", + method.getName(), i)); + } + + gateway.endInvocation(invocationBindings); } - - gateway.endInvocation(this.objectId, method, args); - - return convertedOutput; } @@ -107,7 +125,7 @@ private Object convertOutput(final Method method, final Object output) { throw new Py4JException("Incompatible output type. Expected: " + returnType.getName() + " Actual: " + outputType.getName()); } - return converters.get(0).convert(output); + return converters.getFirst().convert(output); } } diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/server/NiFiGatewayConnection.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/server/NiFiGatewayConnection.java index 7254663f070d..7a17176137a0 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/server/NiFiGatewayConnection.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/server/NiFiGatewayConnection.java @@ -17,17 +17,13 @@ package org.apache.nifi.py4j.server; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import py4j.Gateway; import py4j.GatewayConnection; import py4j.GatewayServerListener; import py4j.commands.Command; -import java.io.BufferedWriter; import java.io.IOException; import java.net.Socket; -import java.net.SocketTimeoutException; import java.util.List; /** @@ -66,10 +62,8 @@ * - */ public class NiFiGatewayConnection extends GatewayConnection { - private static final Logger logger = LoggerFactory.getLogger(NiFiGatewayConnection.class); private final NiFiGatewayServer gatewayServer; - private volatile boolean poisoned = false; private final ClassLoader contextClassLoader; public NiFiGatewayConnection(final NiFiGatewayServer gatewayServer, @@ -83,10 +77,6 @@ public NiFiGatewayConnection(final NiFiGatewayServer gatewayServer, this.contextClassLoader = getClass().getClassLoader(); } - private boolean isContinue() { - return !poisoned && !gatewayServer.isShutdown(); - } - @Override public void run() { final ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader(); @@ -94,9 +84,7 @@ public void run() { Thread.currentThread().setContextClassLoader(this.contextClassLoader); Thread.currentThread().setName(String.format("NiFiGatewayConnection Thread for %s %s", gatewayServer.getComponentType(), gatewayServer.getComponentId())); - while (isContinue()) { - super.run(); - } + super.run(); shutdown(false); } finally { @@ -105,36 +93,4 @@ public void run() { } } } - - protected void quietSendFatalError(final BufferedWriter writer, final Throwable exception) { - if (gatewayServer.isShutdown()) { - super.quietSendFatalError(writer, exception); - } - - if (exception instanceof SocketTimeoutException) { - logger.debug("{} received call to quietSendFatalError with Exception {} but will ignore because it's a SocketTimeoutException", this, exception.toString()); - return; - } - - poisoned = true; - super.quietSendFatalError(writer, exception); - } - - @Override - public void shutdown(final boolean reset) { - if (poisoned) { - logger.debug("Connection {} shutdown and is poisoned so will truly shutdown connection, reset={}", this, reset); - super.shutdown(reset); - return; - } - - if (gatewayServer.isShutdown()) { - logger.debug("Connection {} shutdown and Gateway Server is shutdown so will truly shutdown connection", this); - super.shutdown(false); - return; - } - - // do nothing. - logger.debug("Connection {} shutdown but is not poisoned. Will not shutdown the connection", this); - } } diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java index a370d3d1bb10..96fc6c1e06ff 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/python/processor/PythonProcessorProxy.java @@ -54,6 +54,7 @@ public abstract class PythonProcessorProxy extends Ab private volatile Boolean supportsDynamicProperties; private volatile T currentTransform; + private volatile PythonProcessorAdapter currentAdapter; private volatile ProcessContext currentProcessContext; @@ -112,25 +113,23 @@ public ComponentLog getLogger() { } - @OnScheduled - public void setContext(final ProcessContext context) { - this.currentProcessContext = context; - } - - protected T getTransform() { + protected synchronized T getTransform() { final PythonProcessorBridge bridge = getBridge().orElseThrow(() -> new IllegalStateException(this + " is not finished initializing")); final Optional optionalAdapter = bridge.getProcessorAdapter(); if (optionalAdapter.isEmpty()) { throw new IllegalStateException(this + " is not finished initializing"); } - final T transform = (T) optionalAdapter.get().getProcessor(); - if (transform != currentTransform) { + final PythonProcessorAdapter adapter = optionalAdapter.get(); + if (adapter != currentAdapter) { + final T transform = (T) adapter.getProcessor(); transform.setContext(currentProcessContext); + currentTransform = transform; + currentAdapter = adapter; } - return transform; + return currentTransform; } protected Optional getBridge() { @@ -257,8 +256,7 @@ protected boolean isSupportsDynamicPropertyDescriptor() { return supported; } - @OnScheduled - public void cacheRelationships() { + private void cacheRelationships() { // Get the Relationships from the Python side. Then make a defensive copy and make that copy immutable. // We cache this to avoid having to call into the Python side while the Processor is running. However, once // it is stopped, its relationships may change due to properties, etc. @@ -266,8 +264,7 @@ public void cacheRelationships() { this.cachedRelationships = Set.copyOf(relationships); } - @OnScheduled - public void cacheDynamicPropertyDescriptors(final ProcessContext context) { + private void cacheDynamicPropertyDescriptors(final ProcessContext context) { final Map dynamicDescriptors = new HashMap<>(); final Set descriptors = context.getProperties().keySet(); @@ -299,8 +296,8 @@ private Set fetchRelationshipsFromPythonProcessor() { Set processorRelationships; try { processorRelationships = bridge.getProcessorAdapter() - .map(PythonProcessorAdapter::getRelationships) - .orElseGet(HashSet::new); + .map(PythonProcessorAdapter::getRelationships) + .orElseGet(HashSet::new); } catch (final Exception e) { getLogger().warn("Failed to obtain list of Relationships from Python Processor {}; assuming no explicit relationships", this, e); processorRelationships = new HashSet<>(); @@ -313,14 +310,23 @@ private Set fetchRelationshipsFromPythonProcessor() { @OnScheduled public void onScheduled(final ProcessContext context) { + this.currentProcessContext = context; + if (bridge == null) { throw new IllegalStateException("Processor is not yet initialized"); } reload(); - bridge.getProcessorAdapter() - .orElseThrow(() -> new IllegalStateException("Processor has not finished initializing")) - .onScheduled(context); + + final PythonProcessorAdapter adapter = bridge.getProcessorAdapter() + .orElseThrow(() -> new IllegalStateException("Processor has not finished initializing")); + + adapter.onScheduled(context); + + adapter.getProcessor().setContext(context); + + cacheRelationships(); + cacheDynamicPropertyDescriptors(context); } @OnStopped diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/client/TestNiFiPythonGateway.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/client/TestNiFiPythonGateway.java new file mode 100644 index 000000000000..20401acb99d1 --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/client/TestNiFiPythonGateway.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.py4j.client; + +import org.apache.nifi.py4j.client.NiFiPythonGateway.InvocationBindings; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import py4j.CallbackClient; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class TestNiFiPythonGateway { + + private static final Method NOP_METHOD; + + static { + try { + NOP_METHOD = TestNiFiPythonGateway.class.getMethod("nop", Object[].class); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + private JavaObjectBindings bindings; + private NiFiPythonGateway gateway; + + @BeforeEach + public void setup() { + bindings = new JavaObjectBindings(); + gateway = new NiFiPythonGateway(bindings, this, mock(CallbackClient.class)) { + @Override + protected boolean isUnbind(final Method method) { + return true; + } + }; + } + + @Test + public void testObjectBoundUnboundWithSingleInvocation() { + final Object[] args = new Object[] { new Object() }; + final InvocationBindings invocationBindings = gateway.beginInvocation("o123", NOP_METHOD, args); + final String objectId = gateway.putNewObject(args[0]); + final List objectIds = invocationBindings.getObjectIds(); + assertEquals(List.of(objectId), objectIds); + assertEquals(args[0], gateway.getObject(objectId)); + + gateway.endInvocation(invocationBindings); + gateway.deleteObject(objectId); + assertNull(gateway.getObject(objectId)); + } + + @Test + public void testObjectBoundNotUnboundWhileInvocationActive() { + final Object[] args = new Object[] { new Object() }; + final InvocationBindings invocationBindings = gateway.beginInvocation("o123", NOP_METHOD, args); + final String objectId = gateway.putNewObject(args[0]); + final List objectIds = invocationBindings.getObjectIds(); + assertEquals(List.of(objectId), objectIds); + assertEquals(args[0], gateway.getObject(objectId)); + + gateway.deleteObject(objectId); + + // gateway.deleteObject should not remove the value because the invocation is still active + assertEquals(args[0], gateway.getObject(objectId)); + + // After calling endInvocation, the object should be cleaned up. + gateway.endInvocation(invocationBindings); + assertNull(gateway.getObject(objectId)); + } + + @Test + public void testEndInvocationDifferentThread() throws InterruptedException { + final Object[] args = new Object[] { new Object() }; + final InvocationBindings invocationBindings = gateway.beginInvocation("o123", NOP_METHOD, args); + final String objectId = gateway.putNewObject(args[0]); + final List objectIds = invocationBindings.getObjectIds(); + assertEquals(List.of(objectId), objectIds); + assertEquals(args[0], gateway.getObject(objectId)); + + gateway.deleteObject(objectId); + + // gateway.deleteObject should not remove the value because the invocation is still active + assertEquals(args[0], gateway.getObject(objectId)); + + Thread.ofVirtual().start(() -> { + gateway.endInvocation(invocationBindings); + }).join(); + + assertNull(gateway.getObject(objectId)); + } + + + @Test + public void testMultipleInvocationsActive() { + final Object[] args = new Object[] { new Object() }; + + // Simulate 5 different threads making invocations into the Python process + final List bindings = new ArrayList<>(); + for (int i=0; i < 5; i++) { + final InvocationBindings invocationBindings = gateway.beginInvocation("o123", NOP_METHOD, args); + bindings.add(invocationBindings); + } + + // Create an object while there are 5 invocations active. We don't know which invocation caused + // the object to be created, so we can't unbind it until all invocations are complete. + final String objectId = gateway.putNewObject(args[0]); + + // Simulate Python process garbage collecting the object after 2 of the invocations are complete + gateway.endInvocation(bindings.removeFirst()); + gateway.endInvocation(bindings.removeFirst()); + gateway.deleteObject(objectId); + + // We should now be able to add additional invocations, and they should not prevent the already-bound + // object from being cleaned up. + for (int i=0; i < 3; i++) { + gateway.beginInvocation("o123", NOP_METHOD, args); + } + + // As each of the invocations complete, we should find the object is still bound + for (final InvocationBindings invocationBindings : bindings) { + assertNotNull(gateway.getObject(objectId)); + gateway.endInvocation(invocationBindings); + } + + // When the final invocation completes, the object should be unbound + assertNull(gateway.getObject(objectId)); + } + + public void nop(Object... args) { + } +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/Idempotent.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/Idempotent.java new file mode 100644 index 000000000000..168401481e93 --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/Idempotent.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.python.processor; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * A marker annotation that can be added to an interface's method. + * When this annotation is present, the proxy that is responsible for invoking the method on the Python side + * may opt to cache the value and return the same value in subsequent calls. + */ +@Documented +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface Idempotent { +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessor.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessor.java index 0bf8ea67451e..885ce025c6ad 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessor.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessor.java @@ -19,13 +19,14 @@ import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.python.PythonObjectProxy; import java.util.List; /** * Base interface for any Python based processor */ -public interface PythonProcessor { +public interface PythonProcessor extends PythonObjectProxy { List getSupportedPropertyDescriptors(); diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorAdapter.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorAdapter.java index 3753b20ec590..8e17d80c486b 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorAdapter.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/processor/PythonProcessorAdapter.java @@ -27,6 +27,7 @@ import java.util.Set; public interface PythonProcessorAdapter extends PythonProcessor { + @Idempotent PythonProcessor getProcessor(); @PreserveJavaBinding @@ -40,9 +41,11 @@ public interface PythonProcessorAdapter extends PythonProcessor { void onStopped(ProcessContext context); + @Idempotent String getCapabilityDescription(); PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyName); + @Idempotent boolean isDynamicPropertySupported(); } From 97516de1e0dbfa314a811c317094918cf65794d6 Mon Sep 17 00:00:00 2001 From: Jim Steinebrey Date: Mon, 4 Mar 2024 13:29:12 -0500 Subject: [PATCH 13/73] NIFI-12645 Fix to correctly invoke onStopped method of scripted processor Signed-off-by: Matt Burgess This closes #8471 --- .../apache/nifi/processors/script/InvokeScriptedProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java index 8e8d21b1092b..2853bccb3f76 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java @@ -606,7 +606,7 @@ public void added() { @OnStopped public void stop(ProcessContext context) { // If the script needs to be reloaded at this point, it is because it was empty - if (!scriptNeedsReload.get()) { + if (scriptRunner != null) { invokeScriptedProcessorMethod("onStopped", context); } scriptingComponentHelper.stop(); From 4083fd3f75252235fb9ff82ac1d0f12c3f903bdf Mon Sep 17 00:00:00 2001 From: dan-s1 Date: Mon, 4 Mar 2024 18:39:47 +0000 Subject: [PATCH 14/73] NIFI-12864 Corrected NOTICE Reference to UriComponentsBuilder This closes #8470 Signed-off-by: David Handermann --- NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index 75106360e49a..e61bb33d94d1 100644 --- a/NOTICE +++ b/NOTICE @@ -134,4 +134,4 @@ This includes derived works from Spring Framework available under Apache Softwar The derived work is adapted from https://github.com/spring-projects/spring-framework/blob/main/spring-web/src/main/java/org/springframework/web/util/UriComponentsBuilder.java and can be found in - nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/UriUtils.java \ No newline at end of file + nifi-api/src/main/java/org/apache/nifi/processor/util/URLValidator.java \ No newline at end of file From d1b6aab9d36e713adb377f6fec14ea376d980fbe Mon Sep 17 00:00:00 2001 From: Ferenc Kis Date: Thu, 15 Feb 2024 16:37:46 +0100 Subject: [PATCH 15/73] NIFI-12760 Flow sensitive properties encryption support in toolkit Signed-off-by: Ferenc Erdei This closes #8430. --- .../StandardFlowPropertyEncryptor.java | 4 +- .../minifi-toolkit-assembly/README.md | 19 ++- .../minifi-toolkit-encrypt-config/pom.xml | 12 ++ .../config/command/MiNiFiEncryptConfig.java | 128 ++++++++++++++---- 4 files changed, 134 insertions(+), 29 deletions(-) diff --git a/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/StandardFlowPropertyEncryptor.java b/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/StandardFlowPropertyEncryptor.java index 32f8746d7ed7..9222a41c2bd9 100644 --- a/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/StandardFlowPropertyEncryptor.java +++ b/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/StandardFlowPropertyEncryptor.java @@ -93,7 +93,9 @@ private Map> flowProvidedSensitiveProperties(VersionedDatafl } private Map> runtimeManifestSensitiveProperties() { - return ofNullable(runTimeManifest.getBundles()).orElse(List.of()) + return ofNullable(runTimeManifest) + .map(RuntimeManifest::getBundles) + .orElse(List.of()) .stream() .flatMap(bundle -> Stream.of( ofNullable(bundle.getComponentManifest().getProcessors()).orElse(List.of()), diff --git a/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md b/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md index 29f8c859b6d4..0b0b0444d3a0 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md +++ b/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md @@ -64,7 +64,7 @@ It's not guaranteed in all circumstances that the migration will result in a cor # Encrypting Sensitive Properties in bootstrap.conf ## MiNiFi Encrypt-Config Tool -The encrypt-config command line tool (invoked in minifi-toolkit as ./bin/encrypt-config.sh or bin\encrypt-config.bat) reads from a bootstrap.conf file with plaintext sensitive configuration values and encrypts each value using a random encryption key. It replaces the plain values with the protected value in the same file, or writes to a new bootstrap.conf file if specified. +The encrypt-config command line tool (invoked in minifi-toolkit as ./bin/encrypt-config.sh or bin\encrypt-config.bat) reads from a bootstrap.conf file with plaintext sensitive configuration values and encrypts each value using a random encryption key. It replaces the plain values with the protected value in the same file, or writes to a new bootstrap.conf file if specified. Additionally it can be used to encrypt the unencrypted sensitive properties (if any) in the flow.json.raw. For using this functionality `nifi.minifi.sensitive.props.key` and `nifi.minifi.sensitive.props.algorithm` has to be provided in bootstrap.conf. The supported encryption algorithm utilized is AES/GCM 256-bit. @@ -78,9 +78,12 @@ To show help: The following are the available options: * -b, --bootstrapConf Path to file containing Bootstrap Configuration [bootstrap.conf] * -B, --outputBootstrapConf Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured +* -x, --encryptRawFlowJsonOnly Process Raw Flow Configuration [flow.json.raw] sensitive property values without modifying other configuration files +* -f, --rawFlowJson Path to file containing Raw Flow Configuration [flow.json.raw] that will be updated unless the output argument is provided +* -g, --outputRawFlowJson ath to output file for Raw Flow Configuration [flow.json.raw] with property protection applied * -h, --help Show help message and exit. -### Example +### Example 1 As an example of how the tool works with the following existing values in the bootstrap.conf file: ``` nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword @@ -141,6 +144,18 @@ Sensitive configuration values are encrypted by the tool by default, however you If the bootstrap.conf file already has valid protected values, those property values are not modified by the tool. +### Example 2 +An example to encrypt non encrypted sensitive properties in flow.json.raw +``` +nifi.sensitive.props.key=sensitivePropsKey +nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256 +``` +Enter the following arguments when using the tool: +``` +encrypt-config.sh -x -f flow.json.raw +``` +As a result, the flow.json.raw file is overwritten with encrypted sensitive properties +The algorithm uses the property descriptors in the flow.json.raw to determine if a property is sensitive or not. If that information is missing, no properties will be encrypted even if it is defined sensitive in the agent manifest. ## Getting Help If you have questions, you can reach out to our mailing list: dev@nifi.apache.org diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml index fd936cad8d4c..16c2a449e276 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/pom.xml @@ -52,6 +52,18 @@ org.apache.nifi.minifi minifi-commons-utils + 2.0.0-SNAPSHOT + + + org.apache.nifi.minifi + minifi-commons-framework + 2.0.0-SNAPSHOT + + + org.apache.nifi + nifi-framework-core + + diff --git a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java index f5e624824020..ebeb3ecd71e6 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java +++ b/minifi/minifi-toolkit/minifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/minifi/toolkit/config/command/MiNiFiEncryptConfig.java @@ -14,8 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.minifi.toolkit.config.command; +import static java.nio.file.Files.readAllBytes; +import static java.nio.file.Files.write; +import static java.util.Optional.ofNullable; +import static org.apache.commons.lang3.StringUtils.isAnyBlank; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_KEY; + import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -24,8 +32,13 @@ import java.security.SecureRandom; import java.util.HashSet; import java.util.HexFormat; -import java.util.Optional; import java.util.Set; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.encrypt.PropertyEncryptorBuilder; +import org.apache.nifi.minifi.commons.service.FlowPropertyEncryptor; +import org.apache.nifi.minifi.commons.service.FlowSerDeService; +import org.apache.nifi.minifi.commons.service.StandardFlowPropertyEncryptor; +import org.apache.nifi.minifi.commons.service.StandardFlowSerDeService; import org.apache.nifi.minifi.properties.BootstrapProperties; import org.apache.nifi.minifi.properties.BootstrapPropertiesLoader; import org.apache.nifi.minifi.properties.ProtectedBootstrapProperties; @@ -40,23 +53,23 @@ import picocli.CommandLine.Option; /** - * Shared Encrypt Configuration for NiFi and NiFi Registry + * Encrypt Configuration for MiNiFi */ @Command( - name = "encrypt-config", - sortOptions = false, - mixinStandardHelpOptions = true, - usageHelpWidth = 160, - separator = " ", - version = { - "Java ${java.version} (${java.vendor} ${java.vm.name} ${java.vm.version})" - }, - descriptionHeading = "Description: ", - description = { - "encrypt-config supports protection of sensitive values in Apache MiNiFi" - } + name = "encrypt-config", + sortOptions = false, + mixinStandardHelpOptions = true, + usageHelpWidth = 160, + separator = " ", + version = { + "Java ${java.version} (${java.vendor} ${java.vm.name} ${java.vm.version})" + }, + descriptionHeading = "Description: ", + description = { + "encrypt-config supports protection of sensitive values in Apache MiNiFi" + } ) -public class MiNiFiEncryptConfig implements Runnable{ +public class MiNiFiEncryptConfig implements Runnable { static final String BOOTSTRAP_ROOT_KEY_PROPERTY = "minifi.bootstrap.sensitive.key"; @@ -64,30 +77,52 @@ public class MiNiFiEncryptConfig implements Runnable{ private static final int KEY_LENGTH = 32; @Option( - names = {"-b", "--bootstrapConf"}, - description = "Path to file containing Bootstrap Configuration [bootstrap.conf] for optional root key and property protection scheme settings" + names = {"-b", "--bootstrapConf"}, + description = "Path to file containing Bootstrap Configuration [bootstrap.conf] for optional root key and property protection scheme settings" ) Path bootstrapConfPath; @Option( - names = {"-B", "--outputBootstrapConf"}, - description = "Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured" + names = {"-B", "--outputBootstrapConf"}, + description = "Path to output file for Bootstrap Configuration [bootstrap.conf] with root key configured" ) Path outputBootstrapConf; + @Option( + names = {"-x", "--encryptRawFlowJsonOnly"}, + description = "Process Raw Flow Configuration [flow.json.raw] sensitive property values without modifying other configuration files" + ) + boolean flowConfigurationRequested; + + @Option( + names = {"-f", "--rawFlowJson"}, + description = "Path to file containing Raw Flow Configuration [flow.json.raw] that will be updated unless the output argument is provided" + ) + Path flowConfigurationPath; + + @Option( + names = {"-g", "--outputRawFlowJson"}, + description = "Path to output file for Raw Flow Configuration [flow.json.raw] with property protection applied" + ) + Path outputFlowConfigurationPath; + protected final Logger logger = LoggerFactory.getLogger(getClass()); @Override public void run() { - processBootstrapConf(); + BootstrapProperties unprotectedProperties = BootstrapPropertiesLoader.load(bootstrapConfPath.toFile()); + processBootstrapConf(unprotectedProperties); + processFlowConfiguration(unprotectedProperties); } /** * Process bootstrap.conf writing new Root Key to specified Root Key Property when bootstrap.conf is specified - * */ - protected void processBootstrapConf() { - BootstrapProperties unprotectedProperties = BootstrapPropertiesLoader.load(bootstrapConfPath.toFile()); + protected void processBootstrapConf(BootstrapProperties unprotectedProperties) { + if (flowConfigurationRequested) { + logger.info("Bootstrap Configuration [bootstrap.conf] not modified based on provided arguments"); + return; + } logger.info("Started processing Bootstrap Configuration [{}]", bootstrapConfPath); @@ -98,7 +133,7 @@ protected void processBootstrapConf() { runFileTransformer(fileTransformer2, bootstrapConfPath, outputBootstrapConf); FileTransformer fileTransformer = new BootstrapConfigurationFileTransformer(BOOTSTRAP_ROOT_KEY_PROPERTY, newRootKey); - runFileTransformer(fileTransformer, Optional.ofNullable(outputBootstrapConf).orElse(bootstrapConfPath), outputBootstrapConf); + runFileTransformer(fileTransformer, ofNullable(outputBootstrapConf).orElse(bootstrapConfPath), outputBootstrapConf); logger.info("Completed processing Bootstrap Configuration [{}]", bootstrapConfPath); } @@ -113,8 +148,8 @@ private String getRootKey() { * Run File Transformer using working path based on output path * * @param fileTransformer File Transformer to be invoked - * @param inputPath Input path of file to be transformed - * @param outputPath Output path for transformed file that defaults to the input path when not specified + * @param inputPath Input path of file to be transformed + * @param outputPath Output path for transformed file that defaults to the input path when not specified */ protected void runFileTransformer(FileTransformer fileTransformer, Path inputPath, Path outputPath) { Path configuredOutputPath = outputPath == null ? inputPath : outputPath; @@ -143,4 +178,45 @@ private Path getWorkingPath(Path resourcePath) { String workingFileName = String.format(WORKING_FILE_NAME_FORMAT, fileName, System.currentTimeMillis()); return resourcePath.resolveSibling(workingFileName); } + + private void processFlowConfiguration(BootstrapProperties unprotectedProperties) { + if (flowConfigurationPath == null) { + logger.info("Flow Configuration not specified"); + return; + } + String sensitivePropertiesKey = unprotectedProperties.getProperty(NIFI_MINIFI_SENSITIVE_PROPS_KEY.getKey()); + String sensitivePropertiesAlgorithm = unprotectedProperties.getProperty(NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM.getKey()); + if (isAnyBlank(sensitivePropertiesKey, sensitivePropertiesAlgorithm)) { + logger.info("Sensitive Properties Key or Sensitive Properties Algorithm is not provided"); + return; + } + + logger.info("Started processing Flow Configuration [{}]", flowConfigurationPath); + + byte[] flowAsBytes; + try { + flowAsBytes = readAllBytes(flowConfigurationPath); + } catch (IOException e) { + logger.error("Unable to load Flow Configuration [{}]", flowConfigurationPath); + return; + } + + FlowSerDeService flowSerDeService = StandardFlowSerDeService.defaultInstance(); + FlowPropertyEncryptor flowPropertyEncryptor = new StandardFlowPropertyEncryptor( + new PropertyEncryptorBuilder(sensitivePropertiesKey).setAlgorithm(sensitivePropertiesAlgorithm).build(), null); + + VersionedDataflow flow = flowSerDeService.deserialize(flowAsBytes); + VersionedDataflow encryptedFlow = flowPropertyEncryptor.encryptSensitiveProperties(flow); + byte[] encryptedFlowAsBytes = flowSerDeService.serialize(encryptedFlow); + + Path targetPath = ofNullable(outputFlowConfigurationPath).orElse(flowConfigurationPath); + try { + write(targetPath, encryptedFlowAsBytes); + } catch (IOException e) { + logger.error("Unable to write Flow Configuration [{}]", targetPath); + return; + } + + logger.info("Completed processing Flow Configuration [{}]", flowConfigurationPath); + } } From 264ea30bbd3144b15a87fcb3195a0f3a41b23c8f Mon Sep 17 00:00:00 2001 From: Peter Radden <162112804+pradden@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:36:02 +0000 Subject: [PATCH 16/73] NIFI-12860 Fixed NPE in ExtensionMetadata Constructor This closes #8468 Signed-off-by: David Handermann --- .../cli/impl/result/registry/ExtensionMetadataResult.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/registry/ExtensionMetadataResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/registry/ExtensionMetadataResult.java index f501a122b676..4653a09464f6 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/registry/ExtensionMetadataResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/registry/ExtensionMetadataResult.java @@ -34,7 +34,7 @@ public class ExtensionMetadataResult extends AbstractWritableResult extensionMetadata) { super(resultType); - this.extensionMetadata = Objects.requireNonNull(this.extensionMetadata); + this.extensionMetadata = Objects.requireNonNull(extensionMetadata); } @Override From 26f5fa2be044accc6ae8403daf62be621e47f33e Mon Sep 17 00:00:00 2001 From: David Handermann Date: Tue, 5 Mar 2024 15:55:04 -0600 Subject: [PATCH 17/73] NIFI-11443 Route Python Framework Logging to SLF4J (#8407) * NIFI-11443 Routed Python Framework Logging to SLF4J - Changed Python logging to use standard output stream - Adjusted Python logging format for simplified processing - Updated PythonProcess to pipe standard error and standard output streams to reader - Added Log Reader command with Virtual Thread for each Python Process - Removed Python log properties from NiFi Properties configuration --- .../org/apache/nifi/util/NiFiProperties.java | 2 - .../main/asciidoc/administration-guide.adoc | 3 - .../nifi/controller/FlowController.java | 4 - .../nifi-framework/nifi-resources/pom.xml | 1 - .../src/main/resources/conf/logback.xml | 3 + .../src/main/resources/conf/nifi.properties | 1 - .../nifi-py4j-bundle/nifi-py4j-bridge/pom.xml | 6 + .../org/apache/nifi/py4j/PythonProcess.java | 46 +++- .../nifi/py4j/PythonProcessLogReader.java | 176 ++++++++++++++ .../nifi/py4j/StandardPythonBridge.java | 6 + .../py4j/logback/LevelChangeListener.java | 145 ++++++++++++ .../py4j/logging/LogLevelChangeHandler.java | 37 +++ .../py4j/logging/LogLevelChangeListener.java | 32 +++ .../nifi/py4j/logging/PythonLogLevel.java | 67 ++++++ .../StandardLogLevelChangeHandler.java | 100 ++++++++ .../nifi/py4j/PythonProcessLogReaderTest.java | 220 ++++++++++++++++++ .../py4j/logback/LevelChangeListenerTest.java | 67 ++++++ .../PythonControllerInteractionIT.java | 1 - .../apache/nifi/python/PythonController.java | 8 + .../nifi/python/PythonProcessConfig.java | 25 -- .../src/main/python/framework/Controller.py | 19 +- .../conf/clustered/node1/nifi.properties | 1 - .../conf/clustered/node2/nifi.properties | 1 - .../resources/conf/default/nifi.properties | 1 - .../test/resources/conf/pythonic/logback.xml | 3 + .../resources/conf/pythonic/nifi.properties | 1 - 26 files changed, 919 insertions(+), 57 deletions(-) create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcessLogReader.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logback/LevelChangeListener.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeHandler.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeListener.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/PythonLogLevel.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/StandardLogLevelChangeHandler.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessLogReaderTest.java create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/logback/LevelChangeListenerTest.java diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 7109600631d9..01dd887190bf 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -319,7 +319,6 @@ public class NiFiProperties extends ApplicationProperties { public static final String PYTHON_FRAMEWORK_SOURCE_DIRECTORY = "nifi.python.framework.source.directory"; public static final String PYTHON_EXTENSION_DIRECTORY_PREFIX = "nifi.python.extensions.source.directory."; public static final String PYTHON_WORKING_DIRECTORY = "nifi.python.working.directory"; - public static final String PYTHON_LOGS_DIRECTORY = "nifi.python.logs.directory"; public static final String PYTHON_MAX_PROCESSES = "nifi.python.max.processes"; public static final String PYTHON_MAX_PROCESSES_PER_TYPE = "nifi.python.max.processes.per.extension.type"; public static final String PYTHON_COMMS_TIMEOUT = "nifi.python.comms.timeout"; @@ -327,7 +326,6 @@ public class NiFiProperties extends ApplicationProperties { public static final String PYTHON_CONTROLLER_DEBUGPY_ENABLED = "nifi.python.controller.debugpy.enabled"; public static final String PYTHON_CONTROLLER_DEBUGPY_PORT = "nifi.python.controller.debugpy.port"; public static final String PYTHON_CONTROLLER_DEBUGPY_HOST = "nifi.python.controller.debugpy.host"; - public static final String PYTHON_CONTROLLER_DEBUGPY_LOGS_DIR = "nifi.python.controller.debugpy.logs.directory"; // kubernetes properties public static final String CLUSTER_LEADER_ELECTION_KUBERNETES_LEASE_PREFIX = "nifi.cluster.leader.election.kubernetes.lease.prefix"; diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index db31ffa366e0..72e073c6521a 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -247,9 +247,6 @@ located as a sibling of this directory. For example, if the value of this proper by default, but multiple Python Extension directories can be added by adding additional properties with the prefix `nifi.python.extensions.source.directory.`. | nifi.python.working.directory | ./work/python | The working directory where NiFi should store artifacts, such as any third-party libraries that are downloaded as dependencies for the Python Processors. -| nifi.python.logs.directory | ./logs | The directory where NiFi should store the logs for the Python processes. If Python Processors use the in-built logger, the messages will be logged to the -app-log. However, if they create their own Python logger, the messages will be written to the Python-specific log file(s). Additionally, the Python framework will write to the Python-specific log -file(s). | nifi.python.max.processes.per.extension.type | 10 | The maximum number of Python processes that should be spawned for any one type of Processor. Because Python does not scale vertically, adding many NiFi Processors within the same Python process would yield very poor performance. Instead, NiFi creates a Python process for every Python Processor that is added to the canvas, within limits. This property indicates the maximum number of Python processes that can be created for any particular type of Processor. For example, if there are 5 instances of the diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 17e8c9e01384..727f71f25fa1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -871,7 +871,6 @@ private PythonBridge createPythonBridge(final NiFiProperties nifiProperties, fin final File pythonFrameworkSourceDirectory = nifiProperties.getPythonFrameworkSourceDirectory(); final List pythonExtensionsDirectories = nifiProperties.getPythonExtensionsDirectories(); final File pythonWorkingDirectory = new File(nifiProperties.getProperty(NiFiProperties.PYTHON_WORKING_DIRECTORY)); - final File pythonLogsDirectory = new File(nifiProperties.getProperty(NiFiProperties.PYTHON_LOGS_DIRECTORY)); int maxProcesses = nifiProperties.getIntegerProperty(NiFiProperties.PYTHON_MAX_PROCESSES, 20); int maxProcessesPerType = nifiProperties.getIntegerProperty(NiFiProperties.PYTHON_MAX_PROCESSES_PER_TYPE, 2); @@ -879,7 +878,6 @@ private PythonBridge createPythonBridge(final NiFiProperties nifiProperties, fin final boolean enableControllerDebug = Boolean.parseBoolean(nifiProperties.getProperty(NiFiProperties.PYTHON_CONTROLLER_DEBUGPY_ENABLED, "false")); final int debugPort = nifiProperties.getIntegerProperty(NiFiProperties.PYTHON_CONTROLLER_DEBUGPY_PORT, 5678); final String debugHost = nifiProperties.getProperty(NiFiProperties.PYTHON_CONTROLLER_DEBUGPY_HOST, "localhost"); - final String debugLogs = nifiProperties.getProperty(NiFiProperties.PYTHON_CONTROLLER_DEBUGPY_LOGS_DIR, "logs"); // Validate configuration for max numbers of processes. if (maxProcessesPerType < 1) { @@ -906,7 +904,6 @@ private PythonBridge createPythonBridge(final NiFiProperties nifiProperties, fin .pythonCommand(pythonCommand) .pythonFrameworkDirectory(pythonFrameworkSourceDirectory) .pythonExtensionsDirectories(pythonExtensionsDirectories) - .pythonLogsDirectory(pythonLogsDirectory) .pythonWorkingDirectory(pythonWorkingDirectory) .commsTimeout(commsTimeout == null ? null : Duration.ofMillis(FormatUtils.getTimeDuration(commsTimeout, TimeUnit.MILLISECONDS))) .maxPythonProcesses(maxProcesses) @@ -914,7 +911,6 @@ private PythonBridge createPythonBridge(final NiFiProperties nifiProperties, fin .enableControllerDebug(enableControllerDebug) .debugPort(debugPort) .debugHost(debugHost) - .debugLogsDirectory(new File(debugLogs)) .build(); final ControllerServiceTypeLookup serviceTypeLookup = serviceProvider::getControllerServiceType; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index b973224863a9..56aeebfd4aa6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -268,7 +268,6 @@ ./work/python 100 10 - ./logs diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml index ae33e0484e7c..cbeff47ef6bb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml @@ -174,6 +174,9 @@ + + + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index 922e4f9deaee..314715253561 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -49,7 +49,6 @@ nifi.python.extensions.source.directory.default=${nifi.python.extensions.source. nifi.python.working.directory=${nifi.python.working.directory} nifi.python.max.processes=${nifi.python.max.processes} nifi.python.max.processes.per.extension.type=${nifi.python.max.processes.per.extension.type} -nifi.python.logs.directory=${nifi.python.logs.dir} #################### # State Management # diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml index 655abc75f24d..c7c05cad9f25 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/pom.xml @@ -42,6 +42,12 @@ py4j 0.10.9.7 + + + ch.qos.logback + logback-classic + provided + org.apache.nifi diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java index 2180681c0dc6..f34199b5bcab 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java @@ -17,6 +17,10 @@ package org.apache.nifi.py4j; +import org.apache.nifi.py4j.logging.LogLevelChangeListener; +import org.apache.nifi.py4j.logging.PythonLogLevel; +import org.apache.nifi.py4j.logging.StandardLogLevelChangeHandler; +import org.apache.nifi.logging.LogLevel; import org.apache.nifi.py4j.client.JavaObjectBindings; import org.apache.nifi.py4j.client.NiFiPythonGateway; import org.apache.nifi.py4j.client.StandardPythonClient; @@ -36,6 +40,7 @@ import javax.net.SocketFactory; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.security.SecureRandom; import java.util.ArrayList; @@ -52,6 +57,8 @@ public class PythonProcess { private static final Logger logger = LoggerFactory.getLogger(PythonProcess.class); private static final String PYTHON_CONTROLLER_FILENAME = "Controller.py"; + private static final String LOG_READER_THREAD_NAME_FORMAT = "python-log-%d"; + private final PythonProcessConfig processConfig; private final ControllerServiceTypeLookup controllerServiceTypeLookup; private final File virtualEnvHome; @@ -66,6 +73,8 @@ public class PythonProcess { private volatile boolean shutdown = false; private volatile List extensionDirs; private volatile String workDir; + private Thread logReaderThread; + private String logListenerId; public PythonProcess(final PythonProcessConfig processConfig, final ControllerServiceTypeLookup controllerServiceTypeLookup, final File virtualEnvHome, @@ -118,6 +127,10 @@ public synchronized void start() throws IOException { this.process = launchPythonProcess(listeningPort, authToken); this.process.onExit().thenAccept(this::handlePythonProcessDied); + final String logReaderThreadName = LOG_READER_THREAD_NAME_FORMAT.formatted(process.pid()); + final Runnable logReaderCommand = new PythonProcessLogReader(process.inputReader(StandardCharsets.UTF_8)); + this.logReaderThread = Thread.ofVirtual().name(logReaderThreadName).start(logReaderCommand); + final StandardPythonClient pythonClient = new StandardPythonClient(gateway); controller = pythonClient.getController(); @@ -151,6 +164,9 @@ public synchronized void start() throws IOException { throw new RuntimeException("Failed to start Python Bridge", lastException); } + logListenerId = Long.toString(process.pid()); + StandardLogLevelChangeHandler.getHandler().addListener(logListenerId, new PythonProcessLogLevelChangeListener()); + controller.setControllerServiceTypeLookup(controllerServiceTypeLookup); logger.info("Successfully started and pinged Python Server. Python Process = {}", process); } @@ -211,7 +227,6 @@ private String generateAuthToken() { private Process launchPythonProcess(final int listeningPort, final String authToken) throws IOException { final File pythonFrameworkDirectory = processConfig.getPythonFrameworkDirectory(); final File pythonApiDirectory = new File(pythonFrameworkDirectory.getParentFile(), "api"); - final File pythonLogsDirectory = processConfig.getPythonLogsDirectory(); final File pythonCmdFile = new File(processConfig.getPythonCommand()); final String pythonCmd = pythonCmdFile.getName(); final File pythonCommandFile = new File(virtualEnvHome, "bin/" + pythonCmd); @@ -225,14 +240,12 @@ private Process launchPythonProcess(final int listeningPort, final String authTo String pythonPath = pythonApiDirectory.getAbsolutePath(); - if (processConfig.isDebugController() && "Controller".equals(componentId)) { commands.add("-m"); commands.add("debugpy"); commands.add("--listen"); commands.add(processConfig.getDebugHost() + ":" + processConfig.getDebugPort()); - commands.add("--log-to"); - commands.add(processConfig.getDebugLogsDirectory().getAbsolutePath()); + commands.add("--log-to-stderr"); pythonPath = pythonPath + File.pathSeparator + virtualEnvHome.getAbsolutePath(); } @@ -241,12 +254,13 @@ private Process launchPythonProcess(final int listeningPort, final String authTo processBuilder.command(commands); processBuilder.environment().put("JAVA_PORT", String.valueOf(listeningPort)); - processBuilder.environment().put("LOGS_DIR", pythonLogsDirectory.getAbsolutePath()); processBuilder.environment().put("ENV_HOME", virtualEnvHome.getAbsolutePath()); processBuilder.environment().put("PYTHONPATH", pythonPath); processBuilder.environment().put("PYTHON_CMD", pythonCommandFile.getAbsolutePath()); processBuilder.environment().put("AUTH_TOKEN", authToken); - processBuilder.inheritIO(); + + // Redirect error stream to standard output stream + processBuilder.redirectErrorStream(true); logger.info("Launching Python Process {} {} with working directory {} to communicate with Java on Port {}", pythonCommand, controllerPyFile.getAbsolutePath(), virtualEnvHome, listeningPort); @@ -329,6 +343,8 @@ public void shutdown() { } private synchronized void killProcess() { + StandardLogLevelChangeHandler.getHandler().removeListener(logListenerId); + if (server != null) { try { server.shutdown(); @@ -358,6 +374,10 @@ private synchronized void killProcess() { process = null; } + + if (logReaderThread != null) { + logReaderThread.interrupt(); + } } public void discoverExtensions(final List directories, final String workDirectory) { @@ -433,4 +453,18 @@ public Map getJavaObjectBindingCounts() { } private record CreatedProcessor(String identifier, String type, PythonProcessorBridge processorBridge) {} + + private class PythonProcessLogLevelChangeListener implements LogLevelChangeListener { + /** + * Publish log level changes to Python Controller with conversion from framework log level to Python log level + * + * @param loggerName Name of logger with updated level + * @param logLevel New log level + */ + @Override + public void onLevelChange(final String loggerName, final LogLevel logLevel) { + final PythonLogLevel pythonLogLevel = PythonLogLevel.valueOf(logLevel); + controller.setLoggerLevel(loggerName, pythonLogLevel.getLevel()); + } + } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcessLogReader.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcessLogReader.java new file mode 100644 index 000000000000..4a52bc35db07 --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcessLogReader.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j; + +import org.apache.nifi.py4j.logging.PythonLogLevel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.stream.Collectors; + +/** + * Runnable Command for reading a line from Process Output Stream and writing to a Logger + */ +class PythonProcessLogReader implements Runnable { + private static final int LOG_LEVEL_BEGIN_INDEX = 0; + + private static final int LOG_LEVEL_END_INDEX = 2; + + private static final int MESSAGE_BEGIN_INDEX = 3; + + private static final char NAME_MESSAGE_SEPARATOR = ':'; + + private static final int MINIMUM_LOGGER_NAME_INDEX = 3; + + private static final String LOG_PREFIX = "PY4JLOG"; + + private static final int PREFIXED_LOG_LEVEL_BEGIN_INDEX = 8; + + private static final String LINE_SEPARATOR = System.lineSeparator(); + + private static final Map PYTHON_LOG_LEVELS = Arrays.stream(PythonLogLevel.values()).collect( + Collectors.toUnmodifiableMap( + pythonLogLevel -> Integer.toString(pythonLogLevel.getLevel()), + pythonLogLevel -> pythonLogLevel + ) + ); + + private final Logger processLogger = LoggerFactory.getLogger("org.apache.nifi.py4j.ProcessLog"); + + private final BufferedReader processReader; + + /** + * Standard constructor with Buffered Reader connected to Python Process Output Stream + * + * @param processReader Reader from Process Output Stream + */ + PythonProcessLogReader(final BufferedReader processReader) { + this.processReader = Objects.requireNonNull(processReader, "Reader required"); + } + + /** + * Read lines from Process Reader and write log messages based on parsed level and named logger + */ + @Override + public void run() { + final Queue parsedRecords = new ArrayDeque<>(); + + try { + String line = processReader.readLine(); + while (line != null) { + try { + processLine(line, parsedRecords); + + if (parsedRecords.size() == 2) { + // Log previous record after creating a new record + final ParsedRecord parsedRecord = parsedRecords.remove(); + log(parsedRecord); + } + + if (!processReader.ready()) { + // Log queued records when Process Reader is not ready + while (!parsedRecords.isEmpty()) { + final ParsedRecord parsedRecord = parsedRecords.poll(); + log(parsedRecord); + } + } + } catch (final Exception e) { + processLogger.error("Failed to handle log from Python Process", e); + } + + // Read and block for subsequent lines + line = processReader.readLine(); + } + } catch (final IOException e) { + processLogger.error("Failed to read output of Python Process", e); + } + + // Handle last buffered message following closure of process stream + for (final ParsedRecord parsedRecord : parsedRecords) { + log(parsedRecord); + } + } + + private void processLine(final String line, final Queue parsedRecords) { + final int logPrefixIndex = line.indexOf(LOG_PREFIX); + if (logPrefixIndex == 0) { + final String levelLogLine = line.substring(PREFIXED_LOG_LEVEL_BEGIN_INDEX); + final String levelNumber = levelLogLine.substring(LOG_LEVEL_BEGIN_INDEX, LOG_LEVEL_END_INDEX); + final PythonLogLevel logLevel = PYTHON_LOG_LEVELS.getOrDefault(levelNumber, PythonLogLevel.NOTSET); + final String loggerMessage = levelLogLine.substring(MESSAGE_BEGIN_INDEX); + + final int nameSeparatorIndex = loggerMessage.indexOf(NAME_MESSAGE_SEPARATOR); + final String message; + final Logger logger; + if (nameSeparatorIndex < MINIMUM_LOGGER_NAME_INDEX) { + // Set ProcessLog when named logger not found + logger = processLogger; + message = loggerMessage; + } else { + final String loggerName = loggerMessage.substring(0, nameSeparatorIndex); + logger = LoggerFactory.getLogger(loggerName); + + final int messageBeginIndex = nameSeparatorIndex + 1; + message = loggerMessage.substring(messageBeginIndex); + } + + final StringBuilder buffer = new StringBuilder(message); + final ParsedRecord parsedRecord = new ParsedRecord(logLevel, logger, buffer); + parsedRecords.add(parsedRecord); + } else { + final ParsedRecord lastRecord = parsedRecords.peek(); + if (lastRecord == null) { + final StringBuilder buffer = new StringBuilder(line); + final ParsedRecord parsedRecord = new ParsedRecord(PythonLogLevel.INFO, processLogger, buffer); + parsedRecords.add(parsedRecord); + } else if (!line.isEmpty()) { + // Add line separator for buffering multiple lines in the same record + lastRecord.buffer.append(LINE_SEPARATOR); + lastRecord.buffer.append(line); + } + } + } + + private void log(final ParsedRecord parsedRecord) { + final PythonLogLevel logLevel = parsedRecord.level; + final Logger logger = parsedRecord.logger; + final String message = parsedRecord.buffer.toString(); + + if (PythonLogLevel.DEBUG == logLevel) { + logger.debug(message); + } else if (PythonLogLevel.INFO == logLevel) { + logger.info(message); + } else if (PythonLogLevel.WARNING == logLevel) { + logger.warn(message); + } else if (PythonLogLevel.ERROR == logLevel) { + logger.error(message); + } else if (PythonLogLevel.CRITICAL == logLevel) { + logger.error(message); + } else { + logger.warn(message); + } + } + + private record ParsedRecord(PythonLogLevel level, Logger logger, StringBuilder buffer) {} +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java index bb3cbe496a63..cc2a5d8489d6 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/StandardPythonBridge.java @@ -18,6 +18,9 @@ package org.apache.nifi.py4j; import org.apache.nifi.components.AsyncLoadedProcessor; +import org.apache.nifi.py4j.logback.LevelChangeListener; +import org.apache.nifi.py4j.logging.LogLevelChangeHandler; +import org.apache.nifi.py4j.logging.StandardLogLevelChangeHandler; import org.apache.nifi.python.BoundObjectCounts; import org.apache.nifi.python.ControllerServiceTypeLookup; import org.apache.nifi.python.PythonBridge; @@ -71,6 +74,9 @@ public synchronized void start() throws IOException { logger.debug("{} launching Python Process", this); try { + final LogLevelChangeHandler logLevelChangeHandler = StandardLogLevelChangeHandler.getHandler(); + LevelChangeListener.registerLogbackListener(logLevelChangeHandler); + final File envHome = new File(processConfig.getPythonWorkingDirectory(), "controller"); controllerProcess = new PythonProcess(processConfig, serviceTypeLookup, envHome, "Controller", "Controller"); controllerProcess.start(); diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logback/LevelChangeListener.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logback/LevelChangeListener.java new file mode 100644 index 000000000000..8efc49b36ce8 --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logback/LevelChangeListener.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j.logback; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.spi.LoggerContextListener; +import org.apache.nifi.py4j.logging.LogLevelChangeListener; +import org.apache.nifi.logging.LogLevel; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Logback Listener implementation for tracking Logger Level changes and notifying framework listeners + */ +public class LevelChangeListener implements LoggerContextListener { + private static final Map LEVELS = Map.of( + Level.ALL, LogLevel.TRACE, + Level.TRACE, LogLevel.TRACE, + Level.DEBUG, LogLevel.DEBUG, + Level.INFO, LogLevel.INFO, + Level.WARN, LogLevel.WARN, + Level.ERROR, LogLevel.ERROR, + Level.OFF, LogLevel.NONE + ); + + private static final List EXCLUDED_LOGGERS = List.of( + Pattern.compile("^$"), + Pattern.compile("^org|com|net$"), + Pattern.compile("^jetbrains.*"), + Pattern.compile("^org\\.apache.*"), + Pattern.compile("^org\\.eclipse.*"), + Pattern.compile("^org\\.glassfish.*"), + Pattern.compile("^org\\.springframework.*"), + Pattern.compile("^org\\.opensaml.*"), + Pattern.compile("^software\\.amazon.*") + ); + + private final LogLevelChangeListener logLevelChangeListener; + + LevelChangeListener(final LogLevelChangeListener logLevelChangeListener) { + this.logLevelChangeListener = Objects.requireNonNull(logLevelChangeListener, "Listener required"); + } + + /** + * Register an instance of Logback LevelChangeListener routing to framework LogLevelChangeListener + */ + public static void registerLogbackListener(final LogLevelChangeListener logLevelChangeListener) { + final LevelChangeListener levelChangeListener = new LevelChangeListener(logLevelChangeListener); + final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); + if (loggerFactory instanceof LoggerContext loggerContext) { + loggerContext.addListener(levelChangeListener); + levelChangeListener.onStart(loggerContext); + } + } + + /** + * Reset Resistant enabled to support persistent behavior when the LoggerContext is changed + * + * @return Reset Resistant enabled + */ + @Override + public boolean isResetResistant() { + return true; + } + + /** + * Set initial Logger Levels from registered loggers when starting + * + * @param loggerContext Logback Context + */ + @Override + public void onStart(final LoggerContext loggerContext) { + for (final Logger logger : loggerContext.getLoggerList()) { + onLevelChange(logger, logger.getEffectiveLevel()); + } + } + + /** + * Set Logger Levels from registered loggers when resetting + * + * @param loggerContext Logback Context + */ + @Override + public void onReset(final LoggerContext loggerContext) { + onStart(loggerContext); + } + + @Override + public void onStop(final LoggerContext loggerContext) { + + } + + /** + * Send logger level changes to framework handler for subsequent notification + * + * @param logger Logback Logger with new level + * @param level New level assigned for Logback Logger + */ + @Override + public void onLevelChange(final Logger logger, final Level level) { + Objects.requireNonNull(level, "Level required"); + + final String loggerName = logger.getName(); + if (isLoggerSupported(loggerName)) { + final LogLevel logLevel = LEVELS.getOrDefault(level, LogLevel.WARN); + logLevelChangeListener.onLevelChange(loggerName, logLevel); + } + } + + private boolean isLoggerSupported(final String loggerName) { + boolean supported = true; + + for (final Pattern loggerPattern : EXCLUDED_LOGGERS) { + final Matcher matcher = loggerPattern.matcher(loggerName); + if (matcher.matches()) { + supported = false; + break; + } + } + + return supported; + } +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeHandler.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeHandler.java new file mode 100644 index 000000000000..3cf50a99c080 --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeHandler.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j.logging; + +/** + * Handler abstraction for registering Listeners and invoking Listeners on level changes + */ +public interface LogLevelChangeHandler extends LogLevelChangeListener { + /** + * Register Log Level Change Listener with provided identifier + * + * @param identifier Tracking identifier associated with Log Level Change Listener + * @param listener Log Level Change Listener to be registered + */ + void addListener(String identifier, LogLevelChangeListener listener); + + /** + * Remove registered listener based on provided identifier + * + * @param identifier Tracking identifier of registered listener to be removed + */ + void removeListener(String identifier); +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeListener.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeListener.java new file mode 100644 index 000000000000..d9b2077f4d5e --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/LogLevelChangeListener.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j.logging; + +import org.apache.nifi.logging.LogLevel; + +/** + * Listener abstraction for subscribing to notification of level changes for named loggers + */ +public interface LogLevelChangeListener { + /** + * Handle level change notification for named logger + * + * @param loggerName Name of logger with updated level + * @param logLevel New log level + */ + void onLevelChange(String loggerName, LogLevel logLevel); +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/PythonLogLevel.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/PythonLogLevel.java new file mode 100644 index 000000000000..f32e5245c5bb --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/PythonLogLevel.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j.logging; + +import org.apache.nifi.logging.LogLevel; + +import java.util.Map; + +/** + * Python Log Level enumeration with level numbers according to Python logging module documentation + */ +public enum PythonLogLevel { + NOTSET(0), + + DEBUG(10), + + INFO(20), + + WARNING(30), + + ERROR(40), + + CRITICAL(50); + + private static final Map FRAMEWORK_LEVELS = Map.of( + LogLevel.NONE, NOTSET, + LogLevel.DEBUG, DEBUG, + LogLevel.INFO, INFO, + LogLevel.WARN, WARNING, + LogLevel.ERROR, ERROR, + LogLevel.FATAL, CRITICAL + ); + + private final int level; + + PythonLogLevel(final int level) { + this.level = level; + } + + public int getLevel() { + return level; + } + + /** + * Get Python Log Level equivalent from framework Log Level + * + * @param logLevel Framework Log Level + * @return Python Log Level + */ + public static PythonLogLevel valueOf(LogLevel logLevel) { + return FRAMEWORK_LEVELS.getOrDefault(logLevel, WARNING); + } +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/StandardLogLevelChangeHandler.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/StandardLogLevelChangeHandler.java new file mode 100644 index 000000000000..150798c3177d --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/logging/StandardLogLevelChangeHandler.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j.logging; + +import org.apache.nifi.logging.LogLevel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Standard implementation of Log Level Change Handler with singleton instance for shared collection of listeners + */ +public class StandardLogLevelChangeHandler implements LogLevelChangeHandler { + private static final StandardLogLevelChangeHandler HANDLER = new StandardLogLevelChangeHandler(); + + private static final Logger handlerLogger = LoggerFactory.getLogger(StandardLogLevelChangeHandler.class); + + private final Map listeners = new ConcurrentHashMap<>(); + + private final Map loggerLevels = new ConcurrentHashMap<>(); + + private StandardLogLevelChangeHandler() { + + } + + /** + * Get shared reference to Handler for Listener registration and centralized notification + * + * @return Shared Log Level Change Handler + */ + public static LogLevelChangeHandler getHandler() { + return HANDLER; + } + + /** + * Register Log Level Change Listener with provided identifier + * + * @param identifier Tracking identifier associated with Log Level Change Listener + * @param listener Log Level Change Listener to be registered + */ + @Override + public void addListener(final String identifier, final LogLevelChangeListener listener) { + Objects.requireNonNull(identifier, "Identifier required"); + Objects.requireNonNull(listener, "Listener required"); + + for (final Map.Entry loggerLevel : loggerLevels.entrySet()) { + listener.onLevelChange(loggerLevel.getKey(), loggerLevel.getValue()); + } + + listeners.put(identifier, listener); + handlerLogger.trace("Added Listener [{}]", identifier); + } + + /** + * Remove registered listener based on provided identifier + * + * @param identifier Tracking identifier of registered listener to be removed + */ + @Override + public void removeListener(String identifier) { + Objects.requireNonNull(identifier, "Identifier required"); + listeners.remove(identifier); + handlerLogger.trace("Removed Listener [{}]", identifier); + } + + /** + * Handle level change notification for named logger and broadcast to registered Listeners + * + * @param loggerName Name of logger with updated level + * @param logLevel New log level + */ + @Override + public void onLevelChange(final String loggerName, final LogLevel logLevel) { + Objects.requireNonNull(loggerName, "Logger Name required"); + Objects.requireNonNull(logLevel, "Log Level required"); + handlerLogger.trace("Logger [{}] Level [{}] changed", loggerName, logLevel); + + loggerLevels.put(loggerName, logLevel); + for (final LogLevelChangeListener listener : listeners.values()) { + listener.onLevelChange(loggerName, logLevel); + } + } +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessLogReaderTest.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessLogReaderTest.java new file mode 100644 index 000000000000..2db386df9ad1 --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessLogReaderTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j; + +import org.apache.nifi.py4j.logging.PythonLogLevel; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PythonProcessLogReaderTest { + + private static final String PROCESS_LOGGER = "org.apache.nifi.py4j.ProcessLog"; + + private static final String CONTROLLER_LOGGER = "org.apache.nifi.py4j.Controller"; + + private static final String LOG_MESSAGE = "Testing Python Processing"; + + private static final String LINE_FORMAT = "PY4JLOG %s %s:%s %d%n"; + + private static final String LINE_UNFORMATTED = "Testing message without level or logger"; + + private static final String LINE_MISSING_SEPARATOR = "%s %s".formatted(PythonLogLevel.INFO.getLevel(), LOG_MESSAGE); + + private static final String LINE_EMPTY = ""; + + private static final String MESSAGE_TRACEBACK = String.format("Command Failed%nTrackback (most recent call last):%n File: command.py, line 1%nError: name is not defined"); + + private static final int FIRST_LOG = 1; + + private static final String FIRST_LOG_MESSAGE = String.format("%s %d", LOG_MESSAGE, FIRST_LOG); + + private static final String NUMBERED_LOG_FORMAT = "%s %d"; + + @Mock + private Logger processLogger; + + @Mock + private Logger controllerLogger; + + @Test + void testDebug() { + runCommand(PythonLogLevel.DEBUG, controllerLogger); + + verify(controllerLogger).debug(eq(FIRST_LOG_MESSAGE)); + } + + @Test + void testInfo() { + runCommand(PythonLogLevel.INFO, controllerLogger); + + verify(controllerLogger).info(eq(FIRST_LOG_MESSAGE)); + } + + @Test + void testWarning() { + runCommand(PythonLogLevel.WARNING, controllerLogger); + + verify(controllerLogger).warn(eq(FIRST_LOG_MESSAGE)); + } + + @Test + void testError() { + runCommand(PythonLogLevel.ERROR, controllerLogger); + + verify(controllerLogger).error(eq(FIRST_LOG_MESSAGE)); + } + + @Test + void testCritical() { + runCommand(PythonLogLevel.CRITICAL, controllerLogger); + + verify(controllerLogger).error(eq(FIRST_LOG_MESSAGE)); + } + + @Test + void testUnformatted() { + try (MockedStatic loggerFactory = mockStatic(LoggerFactory.class)) { + setupLogger(loggerFactory, processLogger, PROCESS_LOGGER); + + final BufferedReader reader = new BufferedReader(new StringReader(LINE_UNFORMATTED)); + final Runnable command = new PythonProcessLogReader(reader); + command.run(); + + verify(processLogger).info(eq(LINE_UNFORMATTED)); + } + } + + @Test + void testMissingSeparator() { + try (MockedStatic loggerFactory = mockStatic(LoggerFactory.class)) { + setupLogger(loggerFactory, processLogger, PROCESS_LOGGER); + + final BufferedReader reader = new BufferedReader(new StringReader(LINE_MISSING_SEPARATOR)); + final Runnable command = new PythonProcessLogReader(reader); + command.run(); + + verify(processLogger).info(eq(LINE_MISSING_SEPARATOR)); + } + } + + @Test + void testEmpty() { + try (MockedStatic loggerFactory = mockStatic(LoggerFactory.class)) { + setupLogger(loggerFactory, processLogger, PROCESS_LOGGER); + + final BufferedReader reader = new BufferedReader(new StringReader(LINE_EMPTY)); + final Runnable command = new PythonProcessLogReader(reader); + command.run(); + + verifyNoInteractions(processLogger); + } + } + + @Test + void testInfoMultipleMessages() { + final int messages = 2; + + try (MockedStatic loggerFactory = mockStatic(LoggerFactory.class)) { + setupLogger(loggerFactory, controllerLogger, CONTROLLER_LOGGER); + + final StringBuilder builder = new StringBuilder(); + for (int i = 0; i < messages; i++) { + builder.append(getLine(PythonLogLevel.INFO, i)); + builder.append(System.lineSeparator()); + } + + final String lines = builder.toString(); + final BufferedReader reader = new BufferedReader(new StringReader(lines)); + final Runnable command = new PythonProcessLogReader(reader); + command.run(); + } + + for (int i = 0; i < messages; i++) { + final String expected = NUMBERED_LOG_FORMAT.formatted(LOG_MESSAGE, i); + verify(controllerLogger).info(eq(expected)); + } + } + + @Test + void testErrorMultipleLines() { + try (MockedStatic loggerFactory = mockStatic(LoggerFactory.class)) { + setupLogger(loggerFactory, controllerLogger, CONTROLLER_LOGGER); + + final String lines = LINE_FORMAT.formatted(PythonLogLevel.ERROR.getLevel(), CONTROLLER_LOGGER, MESSAGE_TRACEBACK, FIRST_LOG); + + final BufferedReader reader = new BufferedReader(new StringReader(lines)); + final Runnable command = new PythonProcessLogReader(reader); + command.run(); + } + + final String expected = NUMBERED_LOG_FORMAT.formatted(MESSAGE_TRACEBACK, FIRST_LOG); + verify(controllerLogger).error(eq(expected)); + } + + @Test + void testReaderException() throws IOException { + try (MockedStatic loggerFactory = mockStatic(LoggerFactory.class)) { + setupLogger(loggerFactory, processLogger, PROCESS_LOGGER); + + final BufferedReader reader = mock(BufferedReader.class); + when(reader.readLine()).thenThrow(new IOException()); + + final Runnable command = new PythonProcessLogReader(reader); + command.run(); + + verify(processLogger).error(anyString(), isA(IOException.class)); + } + } + + private void runCommand(final PythonLogLevel logLevel, final Logger logger) { + try (MockedStatic loggerFactory = mockStatic(LoggerFactory.class)) { + setupLogger(loggerFactory, logger, CONTROLLER_LOGGER); + + final String line = getLine(logLevel, FIRST_LOG); + final BufferedReader reader = new BufferedReader(new StringReader(line)); + final Runnable command = new PythonProcessLogReader(reader); + command.run(); + } + } + + private void setupLogger(final MockedStatic loggerFactory, final Logger logger, final String loggerName) { + loggerFactory.when(() -> LoggerFactory.getLogger(eq(loggerName))).thenReturn(logger); + } + + private String getLine(final PythonLogLevel logLevel, final int number) { + return LINE_FORMAT.formatted(logLevel.getLevel(), CONTROLLER_LOGGER, LOG_MESSAGE, number); + } +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/logback/LevelChangeListenerTest.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/logback/LevelChangeListenerTest.java new file mode 100644 index 000000000000..6e4a4c06bb8b --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/logback/LevelChangeListenerTest.java @@ -0,0 +1,67 @@ +package org.apache.nifi.py4j.logback;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import org.apache.nifi.py4j.logging.LogLevelChangeListener; +import org.apache.nifi.py4j.logging.StandardLogLevelChangeHandler; +import org.apache.nifi.logging.LogLevel; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class LevelChangeListenerTest { + + private static final String LOGGER_NAME = Test.class.getName(); + + @Mock + private Logger logger; + + @Mock + private LogLevelChangeListener logLevelChangeListener; + + private LevelChangeListener listener; + + @BeforeEach + void setListener() { + listener = new LevelChangeListener(StandardLogLevelChangeHandler.getHandler()); + + StandardLogLevelChangeHandler.getHandler().addListener(LogLevelChangeListener.class.getSimpleName(), logLevelChangeListener); + } + + @AfterEach + void removeListener() { + StandardLogLevelChangeHandler.getHandler().removeListener(LogLevelChangeListener.class.getSimpleName()); + } + + @Test + void testOnLevelChange() { + when(logger.getName()).thenReturn(LOGGER_NAME); + + listener.onLevelChange(logger, Level.INFO); + + verify(logLevelChangeListener).onLevelChange(eq(LOGGER_NAME), eq(LogLevel.INFO)); + } +} diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java index c41d459a171d..7fb03991661c 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-integration-tests/src/test/java/org.apache.nifi.py4j/PythonControllerInteractionIT.java @@ -89,7 +89,6 @@ public static void launchPython() throws IOException { .commsTimeout(Duration.ofSeconds(0)) .maxPythonProcessesPerType(25) .maxPythonProcesses(100) - .pythonLogsDirectory(logsDir) .build(); Files.createDirectories(logsDir.toPath()); diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java index c5ae45bde25e..b840603b9e77 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonController.java @@ -99,4 +99,12 @@ public interface PythonController { * @return the details that have been discovered */ PythonProcessorDetails getProcessorDetails(String type, String version); + + /** + * Set level for specified logger name in Python logging.Logger objects + * + * @param loggerName Python logger name to be updated + * @param level Python log level according to Python logging module documentation + */ + void setLoggerLevel(String loggerName, int level); } diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessConfig.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessConfig.java index 72c8c85d4ce6..4e32b5e89d8b 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessConfig.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework-api/src/main/java/org/apache/nifi/python/PythonProcessConfig.java @@ -30,20 +30,17 @@ public class PythonProcessConfig { private final File pythonFrameworkDirectory; private final List pythonExtensionsDirectories; private final File pythonWorkingDirectory; - private final File pythonLogsDirectory; private final Duration commsTimeout; private final int maxPythonProcesses; private final int maxPythonProcessesPerType; private final boolean debugController; private final String debugHost; private final int debugPort; - private final File debugLogsDirectory; private PythonProcessConfig(final Builder builder) { this.pythonCommand = builder.pythonCommand; this.pythonFrameworkDirectory = builder.pythonFrameworkDirectory; this.pythonExtensionsDirectories = builder.pythonExtensionsDirectories; - this.pythonLogsDirectory = builder.pythonLogsDirectory; this.pythonWorkingDirectory = builder.pythonWorkingDirectory; this.commsTimeout = builder.commsTimeout; this.maxPythonProcesses = builder.maxProcesses; @@ -51,7 +48,6 @@ private PythonProcessConfig(final Builder builder) { this.debugController = builder.debugController; this.debugPort = builder.debugPort; this.debugHost = builder.debugHost; - this.debugLogsDirectory = builder.debugLogsDirectory; } public String getPythonCommand() { @@ -66,10 +62,6 @@ public List getPythonExtensionsDirectories() { return pythonExtensionsDirectories; } - public File getPythonLogsDirectory() { - return pythonLogsDirectory; - } - public File getPythonWorkingDirectory() { return pythonWorkingDirectory; } @@ -98,15 +90,10 @@ public int getDebugPort() { return debugPort; } - public File getDebugLogsDirectory() { - return debugLogsDirectory; - } - public static class Builder { private String pythonCommand = "python3"; private File pythonFrameworkDirectory = new File("python/framework"); private List pythonExtensionsDirectories = Collections.singletonList(new File("python/extensions")); - private File pythonLogsDirectory = new File("./logs"); private File pythonWorkingDirectory = new File("python"); private Duration commsTimeout = Duration.ofSeconds(0); private int maxProcesses; @@ -114,8 +101,6 @@ public static class Builder { private boolean debugController = false; private String debugHost = "localhost"; private int debugPort = 5678; - private File debugLogsDirectory = new File("logs/"); - public Builder pythonCommand(final String command) { this.pythonCommand = command; @@ -164,11 +149,6 @@ public Builder maxPythonProcessesPerType(final int maxProcessesPerType) { return this; } - public Builder pythonLogsDirectory(final File logsDirectory) { - this.pythonLogsDirectory = logsDirectory; - return this; - } - public Builder enableControllerDebug(final boolean enableDebug) { this.debugController = enableDebug; return this; @@ -184,11 +164,6 @@ public Builder debugHost(final String debugHost) { return this; } - public Builder debugLogsDirectory(final File debugLogsDirectory) { - this.debugLogsDirectory = debugLogsDirectory; - return this; - } - public PythonProcessConfig build() { return new PythonProcessConfig(this); } diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py index 8919924c065f..05e0d4f6b085 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-python-framework/src/main/python/framework/Controller.py @@ -15,6 +15,7 @@ import logging import os +import sys from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor from py4j.java_gateway import JavaGateway, CallbackServerParameters, GatewayParameters @@ -29,19 +30,14 @@ threadpool_attrs = dir(ThreadPoolExecutor) processpool_attrs = dir(ProcessPoolExecutor) - -# Initialize logging -logger = logging.getLogger("org.apache.nifi.py4j.Controller") -logger.setLevel(logging.INFO) - -logging.getLogger("py4j").setLevel(logging.WARN) - -logsDir = os.getenv('LOGS_DIR') -logging.basicConfig(filename=logsDir + '/nifi-python.log', - format='%(asctime)s %(levelname)s %(name)s %(message)s', +# Set log format with level number and separator as expected in PythonProcessReaderCommand +logging.basicConfig(stream=sys.stderr, + format='PY4JLOG %(levelno)s %(name)s:%(message)s', encoding='utf-8', level=logging.INFO) +logger = logging.getLogger("org.apache.nifi.py4j.Controller") + class Controller: @@ -92,6 +88,9 @@ def setGateway(self, gateway): def setControllerServiceTypeLookup(self, typeLookup): self.controllerServiceTypeLookup = typeLookup + def setLoggerLevel(self, loggerName, level): + logging.getLogger(loggerName).setLevel(level) + class Java: implements = ["org.apache.nifi.py4j.PythonController"] diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties index ce4aa9faf178..f9ef29179de2 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties @@ -48,7 +48,6 @@ nifi.python.extensions.source.directory.default=./python/extensions nifi.python.working.directory=./work/python nifi.python.max.processes=100 nifi.python.max.processes.per.extension.type=10 -nifi.python.logs.directory=./logs #################### # State Management # diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties index d507b92b22ac..8b45f32b1710 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties @@ -48,7 +48,6 @@ nifi.python.extensions.source.directory.default=./python/extensions nifi.python.working.directory=./work/python nifi.python.max.processes=100 nifi.python.max.processes.per.extension.type=10 -nifi.python.logs.directory=./logs #################### # State Management # diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties index 6ce80350c4d4..4758d714dca7 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties @@ -48,7 +48,6 @@ nifi.python.extensions.source.directory.default=./python/extensions nifi.python.working.directory=./work/python nifi.python.max.processes=100 nifi.python.max.processes.per.extension.type=10 -nifi.python.logs.directory=./logs #################### # State Management # diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml index 806958ddbfb1..a3047818dd57 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml @@ -115,6 +115,9 @@ + + + diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties index 0069027b3e5f..faf92c6a96ae 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties @@ -52,7 +52,6 @@ nifi.python.extensions.source.directory.default=./python/extensions nifi.python.working.directory=./work/python nifi.python.max.processes=100 nifi.python.max.processes.per.extension.type=10 -nifi.python.logs.directory=./logs #################### # State Management # From bda9b6360d932bb07af387e38a243cb7f3bceb2b Mon Sep 17 00:00:00 2001 From: Jim Steinebrey Date: Tue, 5 Mar 2024 10:52:26 -0500 Subject: [PATCH 18/73] NIFI-12630 Fix NPE getLogger in ConsumeSlack and PublishSlack Signed-off-by: Matt Burgess This closes #8474 --- .../nifi/processors/slack/ConsumeSlack.java | 16 +++++++++++--- .../nifi/processors/slack/PublishSlack.java | 7 +++++- .../processors/slack/TestConsumeSlack.java | 22 +++++++++++++++++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/ConsumeSlack.java b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/ConsumeSlack.java index 43c58245fd97..8b8c4539784e 100644 --- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/ConsumeSlack.java +++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/ConsumeSlack.java @@ -182,7 +182,7 @@ public class ConsumeSlack extends AbstractProcessor implements VerifiableProcess .build(); - private final RateLimit rateLimit = new RateLimit(getLogger()); + private RateLimit rateLimit; private final Queue channels = new LinkedBlockingQueue<>(); private volatile App slackApp; @@ -205,6 +205,7 @@ public boolean isStateful(final ProcessContext context) { @OnScheduled public void setup(final ProcessContext context) throws IOException, SlackApiException { + rateLimit = new RateLimit(getLogger()); slackApp = createSlackApp(context); final List consumeChannels = createChannels(context, slackApp); @@ -212,9 +213,18 @@ public void setup(final ProcessContext context) throws IOException, SlackApiExce } @OnStopped - public void onStopped() { + public void shutdown() { channels.clear(); - slackApp.stop(); + if (slackApp != null) { + slackApp.stop(); + slackApp = null; + } + rateLimit = null; + } + + + public RateLimit getRateLimit() { + return rateLimit; } diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/PublishSlack.java b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/PublishSlack.java index 012c12d1defe..4390d2a4766b 100644 --- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/PublishSlack.java +++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/main/java/org/apache/nifi/processors/slack/PublishSlack.java @@ -241,7 +241,7 @@ public class PublishSlack extends AbstractProcessor { REL_RATE_LIMITED, REL_FAILURE); - private final RateLimit rateLimit = new RateLimit(getLogger()); + private RateLimit rateLimit; private volatile ChannelMapper channelMapper; private volatile App slackApp; @@ -259,6 +259,7 @@ public Set getRelationships() { @OnScheduled public void setup(final ProcessContext context) { + rateLimit = new RateLimit(getLogger()); slackApp = createSlackApp(context); client = slackApp.client(); @@ -267,9 +268,13 @@ public void setup(final ProcessContext context) { @OnStopped public void shutdown() { + channelMapper = null; + client = null; if (slackApp != null) { slackApp.stop(); + slackApp = null; } + rateLimit = null; } private App createSlackApp(final ProcessContext context) { diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/TestConsumeSlack.java b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/TestConsumeSlack.java index a3c73ff81d6b..f791f4bd666b 100644 --- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/TestConsumeSlack.java +++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/TestConsumeSlack.java @@ -35,6 +35,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -55,14 +56,14 @@ public class TestConsumeSlack { private static final ObjectMapper objectMapper = new ObjectMapper(); private TestRunner testRunner; private MockConsumeSlackClient client; - + ConsumeSlack processor; @BeforeEach public void setup() { client = new MockConsumeSlackClient(); // Create an instance of the processor that mocks out the initialize() method to return a client we can use for testing - final ConsumeSlack processor = new ConsumeSlack() { + processor = new ConsumeSlack() { @Override protected ConsumeSlackClient initializeClient(final App slackApp) { return client; @@ -75,6 +76,23 @@ protected ConsumeSlackClient initializeClient(final App slackApp) { testRunner.setProperty(ConsumeSlack.BATCH_SIZE, "5"); } + @Test + public void testRequestRateLimited() { + testRunner.setProperty(ConsumeSlack.CHANNEL_IDS, "cid1,cid2"); + final Message message = createMessage("U12345", "Hello world", "1683903832.350"); + client.addHistoryResponse(noMore(createSuccessfulHistoryResponse(message))); + + testRunner.run(1, false, true); + testRunner.assertAllFlowFilesTransferred(ConsumeSlack.REL_SUCCESS, 1); + testRunner.clearTransferState(); + + // Create another HttpResponse because each response can only be read once. + client.addHistoryResponse(noMore(createSuccessfulHistoryResponse(message))); + // Set processor to be in rate limited state, therefore it will process 0 flowfiles + processor.getRateLimit().retryAfter(Duration.ofSeconds(30)); + testRunner.run(1, true, false); + testRunner.assertAllFlowFilesTransferred(ConsumeSlack.REL_SUCCESS, 0); + } @Test public void testSuccessfullyReceivedSingleMessage() throws JsonProcessingException { From c6cbdd91cdd3139908a76017c234c54608b0258d Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Wed, 6 Mar 2024 08:36:43 -0600 Subject: [PATCH 19/73] NIFI-12868 Upgraded Commons DBCP from 2.11.0 to 2.12.0 Signed-off-by: Pierre Villard This closes #8478. --- nifi-nar-bundles/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml index c28bcd98595c..3149499600b0 100755 --- a/nifi-nar-bundles/pom.xml +++ b/nifi-nar-bundles/pom.xml @@ -24,7 +24,7 @@ nifi-nar-bundles pom - 2.11.0 + 2.12.0 nifi-framework-bundle From ac6c9c6ca47a3ee752037a8bd4e1d1a4cc4db53b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:48:44 +0000 Subject: [PATCH 20/73] NIFI-12869 - Bump langchain from 0.1.2 to 0.1.11 Bumps [langchain](https://github.com/langchain-ai/langchain) from 0.1.2 to 0.1.11. - [Release notes](https://github.com/langchain-ai/langchain/releases) - [Commits](https://github.com/langchain-ai/langchain/compare/v0.1.2...v0.1.11) --- updated-dependencies: - dependency-name: langchain dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Pierre Villard This closes #8479. --- .../src/main/python/vectorstores/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/requirements.txt b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/requirements.txt index f3fea58948ce..ad78d29d03f4 100644 --- a/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/requirements.txt +++ b/nifi-python-extensions/nifi-text-embeddings-module/src/main/python/vectorstores/requirements.txt @@ -26,4 +26,4 @@ requests # Pinecone requirements pinecone-client==3.0.1 tiktoken -langchain==0.1.2 +langchain==0.1.11 From e2e54f2bc8349ca15a8f3b12f9477e26485a6870 Mon Sep 17 00:00:00 2001 From: James Mingardi-Elliott Date: Wed, 6 Mar 2024 17:11:07 -0500 Subject: [PATCH 21/73] NIFI-12721 Button UX (#8464) * NIFI-12721 Button UX Updated all the dialog buttons to change them from stroked and raised to basic. This better aligns with Angular Material guidelines and should address the confusion between disabled and cancelled. * NiFi-12721 incremental update Changed the combo editor and editor components to use the flat button styles. Missed them before because I was looking for dialogs. Also caught a style bleed with button:disabled that should be button.nifi-button:disabled and updated that. * Consolidated the button.nifi-button:disabled rules Also ran Prettier to improve markup in a few spots --- .../add-tenant-to-policy-dialog.component.html | 4 ++-- .../override-policy-dialog.component.html | 4 ++-- .../action-details/action-details.component.html | 2 +- .../purge-history/purge-history.component.html | 4 ++-- .../create-connection/create-connection.component.html | 4 ++-- .../edit-connection/edit-connection.component.html | 4 ++-- .../import-from-registry.component.html | 4 ++-- .../items/port/create-port/create-port.component.html | 4 ++-- .../items/port/edit-port/edit-port.component.html | 4 ++-- .../create-process-group.component.html | 4 ++-- .../edit-process-group.component.html | 4 ++-- .../group-components/group-components.component.html | 4 ++-- .../edit-processor/edit-processor.component.html | 4 ++-- .../create-remote-process-group.component.html | 4 ++-- .../edit-remote-process-group.component.html | 4 ++-- .../no-registry-clients-dialog.component.html | 6 +++--- .../edit-remote-port/edit-remote-port.component.html | 4 ++-- .../edit-parameter-context.component.html | 8 ++++---- .../provenance-search-dialog.component.html | 4 ++-- .../flowfile-dialog/flowfile-dialog.component.html | 2 +- .../edit-flow-analysis-rule.component.html | 4 ++-- .../edit-parameter-provider.component.html | 4 ++-- .../fetch-parameter-provider-parameters.component.html | 6 +++--- .../create-registry-client.component.html | 4 ++-- .../edit-registry-client.component.html | 4 ++-- .../edit-reporting-task.component.html | 4 ++-- .../cluster-summary-dialog.component.html | 2 +- .../user-access-policies.component.html | 2 +- .../common/cancel-dialog/cancel-dialog.component.html | 2 +- .../component-state/component-state.component.html | 2 +- .../disable-controller-service.component.html | 10 ++++------ .../edit-controller-service.component.html | 4 ++-- .../enable-controller-service.component.html | 10 ++++------ .../edit-parameter-dialog.component.html | 4 ++-- .../edit-tenant/edit-tenant-dialog.component.html | 4 ++-- .../extension-creation.component.html | 4 ++-- .../new-property-dialog.component.html | 4 ++-- .../app/ui/common/ok-dialog/ok-dialog.component.html | 2 +- .../editors/combo-editor/combo-editor.component.html | 9 ++------- .../editors/nf-editor/nf-editor.component.html | 9 ++------- .../provenance-event-dialog.component.html | 2 +- .../status-history/status-history.component.html | 2 +- .../system-diagnostics-dialog.component.html | 2 +- .../common/yes-no-dialog/yes-no-dialog.component.html | 4 ++-- .../src/main/nifi/src/assets/themes/nifi.scss | 2 +- .../nifi-web-frontend/src/main/nifi/src/styles.scss | 5 +---- .../dialogs/about/nf-registry-explorer-about.html | 2 +- 47 files changed, 89 insertions(+), 106 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/add-tenant-to-policy-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/add-tenant-to-policy-dialog.component.html index f2706f8e787e..cef0493219a1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/add-tenant-to-policy-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/add-tenant-to-policy-dialog.component.html @@ -48,9 +48,9 @@

Add Users/Groups To Policy

@if ({ value: (saving$ | async)! }; as saving) { - + - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/action-details/action-details.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/action-details/action-details.component.html index 64d58fa5a6d7..7105e8b6ec5a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/action-details/action-details.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/action-details/action-details.component.html @@ -206,7 +206,7 @@

Action Details

- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/purge-history/purge-history.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/purge-history/purge-history.component.html index 956a053ae961..b6f3d310e67b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/purge-history/purge-history.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/purge-history/purge-history.component.html @@ -50,12 +50,12 @@

Purge History

- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/create-connection/create-connection.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/create-connection/create-connection.component.html index 68b72393fba0..696b9e12e477 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/create-connection/create-connection.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/create-connection/create-connection.component.html @@ -175,13 +175,13 @@

Create Connection

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/edit-connection/edit-connection.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/edit-connection/edit-connection.component.html index 13aff2f8c26c..b210df57b2ab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/edit-connection/edit-connection.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/edit-connection/edit-connection.component.html @@ -176,13 +176,13 @@

Edit Connection

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/flow/import-from-registry/import-from-registry.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/flow/import-from-registry/import-from-registry.component.html index 54b92228065c..6774f8cc8652 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/flow/import-from-registry/import-from-registry.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/flow/import-from-registry/import-from-registry.component.html @@ -141,13 +141,13 @@

Import From Registry

- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/create-port/create-port.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/create-port/create-port.component.html index 6edeb7db5ba3..269a2a0176dc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/create-port/create-port.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/create-port/create-port.component.html @@ -47,13 +47,13 @@

Create New {{ portTypeLabel }}

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/edit-port/edit-port.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/edit-port/edit-port.component.html index 61d074a4f9ac..a12ecfe576b1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/edit-port/edit-port.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/port/edit-port/edit-port.component.html @@ -48,13 +48,13 @@

Edit {{ portTypeLabel }}

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/create-process-group.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/create-process-group.component.html index aab67925b12e..678dcd0807f2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/create-process-group.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/create-process-group.component.html @@ -72,13 +72,13 @@

Create Process Group

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/edit-process-group/edit-process-group.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/edit-process-group/edit-process-group.component.html index b390be4c0a06..d95b118c7c23 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/edit-process-group/edit-process-group.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/edit-process-group/edit-process-group.component.html @@ -149,14 +149,14 @@

Edit Process Group

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/group-components/group-components.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/group-components/group-components.component.html index 074bf198152b..b61cd5334b41 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/group-components/group-components.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/group-components/group-components.component.html @@ -44,13 +44,13 @@

Create Process Group

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/edit-processor.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/edit-processor.component.html index c06fcc1c6068..7211a8efc1fb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/edit-processor.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/edit-processor.component.html @@ -192,13 +192,13 @@

Edit Processor

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/create-remote-process-group.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/create-remote-process-group.component.html index 5e01e7e2c28b..5ee118cf6c25 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/create-remote-process-group.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/create-remote-process-group.component.html @@ -103,13 +103,13 @@

Create Remote Process Group

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/edit-remote-process-group/edit-remote-process-group.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/edit-remote-process-group/edit-remote-process-group.component.html index 97257304a1e2..1a1873ca091c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/edit-remote-process-group/edit-remote-process-group.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/edit-remote-process-group/edit-remote-process-group.component.html @@ -97,13 +97,13 @@

Edit Remote Process Group

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/no-registry-clients-dialog/no-registry-clients-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/no-registry-clients-dialog/no-registry-clients-dialog.component.html index 03ddd3fa20e8..92e84119b95f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/no-registry-clients-dialog/no-registry-clients-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/no-registry-clients-dialog/no-registry-clients-dialog.component.html @@ -27,11 +27,11 @@

No Registry clients available

@if (request.controllerPermissions.canRead) { - - + } @else { - + } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html index d2cf80e4d60a..d05944b7d13c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/edit-remote-port/edit-remote-port.component.html @@ -58,13 +58,13 @@

Edit Remote {{ portTypeLabel }}

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/edit-parameter-context/edit-parameter-context.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/edit-parameter-context/edit-parameter-context.component.html index 91296a9ea6c8..4942a4d7216a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/edit-parameter-context/edit-parameter-context.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/edit-parameter-context/edit-parameter-context.component.html @@ -134,21 +134,21 @@

{{ this.isNew ? 'Add' : 'Edit' }} Parameter Context

@if ((updateRequest | async)!; as requestEntity) { @if (requestEntity.request.complete) { - + } @else { - + } } @else { @if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-search-dialog/provenance-search-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-search-dialog/provenance-search-dialog.component.html index 9cdda8c4b4e3..7b1816c69f8a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-search-dialog/provenance-search-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-search-dialog/provenance-search-dialog.component.html @@ -77,13 +77,13 @@

Search Events

} - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/flowfile-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/flowfile-dialog.component.html index 91c109ea3ce1..7be268c5b292 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/flowfile-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/flowfile-dialog.component.html @@ -224,6 +224,6 @@

FlowFile

- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/edit-flow-analysis-rule/edit-flow-analysis-rule.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/edit-flow-analysis-rule/edit-flow-analysis-rule.component.html index 2947baef4200..7489ba4cb172 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/edit-flow-analysis-rule/edit-flow-analysis-rule.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/edit-flow-analysis-rule/edit-flow-analysis-rule.component.html @@ -83,13 +83,13 @@

Edit Flow Analysis Rule

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/edit-parameter-provider/edit-parameter-provider.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/edit-parameter-provider/edit-parameter-provider.component.html index e366766ab12f..d4c238ef1181 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/edit-parameter-provider/edit-parameter-provider.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/edit-parameter-provider/edit-parameter-provider.component.html @@ -80,13 +80,13 @@

Edit Parameter Provider

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html index 44c803772818..48814d22b761 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html @@ -322,18 +322,18 @@

Fetch Parameters

- - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/create-registry-client/create-registry-client.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/create-registry-client/create-registry-client.component.html index 3c3dc511dbbb..1d9a344233ac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/create-registry-client/create-registry-client.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/create-registry-client/create-registry-client.component.html @@ -56,13 +56,13 @@

Add Registry Client

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/edit-registry-client/edit-registry-client.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/edit-registry-client/edit-registry-client.component.html index 2e133e51e03c..fc9bb94db35e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/edit-registry-client/edit-registry-client.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/edit-registry-client/edit-registry-client.component.html @@ -61,13 +61,13 @@

Edit Registry Client

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/edit-reporting-task/edit-reporting-task.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/edit-reporting-task/edit-reporting-task.component.html index ef19c46f8053..aaf32d571a03 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/edit-reporting-task/edit-reporting-task.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/edit-reporting-task/edit-reporting-task.component.html @@ -101,13 +101,13 @@

Edit Reporting Task

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html index 45bd81acef15..d1ca40d32a65 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html @@ -79,7 +79,7 @@

Cluster {{ componentType }} Summary

{{ loadedTimestamp$ | async }}
- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html index 82446ff48405..fc28e7178c45 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html @@ -85,6 +85,6 @@

User Policies

- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/cancel-dialog/cancel-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/cancel-dialog/cancel-dialog.component.html index da55132180c4..0c0c5174d449 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/cancel-dialog/cancel-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/cancel-dialog/cancel-dialog.component.html @@ -20,5 +20,5 @@

{{ request.title }}

{{ request.message }}
- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-state/component-state.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-state/component-state.component.html index 2bf6fe63f6a0..ff41e3be2928 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-state/component-state.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-state/component-state.component.html @@ -87,6 +87,6 @@

Component State

- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/disable-controller-service/disable-controller-service.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/disable-controller-service/disable-controller-service.component.html index e7df41f11200..af84fcf9fcb1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/disable-controller-service/disable-controller-service.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/disable-controller-service/disable-controller-service.component.html @@ -45,8 +45,8 @@

Disable Controller Service

- - + + } @else { @@ -127,11 +127,9 @@

Disable Controller Service

 @if (disableRequest.currentStep === SetEnableStep.Completed || disableRequest.error) { - + } @else { - + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/edit-controller-service/edit-controller-service.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/edit-controller-service/edit-controller-service.component.html index 8cd22f8618b2..898c1367b991 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/edit-controller-service/edit-controller-service.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/edit-controller-service/edit-controller-service.component.html @@ -109,13 +109,13 @@

Edit Controller Service

@if ({ value: (saving$ | async)! }; as saving) { - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/enable-controller-service/enable-controller-service.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/enable-controller-service/enable-controller-service.component.html index c4f6eafe9e98..1abf1dbc3402 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/enable-controller-service/enable-controller-service.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/controller-service/enable-controller-service/enable-controller-service.component.html @@ -58,13 +58,13 @@

Enable Controller Service

- + @@ -148,11 +148,9 @@

Enable Controller Service

@if (enableRequest.currentStep === SetEnableStep.Completed || enableRequest.error) { - + } @else { - + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html index 526c61b9f38e..b99f2db7f145 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html @@ -54,9 +54,9 @@

{{ isNew ? 'Add' : 'Edit' }} Parameter

@if ({ value: (saving$ | async)! }; as saving) { - + + + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html index 4f9ce7eb4355..345c35c078cd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html @@ -36,9 +36,9 @@

Add Property

- + + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/combo-editor/combo-editor.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/combo-editor/combo-editor.component.html index effecfb06d82..2e9a55a00199 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/combo-editor/combo-editor.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/combo-editor/combo-editor.component.html @@ -94,12 +94,7 @@
-
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/nf-editor.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/nf-editor.component.html index b5ed8802f938..67fde7d3de04 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/nf-editor.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/nf-editor.component.html @@ -59,12 +59,7 @@ >
-
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/provenance-event-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/provenance-event-dialog.component.html index de9efe6b5e9e..ddb5fe2ea69f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/provenance-event-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/provenance-event-dialog.component.html @@ -442,6 +442,6 @@

Provenance Event

- + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html index a8595386a331..a5e980cb9b66 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html @@ -172,7 +172,7 @@

Status History

}
- +
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/system-diagnostics-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/system-diagnostics-dialog.component.html index da276fa5a123..4f16aec166e9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/system-diagnostics-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/system-diagnostics-dialog.component.html @@ -236,7 +236,7 @@

System Diagnostics

{{ loadedTimestamp$ | async }}
- +
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/yes-no-dialog/yes-no-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/yes-no-dialog/yes-no-dialog.component.html index e011fdae7871..39f92518a7d2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/yes-no-dialog/yes-no-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/yes-no-dialog/yes-no-dialog.component.html @@ -20,6 +20,6 @@

{{ request.title }}

{{ request.message }}
- - + + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss index b8ff67946d4c..34bf7d8bcc69 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss @@ -393,7 +393,7 @@ $warn-dark-palette: ( ); // Define the palettes for your theme -$material-primary-light: mat.define-palette($material-primary-light-palette); +$material-primary-light: mat.define-palette($material-primary-light-palette, 600, 100, 900); $material-accent-light: mat.define-palette($material-primary-light-palette, A400, A100, A700); $nifi-canvas-primary-light: mat.define-palette($nifi-canvas-light-palette); $nifi-canvas-accent-light: mat.define-palette($nifi-canvas-accent-light-palette, 400, 100, 700); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss index 145863cc7105..2032f471c31b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss @@ -317,12 +317,9 @@ $appFontPath: '~roboto-fontface/fonts'; border-color: $accent-palette-A400; } - button:disabled { + button.nifi-button:disabled { color: $primary-palette-200 !important; cursor: not-allowed; - } - - button.nifi-button:disabled { border: 1px solid $primary-palette-200; i { diff --git a/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/webapp/components/explorer/dialogs/about/nf-registry-explorer-about.html b/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/webapp/components/explorer/dialogs/about/nf-registry-explorer-about.html index 9880368bf6d9..8294f4cd57ca 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/webapp/components/explorer/dialogs/about/nf-registry-explorer-about.html +++ b/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/webapp/components/explorer/dialogs/about/nf-registry-explorer-about.html @@ -32,5 +32,5 @@

About Nifi Registry

- + From cc64c7adc6b262295b0cb10fbca2adf6488c4595 Mon Sep 17 00:00:00 2001 From: James Mingardi-Elliott Date: Wed, 6 Mar 2024 17:16:28 -0500 Subject: [PATCH 22/73] NIFI-12865 BUG - Checkboxes are inconsistently styled with primary or accent (#8472) Updated all the checkboxes to use color="primary" to keep them consistent. Revert "NIFI-12865 BUG - Checkboxes are inconsistently styled with primary or accent" This reverts commit 5833cb1a39837b59de2c664c74c48a579234e284. Removed Registry changes --- .../relationship-settings.component.html | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html index 3c5b90afc619..3f7ba9dff232 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html @@ -37,15 +37,16 @@ name="autoTerminate-{{ i }}" (change)="handleChanged()" [disabled]="isDisabled" - >terminate + >terminate + retry + >retry + @if (hasDescription(relationship)) {
{{ relationship.description }}
From 2a9c4c5de977c60c3988c5b1714e63164e2bd0ff Mon Sep 17 00:00:00 2001 From: James Mingardi-Elliott Date: Wed, 6 Mar 2024 18:12:27 -0500 Subject: [PATCH 23/73] NIFI-12720 BUG - Dark mode skeleton loader styles (#8465) * NIFI-12720 Set an explicit background color for the skeleton-loader element so there is sufficient contrast to show the animation in both light and dark themes. * NIFI-12720 Updates based on comments * Update nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss Co-authored-by: Scott Aslan --------- Co-authored-by: Scott Aslan --- .../nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss index 2032f471c31b..bd9dce75684c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss @@ -331,6 +331,10 @@ $appFontPath: '~roboto-fontface/fonts'; .refresh-timestamp { color: $warn-palette-A400; } + + ngx-skeleton-loader .skeleton-loader { + background: $canvas-primary-palette-400; + } } @mixin nifi-styles() { From bee65b8447303a49a5a244aed027ea387c96a2d8 Mon Sep 17 00:00:00 2001 From: Emilio Setiadarma Date: Mon, 12 Feb 2024 01:22:52 -0800 Subject: [PATCH 24/73] NIFI-12825: implemented ListHBaseRegions processor Signed-off-by: Matt Burgess This closes #8439 --- .../apache/nifi/hbase/ListHBaseRegions.java | 151 +++++++++++++ .../org.apache.nifi.processor.Processor | 1 + .../nifi/hbase/MockHBaseClientService.java | 15 ++ .../nifi/hbase/TestListHBaseRegions.java | 209 ++++++++++++++++++ .../nifi/hbase/HBaseClientException.java | 27 +++ .../apache/nifi/hbase/HBaseClientService.java | 8 + .../apache/nifi/hbase/scan/HBaseRegion.java | 57 +++++ .../nifi/hbase/HBase_2_ClientService.java | 35 +++ 8 files changed, 503 insertions(+) create mode 100644 nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/ListHBaseRegions.java create mode 100644 nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestListHBaseRegions.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientException.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/scan/HBaseRegion.java diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/ListHBaseRegions.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/ListHBaseRegions.java new file mode 100644 index 000000000000..032ec74f788d --- /dev/null +++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/ListHBaseRegions.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.hbase; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.hbase.scan.HBaseRegion; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Set; + +@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN) +@Tags({"hbase", "regions", "scan", "rowkey"}) +@CapabilityDescription("Returns the information about the regions of an HBase table, including ID, name and row key ranges. " + + "This information is helpful to feed into start row key and end row key for scans to HBase, e.g. using the ScanHBase processor.") +@WritesAttributes({ + @WritesAttribute(attribute = "hbase.region.name", description = "The name of the HBase region."), + @WritesAttribute(attribute = "hbase.region.id", description = "The id of the HBase region."), + @WritesAttribute(attribute = "hbase.region.startRowKey", description = "The starting row key (inclusive) of the HBase region. " + + "The bytes returned from HBase is converted into a UTF-8 encoded string."), + @WritesAttribute(attribute = "hbase.region.endRowKey", description = "The ending row key (exclusive) of the HBase region. " + + "The bytes returned from HBase is converted into a UTF-8 encoded string.") +}) +public class ListHBaseRegions extends AbstractProcessor { + static final String HBASE_REGION_NAME_ATTR = "hbase.region.name"; + static final String HBASE_REGION_ID_ATTR = "hbase.region.id"; + static final String HBASE_REGION_START_ROW_ATTR = "hbase.region.startRowKey"; + static final String HBASE_REGION_END_ROW_ATTR = "hbase.region.endRowKey"; + static final PropertyDescriptor HBASE_CLIENT_SERVICE = new PropertyDescriptor.Builder() + .name("HBase Client Service") + .description("Specifies the Controller Service to use for accessing HBase.") + .required(true) + .identifiesControllerService(HBaseClientService.class) + .build(); + + static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder() + .name("Table Name") + .description("The name of the HBase Table to put data into") + .required(true) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + static final PropertyDescriptor ROUTE_DEGENERATE_REGIONS = new PropertyDescriptor.Builder() + .name("Route Degenerate Regions") + .required(false) + .defaultValue("false") + .allowableValues("true", "false") + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) + .build(); + + static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("FlowFiles with information on regions of the HBase table are routed to this relationship.") + .build(); + + static final Relationship REL_DEGENERATE = new Relationship.Builder() + .name("degenerate") + .description("If \\\"Route Degenerate Regions\\\" is set, any " + + "FlowFile(s) that contains information about a region that is degenerate will be routed " + + "to this relationship. Otherwise, they will be sent to the success relationship.") + .autoTerminateDefault(true) + .build(); + + @Override + public Set getRelationships() { + return Set.of(REL_SUCCESS, REL_DEGENERATE); + } + + @Override + protected List getSupportedPropertyDescriptors() { + return List.of( + HBASE_CLIENT_SERVICE, + TABLE_NAME, + ROUTE_DEGENERATE_REGIONS + ); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions().getValue(); + if (StringUtils.isBlank(tableName)) { + getLogger().error("Table Name is blank or null, no regions information to be fetched."); + context.yield(); + return; + } + + final HBaseClientService hBaseClientService = context.getProperty(HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class); + + final boolean routeDegenerateRegions = context.getProperty(ROUTE_DEGENERATE_REGIONS).asBoolean(); + + try { + final List hBaseRegions = hBaseClientService.listHBaseRegions(tableName); + for (final HBaseRegion region : hBaseRegions) { + final FlowFile flowFile = session.create(); + session.putAttribute(flowFile, HBASE_REGION_NAME_ATTR, region.getRegionName()); + session.putAttribute(flowFile, HBASE_REGION_ID_ATTR, String.valueOf(region.getRegionId())); + if (region.getStartRowKey() == null) { + session.putAttribute(flowFile, HBASE_REGION_START_ROW_ATTR, ""); + } else { + session.putAttribute(flowFile, HBASE_REGION_START_ROW_ATTR, new String(region.getStartRowKey(), StandardCharsets.UTF_8)); + } + + if (region.getEndRowKey() == null) { + session.putAttribute(flowFile, HBASE_REGION_END_ROW_ATTR, ""); + } else { + session.putAttribute(flowFile, HBASE_REGION_END_ROW_ATTR, new String(region.getEndRowKey(), StandardCharsets.UTF_8)); + } + + if (region.isDegenerate() && routeDegenerateRegions) { + getLogger().warn("Region with id {} and name {} is degenerate. Routing to degenerate relationship.", region.getRegionId(), region.getRegionName()); + session.transfer(flowFile, REL_DEGENERATE); + } else { + session.transfer(flowFile, REL_SUCCESS); + } + } + } catch (final HBaseClientException e) { + getLogger().error("Failed to receive information on HBase regions for table {} due to {}", tableName, e); + context.yield(); + throw new RuntimeException(e); + } + } +} diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor index a55064ba52ae..c0ea17418ed4 100644 --- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -16,6 +16,7 @@ org.apache.nifi.hbase.DeleteHBaseCells org.apache.nifi.hbase.DeleteHBaseRow org.apache.nifi.hbase.GetHBase +org.apache.nifi.hbase.ListHBaseRegions org.apache.nifi.hbase.PutHBaseCell org.apache.nifi.hbase.PutHBaseJSON org.apache.nifi.hbase.PutHBaseRecord diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java index b913edf440b1..e51de5a6b388 100644 --- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java +++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/MockHBaseClientService.java @@ -20,6 +20,7 @@ import org.apache.nifi.hbase.put.PutColumn; import org.apache.nifi.hbase.put.PutFlowFile; import org.apache.nifi.hbase.scan.Column; +import org.apache.nifi.hbase.scan.HBaseRegion; import org.apache.nifi.hbase.scan.ResultCell; import org.apache.nifi.hbase.scan.ResultHandler; @@ -42,6 +43,7 @@ public class MockHBaseClientService extends AbstractControllerService implements private int numScans = 0; private int numPuts = 0; private int linesBeforeException = -1; + private List regionsToReturn = new ArrayList<>(); @Override public void put(String tableName, Collection puts) throws IOException { @@ -244,6 +246,19 @@ public int getNumScans() { return numScans; } + @Override + public List listHBaseRegions(final String tableName) throws HBaseClientException { + return regionsToReturn; + } + + public void addHBaseRegion(final HBaseRegion region) { + regionsToReturn.add(region); + } + + public void addHBaseRegions(final List regions) { + regionsToReturn.addAll(regions); + } + @Override public byte[] toBytes(final boolean b) { return new byte[] { b ? (byte) -1 : (byte) 0 }; diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestListHBaseRegions.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestListHBaseRegions.java new file mode 100644 index 000000000000..1618041ead2e --- /dev/null +++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/test/java/org/apache/nifi/hbase/TestListHBaseRegions.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.hbase; + +import org.apache.nifi.hbase.scan.HBaseRegion; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@ExtendWith(MockitoExtension.class) +public class TestListHBaseRegions { + private static final String TABLE_NAME = "nifi"; + private static final String HBASE_CLIENT_SERVICE_NAME = "hBaseClientService"; + + private TestRunner runner; + private ListHBaseRegions proc; + + private MockHBaseClientService hBaseClientService; + + @BeforeEach + public void setup() throws InitializationException { + proc = new ListHBaseRegions(); + runner = TestRunners.newTestRunner(proc); + + hBaseClientService = new MockHBaseClientService(); + runner.addControllerService(HBASE_CLIENT_SERVICE_NAME, hBaseClientService); + runner.enableControllerService(hBaseClientService); + + runner.setProperty(ListHBaseRegions.TABLE_NAME, TABLE_NAME); + runner.setProperty(ListHBaseRegions.HBASE_CLIENT_SERVICE, HBASE_CLIENT_SERVICE_NAME); + } + + @Test + public void testAllFlowFilesToSuccess() throws HBaseClientException { + runner.setProperty(ListHBaseRegions.ROUTE_DEGENERATE_REGIONS, "false"); + runner.assertValid(); + + final String startRowKey1 = "1"; + final String endRowKey1 = "5"; + final String regionName1 = "region-1"; + final long regionId1 = 1L; + final boolean isDegenerate1 = false; + final HBaseRegion hBaseRegion1 = new HBaseRegion( + startRowKey1.getBytes(StandardCharsets.UTF_8), + endRowKey1.getBytes(StandardCharsets.UTF_8), + regionName1, + regionId1, + isDegenerate1 + ); + + // this is a "degenerate" region where startRowKey > endRowKey + final String startRowKey2 = "10"; + final String endRowKey2 = "6"; + final String regionName2 = "region-2"; + final long regionId2 = 2L; + final boolean isDegenerate2 = true; + final HBaseRegion hBaseRegion2 = new HBaseRegion( + startRowKey2.getBytes(StandardCharsets.UTF_8), + endRowKey2.getBytes(StandardCharsets.UTF_8), + regionName2, + regionId2, + isDegenerate2 + ); + + final List regions = Arrays.asList(hBaseRegion1, hBaseRegion2); + hBaseClientService.addHBaseRegions(regions); + + runner.run(1); + runner.assertAllFlowFilesTransferred(ListHBaseRegions.REL_SUCCESS, 2); + final List flowFiles = runner.getFlowFilesForRelationship(ListHBaseRegions.REL_SUCCESS); + + assertEquals(String.valueOf(regionId1), flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_ID_ATTR)); + assertEquals(regionName1, flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_NAME_ATTR)); + assertEquals(startRowKey1, flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_START_ROW_ATTR)); + assertEquals(endRowKey1, flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_END_ROW_ATTR)); + + assertEquals(String.valueOf(regionId2), flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_ID_ATTR)); + assertEquals(regionName2, flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_NAME_ATTR)); + assertEquals(startRowKey2, flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_START_ROW_ATTR)); + assertEquals(endRowKey2, flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_END_ROW_ATTR)); + } + + @Test + public void testDegenerateRegionsToDegenerateRelationship() throws HBaseClientException { + runner.setProperty(ListHBaseRegions.ROUTE_DEGENERATE_REGIONS, "true"); + runner.assertValid(); + + final String startRowKey1 = "1"; + final String endRowKey1 = "5"; + final String regionName1 = "region-1"; + final long regionId1 = 1L; + final boolean isDegenerate1 = false; + final HBaseRegion hBaseRegion1 = new HBaseRegion( + startRowKey1.getBytes(StandardCharsets.UTF_8), + endRowKey1.getBytes(StandardCharsets.UTF_8), + regionName1, + regionId1, + isDegenerate1 + ); + + // this is a "degenerate" region where startRowKey > endRowKey + final String startRowKey2 = "10"; + final String endRowKey2 = "6"; + final String regionName2 = "region-2"; + final long regionId2 = 2L; + final boolean isDegenerate2 = true; + final HBaseRegion hBaseRegion2 = new HBaseRegion( + startRowKey2.getBytes(StandardCharsets.UTF_8), + endRowKey2.getBytes(StandardCharsets.UTF_8), + regionName2, + regionId2, + isDegenerate2 + ); + + final List regions = Arrays.asList(hBaseRegion1, hBaseRegion2); + hBaseClientService.addHBaseRegions(regions); + + runner.run(1); + runner.assertTransferCount(ListHBaseRegions.REL_SUCCESS, 1); + final List successFlowFiles = runner.getFlowFilesForRelationship(ListHBaseRegions.REL_SUCCESS); + + assertEquals(String.valueOf(regionId1), successFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_ID_ATTR)); + assertEquals(regionName1, successFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_NAME_ATTR)); + assertEquals(startRowKey1, successFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_START_ROW_ATTR)); + assertEquals(endRowKey1, successFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_END_ROW_ATTR)); + + runner.assertTransferCount(ListHBaseRegions.REL_DEGENERATE, 1); + final List degenerateFlowFiles = runner.getFlowFilesForRelationship(ListHBaseRegions.REL_DEGENERATE); + + assertEquals(String.valueOf(regionId2), degenerateFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_ID_ATTR)); + assertEquals(regionName2, degenerateFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_NAME_ATTR)); + assertEquals(startRowKey2, degenerateFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_START_ROW_ATTR)); + assertEquals(endRowKey2, degenerateFlowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_END_ROW_ATTR)); + } + + @Test + public void testShouldNotRouteToDegenerateIfNoDegenerateRegions() throws HBaseClientException { + runner.setProperty(ListHBaseRegions.ROUTE_DEGENERATE_REGIONS, "false"); + runner.assertValid(); + + final String startRowKey1 = "1"; + final String endRowKey1 = "5"; + final String regionName1 = "region-1"; + final long regionId1 = 1L; + final boolean isDegenerate1 = false; + final HBaseRegion hBaseRegion1 = new HBaseRegion( + startRowKey1.getBytes(StandardCharsets.UTF_8), + endRowKey1.getBytes(StandardCharsets.UTF_8), + regionName1, + regionId1, + isDegenerate1 + ); + + final String startRowKey2 = "5"; + final String endRowKey2 = "10"; + final String regionName2 = "region-2"; + final long regionId2 = 2L; + final boolean isDegenerate2 = false; + final HBaseRegion hBaseRegion2 = new HBaseRegion( + startRowKey2.getBytes(StandardCharsets.UTF_8), + endRowKey2.getBytes(StandardCharsets.UTF_8), + regionName2, + regionId2, + isDegenerate2 + ); + + final List regions = Arrays.asList(hBaseRegion1, hBaseRegion2); + hBaseClientService.addHBaseRegions(regions); + + runner.run(1); + runner.assertAllFlowFilesTransferred(ListHBaseRegions.REL_SUCCESS, 2); + final List flowFiles = runner.getFlowFilesForRelationship(ListHBaseRegions.REL_SUCCESS); + + assertEquals(String.valueOf(regionId1), flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_ID_ATTR)); + assertEquals(regionName1, flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_NAME_ATTR)); + assertEquals(startRowKey1, flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_START_ROW_ATTR)); + assertEquals(endRowKey1, flowFiles.get(0).getAttribute(ListHBaseRegions.HBASE_REGION_END_ROW_ATTR)); + + assertEquals(String.valueOf(regionId2), flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_ID_ATTR)); + assertEquals(regionName2, flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_NAME_ATTR)); + assertEquals(startRowKey2, flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_START_ROW_ATTR)); + assertEquals(endRowKey2, flowFiles.get(1).getAttribute(ListHBaseRegions.HBASE_REGION_END_ROW_ATTR)); + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientException.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientException.java new file mode 100644 index 000000000000..4208d17299ba --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.hbase; + +public class HBaseClientException extends Exception { + public HBaseClientException(final String message) { + super(message); + } + + public HBaseClientException(final Throwable cause) { + super(cause); + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java index cc7709056e09..87972fdc36f1 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/HBaseClientService.java @@ -22,6 +22,7 @@ import org.apache.nifi.hbase.put.PutColumn; import org.apache.nifi.hbase.put.PutFlowFile; import org.apache.nifi.hbase.scan.Column; +import org.apache.nifi.hbase.scan.HBaseRegion; import org.apache.nifi.hbase.scan.ResultHandler; import java.io.IOException; @@ -169,6 +170,13 @@ public interface HBaseClientService extends ControllerService { void scan(String tableName, String startRow, String endRow, String filterExpression, Long timerangeMin, Long timerangeMax, Integer limitRows, Boolean isReversed, Boolean blockCache, Collection columns, List authorizations, ResultHandler handler) throws IOException; + /** + * Returns a {@link List} of {@link HBaseRegion} objects that represent information about the HBase table + * regions for all regions in the HBase table. + * @param tableName the name of the HBase table to fetch region information for + */ + List listHBaseRegions(String tableName) throws HBaseClientException; + /** * Converts the given boolean to it's byte representation. * diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/scan/HBaseRegion.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/scan/HBaseRegion.java new file mode 100644 index 000000000000..df23c07ed2c4 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase-client-service-api/src/main/java/org/apache/nifi/hbase/scan/HBaseRegion.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.hbase.scan; + +public class HBaseRegion { + private byte[] startRowKey; + private byte[] endRowKey; + private String regionName; + private long regionId; + private boolean isDegenerate; + + public HBaseRegion(final byte[] startRowKey, + final byte[] endRowKey, + final String regionName, + final long regionId, + final boolean isDegenerate) { + this.startRowKey = startRowKey; + this.endRowKey = endRowKey; + this.regionName = regionName; + this.regionId = regionId; + this.isDegenerate = isDegenerate; + } + + public byte[] getStartRowKey() { + return startRowKey; + } + + public byte[] getEndRowKey() { + return endRowKey; + } + + public String getRegionName() { + return regionName; + } + + public long getRegionId() { + return regionId; + } + + public boolean isDegenerate() { + return isDegenerate; + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/nifi-hbase_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_2_ClientService.java b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/nifi-hbase_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_2_ClientService.java index 627ba83b161f..80894e73bb1e 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/nifi-hbase_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_2_ClientService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/nifi-hbase_2-client-service/src/main/java/org/apache/nifi/hbase/HBase_2_ClientService.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -42,6 +44,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -76,6 +79,7 @@ import org.apache.nifi.hbase.put.PutColumn; import org.apache.nifi.hbase.put.PutFlowFile; import org.apache.nifi.hbase.scan.Column; +import org.apache.nifi.hbase.scan.HBaseRegion; import org.apache.nifi.hbase.scan.ResultCell; import org.apache.nifi.hbase.scan.ResultHandler; import org.apache.nifi.kerberos.KerberosCredentialsService; @@ -864,6 +868,37 @@ private ResultCell getResultCell(Cell cell) { return resultCell; } + @Override + public List listHBaseRegions(final String tableName) throws HBaseClientException { + if (connection == null || connection.isClosed() || connection.isAborted()) { + final String errorMsg = String.format( + "Unable to fetch regions for table %s since there is no active connection to HBase.", + tableName + ); + throw new IllegalStateException(errorMsg); + } + + try { + final List regionInfos = connection.getAdmin().getRegions(TableName.valueOf(tableName)); + // maps to the NiFi HBaseRegion object + final List regions = regionInfos.stream() + .map(regionInfo -> + new HBaseRegion( + regionInfo.getStartKey(), + regionInfo.getEndKey(), + regionInfo.getRegionNameAsString(), + regionInfo.getRegionId(), + regionInfo.isDegenerate() + ) + ) + .collect(Collectors.toList()); + return regions; + } catch (final IOException e) { + logger.error("Encountered error while communicating with HBase.", e); + throw new HBaseClientException(e); + } + } + static protected class ValidationResources { private final String configResources; private final Configuration configuration; From 04d213a00c9b2072d0493578e542d4d042724c23 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Thu, 7 Mar 2024 08:25:25 -0600 Subject: [PATCH 25/73] NIFI-12872 Upgraded Groovy from 4.0.18 to 4.0.19 Signed-off-by: Pierre Villard This closes #8481. --- .../nifi-graph-bundle/nifi-other-graph-services/pom.xml | 4 ++-- .../nifi-groovyx-bundle/nifi-groovyx-processors/pom.xml | 6 +++--- nifi-nar-bundles/nifi-groovyx-bundle/pom.xml | 2 +- .../nifi-scripting-processors/pom.xml | 8 ++++---- nifi-nar-bundles/nifi-scripting-bundle/pom.xml | 2 +- pom.xml | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nifi-nar-bundles/nifi-graph-bundle/nifi-other-graph-services/pom.xml b/nifi-nar-bundles/nifi-graph-bundle/nifi-other-graph-services/pom.xml index 224ef242deab..f3cc740d22e4 100644 --- a/nifi-nar-bundles/nifi-graph-bundle/nifi-other-graph-services/pom.xml +++ b/nifi-nar-bundles/nifi-graph-bundle/nifi-other-graph-services/pom.xml @@ -94,12 +94,12 @@ org.apache.groovy groovy - ${nifi.groovy.version} + ${groovy.version} org.apache.groovy groovy-dateutil - ${nifi.groovy.version} + ${groovy.version} commons-codec diff --git a/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/pom.xml b/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/pom.xml index 77dec1f8c3cc..78c4f1b186fc 100644 --- a/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/pom.xml +++ b/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/pom.xml @@ -43,20 +43,20 @@ org.apache.groovy groovy-json - ${nifi.groovy.version} + ${groovy.version} provided org.apache.groovy groovy-sql - ${nifi.groovy.version} + ${groovy.version} provided org.apache.groovy groovy-dateutil - ${nifi.groovy.version} + ${groovy.version} org.apache.ivy diff --git a/nifi-nar-bundles/nifi-groovyx-bundle/pom.xml b/nifi-nar-bundles/nifi-groovyx-bundle/pom.xml index d24cc02f6b8f..5ba3a0037558 100644 --- a/nifi-nar-bundles/nifi-groovyx-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-groovyx-bundle/pom.xml @@ -44,7 +44,7 @@ org.apache.groovy groovy-all - ${nifi.groovy.version} + ${groovy.version} pom provided diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml index 10f4ebd58adc..3d395c3cb13f 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml @@ -102,26 +102,26 @@ org.apache.groovy groovy-json - ${nifi.groovy.version} + ${groovy.version} provided org.apache.groovy groovy-jsr223 - ${nifi.groovy.version} + ${groovy.version} provided org.apache.groovy groovy-xml - ${nifi.groovy.version} + ${groovy.version} provided org.apache.groovy groovy-dateutil - ${nifi.groovy.version} + ${groovy.version} diff --git a/nifi-nar-bundles/nifi-scripting-bundle/pom.xml b/nifi-nar-bundles/nifi-scripting-bundle/pom.xml index 9976d0e29bf3..8f5e4f48b892 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-scripting-bundle/pom.xml @@ -73,7 +73,7 @@ org.apache.groovy groovy-all - ${nifi.groovy.version} + ${groovy.version} pom provided diff --git a/pom.xml b/pom.xml index 6f0372489e8b..7e764eb76a68 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ 4.0.1 3.1.0 2.5.0 - 4.0.18 + 4.0.19 3.1.2 3.3.6 1.2.1 @@ -339,7 +339,7 @@ org.apache.groovy groovy-all - ${nifi.groovy.version} + ${groovy.version} pom From 7ddee361e7a673740c5029d0677288ff5407b183 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Thu, 7 Mar 2024 10:57:37 -0600 Subject: [PATCH 26/73] NIFI-12874 Upgraded Log4j from 2.20.0 to 2.23.0 Signed-off-by: Pierre Villard This closes #8482. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7e764eb76a68..6d1f605c54b9 100644 --- a/pom.xml +++ b/pom.xml @@ -144,7 +144,7 @@ 2.1.5 1.9.21 3.1.4 - 2.20.0 + 2.23.0 1.4.14 5.8.0 3.10.6.Final From df524b18b1372193e3021c1ae04820895d66a8af Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Thu, 7 Mar 2024 12:13:26 -0600 Subject: [PATCH 27/73] NIFI-12876 Upgraded Surefire Plugin from 3.1.2 to 3.2.5 Signed-off-by: Pierre Villard This closes #8483. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6d1f605c54b9..711916f20253 100644 --- a/pom.xml +++ b/pom.xml @@ -138,7 +138,7 @@ 3.1.0 2.5.0 4.0.19 - 3.1.2 + 3.2.5 3.3.6 1.2.1 2.1.5 From 942d13c118be59cb360f7e334c87c0897da8e3fb Mon Sep 17 00:00:00 2001 From: dan-s1 Date: Wed, 28 Feb 2024 19:39:05 +0000 Subject: [PATCH 28/73] NIFI-12785 Corrected InvokeHTTP URL handling to avoid double encoding This closes #8458 Signed-off-by: David Handermann --- .../nifi/processors/standard/InvokeHTTP.java | 23 ++++++------- .../processors/standard/InvokeHTTPTest.java | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java index 7822d4d543d8..15a2740ed444 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java @@ -203,7 +203,8 @@ public class InvokeHTTP extends AbstractProcessor { public static final PropertyDescriptor HTTP_URL = new PropertyDescriptor.Builder() .name("HTTP URL") - .description("HTTP remote URL including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.") + .description("HTTP remote URL including a scheme of http or https, as well as a hostname or IP address with optional port and path elements." + + " Any encoding of the URL must be done by the user.") .required(true) .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .addValidator(StandardValidators.URL_VALIDATOR) @@ -852,19 +853,18 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro FlowFile responseFlowFile = null; try { final String urlProperty = trimToEmpty(context.getProperty(HTTP_URL).evaluateAttributeExpressions(requestFlowFile).getValue()); - final URL url = URLValidator.createURL(urlProperty); - Request httpRequest = configureRequest(context, session, requestFlowFile, url); + Request httpRequest = configureRequest(context, session, requestFlowFile, urlProperty); logRequest(logger, httpRequest); if (httpRequest.body() != null) { - session.getProvenanceReporter().send(requestFlowFile, url.toExternalForm(), true); + session.getProvenanceReporter().send(requestFlowFile, urlProperty, true); } final long startNanos = System.nanoTime(); try (Response responseHttp = okHttpClient.newCall(httpRequest).execute()) { - logResponse(logger, url, responseHttp); + logResponse(logger, urlProperty, responseHttp); // store the status code and message int statusCode = responseHttp.code(); @@ -874,7 +874,7 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro Map statusAttributes = new HashMap<>(); statusAttributes.put(STATUS_CODE, String.valueOf(statusCode)); statusAttributes.put(STATUS_MESSAGE, statusMessage); - statusAttributes.put(REQUEST_URL, url.toExternalForm()); + statusAttributes.put(REQUEST_URL, urlProperty); statusAttributes.put(REQUEST_DURATION, Long.toString(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos))); statusAttributes.put(RESPONSE_URL, responseHttp.request().url().toString()); statusAttributes.put(TRANSACTION_ID, txId.toString()); @@ -927,6 +927,7 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro // update FlowFile's filename attribute with an extracted value from the remote URL if (FlowFileNamingStrategy.URL_PATH.equals(getFlowFileNamingStrategy(context)) && HttpMethod.GET.name().equals(httpRequest.method())) { + final URL url = URLValidator.createURL(urlProperty); String fileName = getFileNameFromUrl(url); if (fileName != null) { responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.FILENAME.key(), fileName); @@ -950,9 +951,9 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro // emit provenance event final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); if (requestFlowFile != null) { - session.getProvenanceReporter().fetch(responseFlowFile, url.toExternalForm(), millis); + session.getProvenanceReporter().fetch(responseFlowFile, urlProperty, millis); } else { - session.getProvenanceReporter().receive(responseFlowFile, url.toExternalForm(), millis); + session.getProvenanceReporter().receive(responseFlowFile, urlProperty, millis); } } } @@ -1012,7 +1013,7 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro } } - private Request configureRequest(final ProcessContext context, final ProcessSession session, final FlowFile requestFlowFile, URL url) { + private Request configureRequest(final ProcessContext context, final ProcessSession session, final FlowFile requestFlowFile, String url) { final Request.Builder requestBuilder = new Request.Builder(); requestBuilder.url(url); @@ -1231,10 +1232,10 @@ private void logRequest(ComponentLog logger, Request request) { } } - private void logResponse(ComponentLog logger, URL url, Response response) { + private void logResponse(ComponentLog logger, String url, Response response) { if (logger.isDebugEnabled()) { logger.debug("\nResponse from remote service:\n\t{}\n{}", - url.toExternalForm(), getLogString(response.headers().toMultimap())); + url, getLogString(response.headers().toMultimap())); } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/InvokeHTTPTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/InvokeHTTPTest.java index 532b853c14ac..6f69ce257fb7 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/InvokeHTTPTest.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/InvokeHTTPTest.java @@ -25,11 +25,14 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.oauth2.OAuth2AccessTokenProvider; import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.URLValidator; import org.apache.nifi.processors.standard.http.ContentEncodingStrategy; import org.apache.nifi.processors.standard.http.FlowFileNamingStrategy; import org.apache.nifi.processors.standard.http.CookieStrategy; import org.apache.nifi.processors.standard.http.HttpHeader; import org.apache.nifi.processors.standard.http.HttpMethod; +import org.apache.nifi.provenance.ProvenanceEventRecord; +import org.apache.nifi.provenance.ProvenanceEventType; import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.proxy.ProxyConfigurationService; import org.apache.nifi.reporting.InitializationException; @@ -79,6 +82,7 @@ import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -548,6 +552,35 @@ public void testRunGetHttp200SuccessUserAgentConfigured() throws InterruptedExce assertEquals(userAgent, userAgentHeader); } + @Test + void testRunGetHttp200SuccessWithEncodableUrl() throws Exception { + final String partialEncodableUrl = "/gitlab/ftp%2Fstage%2F15m%2FsomeFile.yaml/raw?ref=main"; + final String nonEncodedUrl = mockWebServer.url(partialEncodableUrl).newBuilder().host(LOCALHOST).build().toString(); + final String encodedUrl = URLValidator.createURL(nonEncodedUrl).toExternalForm(); + + runner.setProperty(InvokeHTTP.HTTP_URL, nonEncodedUrl); + mockWebServer.enqueue(new MockResponse().setResponseCode(HTTP_OK)); + runner.enqueue(FLOW_FILE_CONTENT); + runner.run(); + + assertResponseSuccessRelationships(); + assertRelationshipStatusCodeEquals(InvokeHTTP.RESPONSE, HTTP_OK); + + MockFlowFile flowFile = getResponseFlowFile(); + final String actualUrl = flowFile.getAttribute(InvokeHTTP.REQUEST_URL); + assertNotEquals(encodedUrl, actualUrl); + assertTrue(actualUrl.endsWith(partialEncodableUrl)); + + final ProvenanceEventRecord event = runner.getProvenanceEvents().stream() + .filter(record -> record.getEventType() == ProvenanceEventType.FETCH) + .findFirst() + .orElse(null); + assertNotNull(event); + final String transitUri = event.getTransitUri(); + assertNotEquals(encodedUrl, transitUri); + assertTrue(transitUri.endsWith(partialEncodableUrl)); + } + @Test public void testRunGetHttp302NoRetryResponseRedirectsDefaultEnabled() { mockWebServer.enqueue(new MockResponse().setResponseCode(HTTP_MOVED_TEMP).setHeader(LOCATION_HEADER, getMockWebServerUrl())); From 8ad3c731dabdf9f951b949ce733e81cdebc06749 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Tue, 13 Feb 2024 08:31:29 -0500 Subject: [PATCH 29/73] NIFI-12797 Refactored Record.incorporateInactiveFields Refactored Record.incorporateInactiveFields to handle when an updated field and an inactive field have the same name (which can happen if incorporateInactiveFields is called multiple times). Also refactored the setValue(String, Object) method to call setValue(RecordField, Object) because the logic had diverged. Also exposed the text of Expression Language, which led to the discovery of this bug. This closes #8413 Signed-off-by: David Handermann --- .../language/CompiledExpression.java | 1 + .../language/EmptyPreparedQuery.java | 6 +++ .../expression/language/Expression.java | 5 ++ .../language/InvalidPreparedQuery.java | 6 +++ .../language/ParameterExpression.java | 5 ++ .../expression/language/PreparedQuery.java | 6 +++ .../language/StandardPreparedQuery.java | 5 ++ .../language/StringLiteralExpression.java | 5 ++ nifi-commons/nifi-record-path/pom.xml | 4 -- .../nifi/record/path/functions/Format.java | 2 +- .../nifi/record/path/functions/PadLeft.java | 2 +- .../nifi/record/path/functions/PadRight.java | 2 +- .../nifi/record/path/functions/ToDate.java | 2 +- .../nifi/serialization/record/MapRecord.java | 52 ++++++++++++------- .../record/util/DataTypeUtils.java | 7 +-- .../serialization/record/TestMapRecord.java | 50 ++++++++++++++++++ 16 files changed, 129 insertions(+), 31 deletions(-) diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/CompiledExpression.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/CompiledExpression.java index 74741b157ffb..5ac4f559c486 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/CompiledExpression.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/CompiledExpression.java @@ -44,6 +44,7 @@ public Tree getTree() { return tree; } + @Override public String getExpression() { return expression; } diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java index b33fcf4c43a2..e00d1719125b 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java @@ -20,6 +20,7 @@ import org.apache.nifi.processor.exception.ProcessException; import java.util.Collections; +import java.util.List; import java.util.Set; public class EmptyPreparedQuery implements PreparedQuery { @@ -49,4 +50,9 @@ public VariableImpact getVariableImpact() { public Set getExplicitlyReferencedAttributes() { return Collections.emptySet(); } + + @Override + public List getExpressions() { + return List.of(); + } } diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Expression.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Expression.java index 188c503ced58..3196a75f7d26 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Expression.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Expression.java @@ -28,4 +28,9 @@ public interface Expression { * @return the evaluated value */ String evaluate(EvaluationContext evaluationContext, AttributeValueDecorator decorator); + + /** + * @return the expression as a String + */ + String getExpression(); } diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java index 953e9ff10053..9670e6769e45 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java @@ -22,6 +22,7 @@ import org.apache.nifi.processor.exception.ProcessException; import java.util.Collections; +import java.util.List; import java.util.Set; /** @@ -59,4 +60,9 @@ public VariableImpact getVariableImpact() { public Set getExplicitlyReferencedAttributes() { return Collections.emptySet(); } + + @Override + public List getExpressions() { + return List.of(); + } } diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/ParameterExpression.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/ParameterExpression.java index fe8595f574f1..660c3b3772fb 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/ParameterExpression.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/ParameterExpression.java @@ -41,4 +41,9 @@ public String evaluate(final EvaluationContext evaluationContext, final Attribut return parameter.getValue(); } + + @Override + public String getExpression() { + return "#{" + parameterName + "}"; + } } diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java index f7a73e33c10c..004d1981292e 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java @@ -19,6 +19,7 @@ import org.apache.nifi.expression.AttributeValueDecorator; import org.apache.nifi.processor.exception.ProcessException; +import java.util.List; import java.util.Set; public interface PreparedQuery { @@ -45,4 +46,9 @@ public interface PreparedQuery { * @return a Set of all attributes that are explicitly referenced by the Prepared Query */ Set getExplicitlyReferencedAttributes(); + + /** + * @return the list of all Expressions that are used to make up the Prepared Query + */ + List getExpressions(); } diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java index 787ded85d742..546ef5ebd47f 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java @@ -178,4 +178,9 @@ public VariableImpact getVariableImpact() { this.variableImpact = impact; return impact; } + + @Override + public List getExpressions() { + return Collections.unmodifiableList(expressions); + } } diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StringLiteralExpression.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StringLiteralExpression.java index d6a87d58dd71..c77fa5e6236e 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StringLiteralExpression.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StringLiteralExpression.java @@ -30,4 +30,9 @@ public StringLiteralExpression(final String value) { public String evaluate(final EvaluationContext evaluationContext, AttributeValueDecorator decorator) { return value; } + + @Override + public String getExpression() { + return value; + } } diff --git a/nifi-commons/nifi-record-path/pom.xml b/nifi-commons/nifi-record-path/pom.xml index ead593722791..9a33fad56749 100644 --- a/nifi-commons/nifi-record-path/pom.xml +++ b/nifi-commons/nifi-record-path/pom.xml @@ -94,10 +94,6 @@ org.apache.nifi nifi-record - - org.apache.nifi - nifi-properties - org.apache.nifi nifi-uuid5 diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Format.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Format.java index b621f3dd5823..0ac3ad0dc724 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Format.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/Format.java @@ -16,12 +16,12 @@ */ package org.apache.nifi.record.path.functions; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.record.path.FieldValue; import org.apache.nifi.record.path.RecordPathEvaluationContext; import org.apache.nifi.record.path.StandardFieldValue; import org.apache.nifi.record.path.paths.RecordPathSegment; import org.apache.nifi.record.path.util.RecordPathUtils; -import org.apache.nifi.util.StringUtils; import java.time.Instant; import java.time.ZoneId; diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadLeft.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadLeft.java index 67087a3a61b3..3b819f400a39 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadLeft.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadLeft.java @@ -17,8 +17,8 @@ package org.apache.nifi.record.path.functions; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.record.path.paths.RecordPathSegment; -import org.apache.nifi.util.StringUtils; public class PadLeft extends Padding { diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadRight.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadRight.java index f45d4910af7f..9da50a6f58a4 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadRight.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/PadRight.java @@ -17,8 +17,8 @@ package org.apache.nifi.record.path.functions; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.record.path.paths.RecordPathSegment; -import org.apache.nifi.util.StringUtils; public class PadRight extends Padding { diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToDate.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToDate.java index 6637cc4079e9..9a076c2450a2 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToDate.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/functions/ToDate.java @@ -16,12 +16,12 @@ */ package org.apache.nifi.record.path.functions; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.record.path.FieldValue; import org.apache.nifi.record.path.RecordPathEvaluationContext; import org.apache.nifi.record.path.StandardFieldValue; import org.apache.nifi.record.path.paths.RecordPathSegment; import org.apache.nifi.record.path.util.RecordPathUtils; -import org.apache.nifi.util.StringUtils; import java.time.Instant; import java.time.ZoneId; diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java index 1ab653929767..35f3974a2453 100644 --- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java +++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java @@ -473,6 +473,25 @@ public void setValue(final RecordField field, final Object value) { } } + @Override + public void setValue(final String fieldName, final Object value) { + final Optional existingField = getSchema().getField(fieldName); + RecordField recordField = null; + if (existingField.isPresent()) { + final DataType existingDataType = existingField.get().getDataType(); + final boolean compatible = DataTypeUtils.isCompatibleDataType(value, existingDataType); + if (compatible) { + recordField = existingField.get(); + } + } + if (recordField == null) { + final DataType inferredDataType = DataTypeUtils.inferDataType(value, RecordFieldType.STRING.getDataType()); + recordField = new RecordField(fieldName, inferredDataType); + } + + setValue(recordField, value); + } + @Override public void remove(final RecordField field) { final Optional existingField = resolveField(field); @@ -521,21 +540,6 @@ public void regenerateSchema() { schema = new SimpleRecordSchema(schemaFields); } - @Override - public void setValue(final String fieldName, final Object value) { - final Optional existingField = setValueAndGetField(fieldName, value); - - if (existingField.isEmpty()) { - if (inactiveFields == null) { - inactiveFields = new LinkedHashSet<>(); - } - - final DataType inferredDataType = DataTypeUtils.inferDataType(value, RecordFieldType.STRING.getDataType()); - final RecordField field = new RecordField(fieldName, inferredDataType); - inactiveFields.add(field); - } - } - private Optional setValueAndGetField(final String fieldName, final Object value) { final Optional field = getSchema().getField(fieldName); @@ -642,7 +646,7 @@ public void incorporateSchema(RecordSchema other) { @Override public void incorporateInactiveFields() { - final List updatedFields = new ArrayList<>(); + final Map fieldsByName = new LinkedHashMap<>(); boolean fieldUpdated = false; for (final RecordField field : schema.getFields()) { @@ -651,7 +655,7 @@ public void incorporateInactiveFields() { fieldUpdated = true; } - updatedFields.add(updated); + fieldsByName.put(updated.getFieldName(), updated); } if (!fieldUpdated && (inactiveFields == null || inactiveFields.isEmpty())) { @@ -660,13 +664,21 @@ public void incorporateInactiveFields() { if (inactiveFields != null) { for (final RecordField field : inactiveFields) { - if (!updatedFields.contains(field)) { - updatedFields.add(field); + final RecordField existingField = fieldsByName.get(field.getFieldName()); + if (existingField == null) { + fieldsByName.put(field.getFieldName(), field); + } else { + if (Objects.equals(existingField, field)) { + continue; + } + + final RecordField merged = DataTypeUtils.merge(existingField, field); + fieldsByName.put(field.getFieldName(), merged); } } } - this.schema = new SimpleRecordSchema(updatedFields); + this.schema = new SimpleRecordSchema(new ArrayList<>(fieldsByName.values())); } private RecordField getUpdatedRecordField(final RecordField field) { diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java index cd7fe2da8243..de4ddf27a7c0 100644 --- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java +++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java @@ -798,13 +798,14 @@ public static boolean isArrayTypeCompatible(final Object value, final DataType e return false; } // Either an object array (check the element type) or a String to be converted to byte[] - if (value instanceof Object[]) { - for (Object o : ((Object[]) value)) { + if (value instanceof final Object[] array) { + for (final Object element : array) { // Check each element to ensure its type is the same or can be coerced (if need be) - if (!isCompatibleDataType(o, elementDataType, strict)) { + if (!isCompatibleDataType(element, elementDataType, strict)) { return false; } } + return true; } else { return value instanceof String && RecordFieldType.BYTE.getDataType().equals(elementDataType); diff --git a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestMapRecord.java b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestMapRecord.java index 2240e57270cc..70219ac8a307 100644 --- a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestMapRecord.java +++ b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestMapRecord.java @@ -19,6 +19,7 @@ import org.apache.nifi.serialization.SimpleRecordSchema; import org.apache.nifi.serialization.record.type.ArrayDataType; +import org.apache.nifi.serialization.record.type.ChoiceDataType; import org.apache.nifi.serialization.record.type.RecordDataType; import org.junit.jupiter.api.Test; @@ -32,11 +33,60 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestMapRecord { + @Test + public void testIncorporateInactiveFieldsWithUpdate() { + final List fields = new ArrayList<>(); + fields.add(new RecordField("string", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("number", RecordFieldType.INT.getDataType())); + + final RecordSchema schema = new SimpleRecordSchema(fields); + final Map values = new HashMap<>(); + final Record record = new MapRecord(schema, values); + record.setValue("number", "value"); + record.incorporateInactiveFields(); + + final RecordSchema updatedSchema = record.getSchema(); + final DataType dataType = updatedSchema.getDataType("number").orElseThrow(); + assertSame(RecordFieldType.CHOICE, dataType.getFieldType()); + + final ChoiceDataType choiceDataType = (ChoiceDataType) dataType; + final List subTypes = choiceDataType.getPossibleSubTypes(); + assertEquals(2, subTypes.size()); + assertTrue(subTypes.contains(RecordFieldType.INT.getDataType())); + assertTrue(subTypes.contains(RecordFieldType.STRING.getDataType())); + } + + @Test + public void testIncorporateInactiveFieldsWithConflict() { + final List fields = new ArrayList<>(); + fields.add(new RecordField("string", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("number", RecordFieldType.INT.getDataType())); + + final RecordSchema schema = new SimpleRecordSchema(fields); + final Map values = new HashMap<>(); + final Record record = new MapRecord(schema, values); + record.setValue("new", 8); + record.incorporateInactiveFields(); + + record.setValue("new", "eight"); + record.incorporateInactiveFields(); + + final DataType dataType = record.getSchema().getDataType("new").orElseThrow(); + assertSame(RecordFieldType.CHOICE, dataType.getFieldType()); + + final ChoiceDataType choiceDataType = (ChoiceDataType) dataType; + final List subTypes = choiceDataType.getPossibleSubTypes(); + assertEquals(2, subTypes.size()); + assertTrue(subTypes.contains(RecordFieldType.INT.getDataType())); + assertTrue(subTypes.contains(RecordFieldType.STRING.getDataType())); + } + @Test public void testDefaultValue() { final List fields = new ArrayList<>(); From fabf6bf53633bbc0534f85d0ad7fc7c312477dab Mon Sep 17 00:00:00 2001 From: EndzeitBegins <16666115+EndzeitBegins@users.noreply.github.com> Date: Fri, 8 Mar 2024 16:03:46 +0100 Subject: [PATCH 30/73] NIFI-12617 Set default nifi.web.https.host to localhost This closes #8486 Signed-off-by: David Handermann --- nifi-docs/src/main/asciidoc/administration-guide.adoc | 2 +- .../nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index 72e073c6521a..e8efbb457ceb 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -3839,7 +3839,7 @@ For example, to provide two additional network interfaces, a user could also spe `nifi.web.http.network.interface.eth1=eth1` + + Providing three total network interfaces, including `nifi.web.http.network.interface.default`. -|`nifi.web.https.host`|The HTTPS host. The default value is `127.0.0.1`. +|`nifi.web.https.host`|The HTTPS host. The default value is `localhost`. |`nifi.web.https.port`|The HTTPS port. The default value is `8443`. |`nifi.web.https.port.forwarding`|Same as `nifi.web.http.port.forwarding`, but with HTTPS for secure communication. It is blank by default. |`nifi.web.https.ciphersuites.include`|Cipher suites used to initialize the SSLContext of the Jetty HTTPS port. If unspecified, the runtime SSLContext defaults are used. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index 56aeebfd4aa6..63e7308f2474 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -121,7 +121,7 @@ - 127.0.0.1 + localhost 8443 h2 http/1.1 From 63cc0520dca09acee2dc8f1abb1e2c48c8f4e3e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 20:58:36 +0000 Subject: [PATCH 31/73] NIFI-12879 Upgraded Clojure from 1.11.1 to 1.11.2 This closes #8487 Signed-off-by: David Handermann --- .../nifi-scripting-bundle/nifi-scripting-processors/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml index 3d395c3cb13f..fa1a3205283e 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml @@ -75,7 +75,7 @@ org.clojure clojure - 1.11.1 + 1.11.2 org.apache.commons From 90c7dba34f08921b0b3b528c6a9fc20bf298a288 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Fri, 8 Mar 2024 20:37:12 -0600 Subject: [PATCH 32/73] NIFI-12871 Upgraded Commons Compress from 1.25.0 to 1.26.1 - Adjusted Excel Record Reader test failure to use OpenXML Exception instead of message matching Signed-off-by: Pierre Villard This closes #8488. --- .../java/org/apache/nifi/excel/TestExcelRecordReader.java | 4 +++- pom.xml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java index b83bb52968c9..6fde3d07f115 100644 --- a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java +++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java @@ -25,6 +25,7 @@ import org.apache.nifi.serialization.record.RecordField; import org.apache.nifi.serialization.record.RecordFieldType; import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.poi.openxml4j.exceptions.OpenXML4JRuntimeException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -59,7 +60,8 @@ public void testNonExcelFile() { .build(); MalformedRecordException mre = assertThrows(MalformedRecordException.class, () -> new ExcelRecordReader(configuration, getInputStream("notExcel.txt"), logger)); - assertTrue(ExceptionUtils.getStackTrace(mre).contains("this is not a valid OOXML")); + final Throwable cause = mre.getCause(); + assertInstanceOf(OpenXML4JRuntimeException.class, cause); } @Test diff --git a/pom.xml b/pom.xml index 711916f20253..1fadf4fcadc4 100644 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,7 @@ 3.8.0 1.6.0 1.16.1 - 1.25.0 + 1.26.1 3.14.0 3.10.0 2.15.1 From 7e594d58dc9cea9317b1e31b9cc161e2230900c9 Mon Sep 17 00:00:00 2001 From: Mark Bathori Date: Mon, 11 Mar 2024 14:37:24 +0100 Subject: [PATCH 33/73] NIFI-12884 Corrected documentation for python debugging This closes #8490 Signed-off-by: David Handermann --- nifi-nar-bundles/nifi-py4j-bundle/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-py4j-bundle/README.md b/nifi-nar-bundles/nifi-py4j-bundle/README.md index 0a834880a1d6..df5292e4a5d2 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/README.md +++ b/nifi-nar-bundles/nifi-py4j-bundle/README.md @@ -46,7 +46,7 @@ This process is used to discover available Processors and to create Processors. The following properties may be added to nifi.properties in order to enable remote debugging of the Controller process: -`nifi.python.controller.debugpy.enable` : Indicates whether or not DebugPy should be used when launching hte Controller. +`nifi.python.controller.debugpy.enabled` : Indicates whether DebugPy should be used when launching the Controller. Defaults to `false`. If set to `true`, the Python process that is responsible for discovering and creating Processors will be launched using DebugPy. From e96201ddd1911b5bbfa45ae3fd56336e170a9a80 Mon Sep 17 00:00:00 2001 From: kapkiai Date: Mon, 4 Mar 2024 18:35:56 +0300 Subject: [PATCH 34/73] NIFI-12861 Documented that JASN1Reader requires the JDK This closes #8469 Signed-off-by: David Handermann --- .../src/main/java/org/apache/nifi/jasn1/JASN1Reader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/main/java/org/apache/nifi/jasn1/JASN1Reader.java b/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/main/java/org/apache/nifi/jasn1/JASN1Reader.java index 3abc06089ae2..a577dc3c6be4 100644 --- a/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/main/java/org/apache/nifi/jasn1/JASN1Reader.java +++ b/nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/main/java/org/apache/nifi/jasn1/JASN1Reader.java @@ -71,7 +71,8 @@ import java.util.stream.Collectors; @Tags({"asn", "ans1", "jasn.1", "jasn1", "record", "reader", "parser"}) -@CapabilityDescription("Reads ASN.1 content and creates NiFi records.") +@CapabilityDescription("Reads ASN.1 content and creates NiFi records. " + + "NOTE: ASN.1 schema preparation requires the JDK at runtime for model compilation.") public class JASN1Reader extends AbstractConfigurableComponent implements RecordReaderFactory { private static final PropertyDescriptor ROOT_MODEL_NAME = new PropertyDescriptor.Builder() From 7db1664a7e2d473a5e30633bea547004e22487c6 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Mon, 11 Mar 2024 19:30:23 -0500 Subject: [PATCH 35/73] NIFI-12886 Upgraded Jackson JSON from 2.16.1 to 2.16.2 Signed-off-by: Pierre Villard This closes #8492. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1fadf4fcadc4..774b43500742 100644 --- a/pom.xml +++ b/pom.xml @@ -130,7 +130,7 @@ 2.9.0 10.17.1.0 12.0.6 - 2.16.1 + 2.16.2 1.11.3 4.0.4 1.3.2 From 06c011306356992ef6fa9fcb3b31c9ad658d7589 Mon Sep 17 00:00:00 2001 From: James Mingardi-Elliott Date: Wed, 13 Mar 2024 17:29:29 -0400 Subject: [PATCH 36/73] NIFI-12870 Semantic colors (#8480) Next step to color theming Update theming to reference colors semantically Material and Canvas palettes are reordered so that in all cases they go from 50 = lightest / least amount of color to 900 = darkest / most amount of color applied. Usage of color has been changed so that Material's primary, accent, and warn values are used by semantic reference of 'default', 'lighter' and 'darker' rather than explicit number values. The Canvas palettes still have values referenced directly because they are a special case. Added SASS utilities: - To help ensure color contrast for text and backgrounds by checking for a 4.5:1 contrast ratio. - To provide helper functions that somewhat replicate Material designs approach to Surface and On Surface concepts. This is how the same Canvas palettes can be used for light and dark modes. Some minor tweaks to the styling of the flow canvas to bring custom NiFi components and the Angular Material components closer together visually. Moved the Canvas theme declaration to a separate file so the Material themes can be more easily swapped out without needing to redeclare the Canvas themes. This closes #8480 --- .../src/main/nifi/angular.json | 3 +- .../nifi/src/app/_app.component-theme.scss | 16 +- .../_access-policies.component-theme.scss | 4 +- ...nant-to-policy-dialog.component-theme.scss | 4 +- .../override-policy-dialog.component.html | 4 +- ...onent-access-policies.component-theme.scss | 16 +- .../feature/_bulletins.component-theme.scss | 4 +- .../_bulletin-board-list.component-theme.scss | 13 +- .../bulletin-board.component.html | 2 +- .../feature/_counters.component-theme.scss | 4 +- .../service/canvas-utils.service.ts | 4 +- .../manager/connection-manager.service.ts | 6 +- .../service/manager/port-manager.service.ts | 8 +- .../manager/process-group-manager.service.ts | 4 +- .../manager/processor-manager.service.ts | 10 +- .../ui/canvas/_canvas.component-theme.scss | 326 +++++++------ .../ui/canvas/canvas.component.scss | 4 - .../ui/canvas/canvas.component.ts | 4 +- .../footer/_footer.component-theme.scss | 19 +- .../_navigation-control.component-theme.scss | 28 +- .../birdseye/_birdseye.component-theme.scss | 13 +- .../_operation-control.component-theme.scss | 38 +- .../header/_header.component-theme.scss | 7 - .../_flow-status.component-theme.scss | 34 +- .../_new-canvas-item.component-theme.scss | 43 +- .../search/_search.component-theme.scss | 29 +- .../_prioritizers.component-theme.scss | 25 +- ..._create-process-group.component-theme.scss | 6 +- .../relationship-settings.component.html | 4 +- ...-remote-process-group.component-theme.scss | 2 +- .../banner/_banner.component-theme.scss | 8 +- .../_controller-services.component-theme.scss | 4 +- .../_manage-remote-ports.component-theme.scss | 10 +- .../login/feature/_login.component-theme.scss | 10 +- .../_login-form.component-theme.scss | 4 +- .../_parameter-contexts.component-theme.scss | 4 +- ...r-context-inheritance.component-theme.scss | 25 +- .../feature/_provenance.component-theme.scss | 36 +- ...rovenance-event-table.component-theme.scss | 13 +- .../lineage/_lineage.component-theme.scss | 26 +- .../lineage/lineage.component.ts | 2 +- .../_flowfile-dialog.component-theme.scss | 4 +- .../_flowfile-table.component-theme.scss | 8 +- .../feature/_settings.component-theme.scss | 11 +- .../feature/_summary.component-theme.scss | 11 +- .../users/feature/_users.component-theme.scss | 4 +- .../_component-context.component-theme.scss | 14 +- .../_context-menu.component-theme.scss | 30 +- .../edit-parameter-dialog.component.html | 4 +- .../edit-tenant-dialog.component.html | 4 +- .../_extension-creation.component-theme.scss | 54 +- .../_navigation.component-theme.scss | 43 +- .../new-property-dialog.component.html | 4 +- .../nf-editor/_nf-editor.component-theme.scss | 30 +- ...ovenance-event-dialog.component-theme.scss | 4 +- .../resizable/_resizable.component-theme.scss | 6 +- .../_status-history.component-theme.scss | 9 +- ...em-diagnostics-dialog.component-theme.scss | 4 +- .../_property-hint-tip.component-theme.scss | 4 +- .../assets/styles/_listing-table-theme.scss | 46 +- .../nifi/src/assets/themes/nifi-canvas.scss | 186 +++++++ .../src/main/nifi/src/assets/themes/nifi.scss | 461 ++++-------------- .../main/nifi/src/assets/themes/purple.scss | 422 +++------------- .../src/main/nifi/src/assets/utils.scss | 186 +++++++ .../src/main/nifi/src/styles.scss | 150 +++--- 65 files changed, 1173 insertions(+), 1352 deletions(-) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi-canvas.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json index 7ec3cfdd5448..84c1979e8237 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json @@ -98,6 +98,7 @@ } }, "cli": { - "schematicCollections": ["@angular-eslint/schematics"] + "schematicCollections": ["@angular-eslint/schematics"], + "analytics": false } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss index b3f95a096847..9c9b304e34dd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss @@ -30,19 +30,19 @@ $canvas-accent-palette: map.get($canvas-color-config, 'accent'); // Get hues from palette - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); + $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); - $canvas-accent-palette-500: mat.get-color-from-palette($canvas-accent-palette, 500); + $canvas-accent-palette-default: mat.get-color-from-palette($canvas-accent-palette, default); .splash { - background-color: $primary-palette-500; + background-color: $primary-palette-default; } // https://github.com/angular/components/issues/11426 .nifi-snackbar .mdc-snackbar__surface { - background-color: $primary-palette-300 !important; + background-color: $primary-palette-lighter !important; } // https://github.com/angular/components/issues/11426 @@ -52,10 +52,10 @@ // https://github.com/angular/components/issues/11426 .nifi-snackbar .mat-mdc-button:not(:disabled) .mdc-button__label { - color: $accent-palette-A400; + color: $accent-palette-default; } .fa.fa-check.complete { - color: $canvas-accent-palette-500; + color: $canvas-accent-palette-default; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/feature/_access-policies.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/feature/_access-policies.component-theme.scss index 4cbbf1465a74..ac09e0890495 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/feature/_access-policies.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/feature/_access-policies.component-theme.scss @@ -26,9 +26,9 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .access-policies-header { - color: $primary-palette-500; + color: $primary-palette-default; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/_add-tenant-to-policy-dialog.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/_add-tenant-to-policy-dialog.component-theme.scss index 2476207bdd4f..89844e18b2ab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/_add-tenant-to-policy-dialog.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/_add-tenant-to-policy-dialog.component-theme.scss @@ -26,11 +26,11 @@ $accent-palette: map.get($color-config, 'accent'); // Get hues from palette - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); .add-tenant-to-policy-form { .fa { - color: $accent-palette-A400; + color: $accent-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/override-policy-dialog/override-policy-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/override-policy-dialog/override-policy-dialog.component.html index 6fbdd6c2fa05..715a7e693396 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/override-policy-dialog/override-policy-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/override-policy-dialog/override-policy-dialog.component.html @@ -20,8 +20,8 @@

Override Policy

Do you want to override with a copy of the inherited policy or an empty policy?
- Copy - Empty + Copy + Empty diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/_component-access-policies.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/_component-access-policies.component-theme.scss index 0878ecc65d99..2055d2131ecf 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/_component-access-policies.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/_component-access-policies.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -25,27 +26,26 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $warn-palette: map.get($color-config, 'warn'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); + $accent-palette: map.get($color-config, 'accent'); // Get hues from palette - $primary-palette-700: mat.get-color-from-palette($primary-palette, 700); - $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); - $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); + $primary-palette-darker: mat.get-color-from-palette($primary-palette, darker); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); + $on-surface: utils.get-on-surface($canvas-color-config); .component-access-policies { .operation-context-logo { .icon { - color: $warn-palette-A200; + color: $accent-palette-default; } } .operation-context-name { - color: $canvas-primary-palette-A200; + color: $on-surface; } .operation-context-type { - color: $primary-palette-700; + color: $primary-palette-darker; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/feature/_bulletins.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/feature/_bulletins.component-theme.scss index 572454e25313..6aec26c0d06a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/feature/_bulletins.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/feature/_bulletins.component-theme.scss @@ -26,9 +26,9 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .bulletin-board-header { - color: $primary-palette-500; + color: $primary-palette-default; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/_bulletin-board-list.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/_bulletin-board-list.component-theme.scss index 3e8f1351d61a..e28a0568bc7e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/_bulletin-board-list.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/_bulletin-board-list.component-theme.scss @@ -26,21 +26,22 @@ // Get the color palette from the color-config. $warn-palette: map.get($color-config, 'warn'); $canvas-accent-palette: map.get($canvas-color-config, 'accent'); + $canvas-warn-palette: map.get($canvas-color-config, 'warn'); // Get hues from palette - $canvas-accent-palette-700: mat.get-color-from-palette($canvas-accent-palette, 700); - $canvas-accent-palette-A400: mat.get-color-from-palette($canvas-accent-palette, 'A400'); - $warn-palette-600: mat.get-color-from-palette($warn-palette, 600); + $canvas-accent-palette-darker: mat.get-color-from-palette($canvas-accent-palette, darker); + $canvas-warn-palette-A400: mat.get-color-from-palette($canvas-warn-palette, 'A400'); + $warn-palette-default: mat.get-color-from-palette($warn-palette, default); .bulletin-error { - color: $warn-palette-600; + color: $warn-palette-default; } .bulletin-warn { - color: $canvas-accent-palette-A400; + color: $canvas-warn-palette-A400; } .bulletin-normal { - color: $canvas-accent-palette-700; + color: $canvas-accent-palette-darker; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html index 043f3a6fa8fd..1b5df94263d1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html @@ -30,7 +30,7 @@
- Auto-refresh
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/feature/_counters.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/feature/_counters.component-theme.scss index d1954ee3dd5a..4f997617a8ee 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/feature/_counters.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/feature/_counters.component-theme.scss @@ -26,9 +26,9 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .counter-header { - color: $primary-palette-500; + color: $primary-palette-default; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-utils.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-utils.service.ts index 4f3c284e822c..30d4cb21ca7d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-utils.service.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/canvas-utils.service.ts @@ -1253,9 +1253,9 @@ export class CanvasUtils { }) .attr('class', function () { if (terminatedThreads > 0) { - return `active-thread-count-icon warn-400`; + return `active-thread-count-icon warn-default`; } else { - return `active-thread-count-icon primary-500`; + return `active-thread-count-icon primary-default`; } }) .style('display', 'block') diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/connection-manager.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/connection-manager.service.ts index 1fbc0d7f0347..616e77b860a3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/connection-manager.service.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/connection-manager.service.ts @@ -1424,16 +1424,16 @@ export class ConnectionManager { // update the coloring of the backgrounds backgrounds.forEach((background, i) => { if (i % 2 === 0) { - background.attr('class', 'primary-contrast-800'); + background.attr('class', 'surface-darker'); } else { - background.attr('class', 'primary-contrast-900'); + background.attr('class', 'surface'); } }); // update the coloring of the label borders borders.forEach((border, i) => { if (i > 0) { - border.attr('class', 'primary-200'); + border.attr('class', 'canvas-primary-lighter'); } else { border.attr('class', 'transparent'); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/port-manager.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/port-manager.service.ts index 0def655caaf0..39025cce2c88 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/port-manager.service.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/port-manager.service.ts @@ -386,14 +386,14 @@ export class PortManager { updated .select('text.run-status-icon') .attr('class', function (d: any) { - let clazz = 'primary-500'; + let clazz = 'primary-default'; if (d.status.aggregateSnapshot.runStatus === 'Invalid') { - clazz = 'canvas-accent-A200'; + clazz = 'canvas-warn-A200'; } else if (d.status.aggregateSnapshot.runStatus === 'Running') { - clazz = 'canvas-accent-200'; + clazz = 'canvas-accent-lighter'; } else if (d.status.aggregateSnapshot.runStatus === 'Stopped') { - clazz = 'warn-200'; + clazz = 'warn-lighter'; } return `run-status-icon ${clazz}`; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/process-group-manager.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/process-group-manager.service.ts index d7120dab75b3..d5d4e09dfe64 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/process-group-manager.service.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/process-group-manager.service.ts @@ -1087,10 +1087,10 @@ export class ProcessGroupManager { } else if (vciState === 'LOCALLY_MODIFIED') { return `version-control primary-contrast-A700`; } else { - return `version-control canvas-accent-600`; + return `version-control canvas-accent-darker`; } } else { - return 'version-control primary-contrast-200'; + return 'version-control on-surface'; } }) .text(function () { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/processor-manager.service.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/processor-manager.service.ts index 21b86db0aa1a..9c176c040b11 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/processor-manager.service.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/service/manager/processor-manager.service.ts @@ -670,16 +670,16 @@ export class ProcessorManager { updated .select('text.run-status-icon') .attr('class', function (d: any) { - let clazz = 'primary-500'; + let clazz = 'primary-default'; if (d.status.aggregateSnapshot.runStatus === 'Validating') { - clazz = 'warn-contrast-300'; + clazz = 'canvas-primary-500'; } else if (d.status.aggregateSnapshot.runStatus === 'Invalid') { - clazz = 'canvas-accent-A400'; + clazz = 'canvas-warn-A200'; } else if (d.status.aggregateSnapshot.runStatus === 'Running') { - clazz = 'canvas-accent-200'; + clazz = 'canvas-accent-lighter'; } else if (d.status.aggregateSnapshot.runStatus === 'Stopped') { - clazz = 'warn-200'; + clazz = 'warn-lighter'; } return `run-status-icon ${clazz}`; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/_canvas.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/_canvas.component-theme.scss index 5dee394e9900..ea4df773854d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/_canvas.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/_canvas.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -29,51 +30,58 @@ $warn-palette: map.get($color-config, 'warn'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); $canvas-accent-palette: map.get($canvas-color-config, 'accent'); + $canvas-warn-palette: map.get($canvas-color-config, 'warn'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-200: mat.get-color-from-palette($primary-palette, 200); - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $accent-palette-A200: mat.get-color-from-palette($accent-palette, 'A200'); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-200: mat.get-color-from-palette($canvas-primary-palette, 200); - $canvas-primary-palette-300: mat.get-color-from-palette($canvas-primary-palette, 300); - $canvas-primary-palette-400: mat.get-color-from-palette($canvas-primary-palette, 400); + $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); + $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); + $primary-palette-lighter-contrast: mat.get-color-from-palette($primary-palette, lighter-contrast); + $primary-palette-default-contrast: mat.get-color-from-palette($primary-palette, 'default-contrast'); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); + $accent-palette-darker: mat.get-color-from-palette($accent-palette, 'darker'); + + // Canvas colors $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); - $canvas-primary-palette-600: mat.get-color-from-palette($canvas-primary-palette, 600); - $canvas-primary-palette-700: mat.get-color-from-palette($canvas-primary-palette, 700); - $canvas-primary-palette-800: mat.get-color-from-palette($canvas-primary-palette, 800); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); + $canvas-primary-palette-A100: mat.get-color-from-palette($canvas-primary-palette, A100); $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); - $canvas-primary-palette-A400: mat.get-color-from-palette($canvas-primary-palette, 'A400'); $canvas-primary-palette-A700: mat.get-color-from-palette($canvas-primary-palette, 'A700'); - $canvas-accent-palette-200: mat.get-color-from-palette($canvas-accent-palette, 200); - $canvas-accent-palette-300: mat.get-color-from-palette($canvas-accent-palette, 300); - $canvas-accent-palette-600: mat.get-color-from-palette($canvas-accent-palette, 600); - $canvas-accent-palette-800: mat.get-color-from-palette($canvas-accent-palette, 800); - $canvas-accent-palette-900: mat.get-color-from-palette($canvas-accent-palette, 900); - $canvas-accent-palette-A200: mat.get-color-from-palette($canvas-accent-palette, 'A200'); - $canvas-accent-palette-A400: mat.get-color-from-palette($canvas-accent-palette, 'A400'); + $canvas-accent-palette-lighter: mat.get-color-from-palette($canvas-accent-palette, lighter); + $canvas-accent-palette-default: mat.get-color-from-palette($canvas-accent-palette, default); + $canvas-accent-palette-darker: mat.get-color-from-palette($canvas-accent-palette, darker); $canvas-accent-palette-A700: mat.get-color-from-palette($canvas-accent-palette, 'A700'); - $warn-palette-200: mat.get-color-from-palette($warn-palette, 200); + $canvas-accent-palette-500: mat.get-color-from-palette($canvas-accent-palette, 500); + $canvas-warn-palette-800: mat.get-color-from-palette($canvas-warn-palette, 800); + $canvas-warn-palette-A200: mat.get-color-from-palette($canvas-warn-palette, 'A200'); + $canvas-warn-palette-A400: mat.get-color-from-palette($canvas-warn-palette, 'A400'); + $canvas-warn-palette-A700: mat.get-color-from-palette($canvas-warn-palette, 'A700'); $warn-palette-400: mat.get-color-from-palette($warn-palette, 400); - $warn-palette-800: mat.get-color-from-palette($warn-palette, 800); - $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); - $warn-palette-A400: mat.get-color-from-palette($warn-palette, 'A400'); - $warn-palette-A700: mat.get-color-from-palette($warn-palette, 'A700'); + $warn-palette-700: mat.get-color-from-palette($warn-palette, 700); + $warn-palette-lighter: mat.get-color-from-palette($warn-palette, lighter); + $warn-palette-default: mat.get-color-from-palette($warn-palette, 'default'); + $warn-palette-darker: mat.get-color-from-palette($warn-palette, 'darker'); + + // Shadows should always be darker. We explicitly set this so the SVG shadows are correct in both modes. + $drop-shadow-color: black; + $is-dark: map-get($canvas-color-config, is-dark); + + $surface: utils.get-surface($canvas-color-config); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $on-surface: utils.get-on-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + $on-surface-medium: utils.get-on-surface($canvas-color-config, medium); + $surface-alt: utils.get-surface($canvas-color-config, default, 'accent'); .canvas-background { - background-color: $canvas-primary-palette-600; - background-image: linear-gradient(to right, $canvas-primary-palette-700 1px, transparent 1px), - linear-gradient(to bottom, $canvas-primary-palette-700 1px, transparent 1px); + background-color: $surface-darker; + background-image: linear-gradient(to right, $surface-alt 1px, transparent 1px), + linear-gradient(to bottom, $surface-alt 1px, transparent 1px); } /* svg styles */ svg.canvas-svg { text.unset { - fill: $canvas-primary-palette-400; + fill: $canvas-primary-palette-500; } /* @@ -83,60 +91,60 @@ fill: transparent; } - .warn-200 { - fill: $warn-palette-200; + .warn-lighter { + fill: $warn-palette-lighter; } .warn-400 { fill: $warn-palette-400; } - .warn-A700 { - fill: $warn-palette-A700; + .warn-default { + fill: $warn-palette-default; } - .warn-contrast-300 { - fill: $canvas-primary-palette-400; + .canvas-primary-500 { + fill: $canvas-primary-palette-500; } - .canvas-accent-200 { - fill: $canvas-accent-palette-200; + .canvas-accent-lighter { + fill: $canvas-accent-palette-lighter; } - .canvas-accent-600 { - fill: $canvas-accent-palette-600; + .canvas-primary-lighter { + fill: $canvas-primary-palette-A200; } - .canvas-accent-A200 { - fill: $canvas-accent-palette-A200; + .canvas-accent-darker { + fill: $canvas-accent-palette-darker; } - .canvas-accent-A400 { - fill: $canvas-accent-palette-A400; + .canvas-warn-A200 { + fill: $canvas-warn-palette-A200; } - .primary-200 { - fill: $primary-palette-200; + .primary-lighter { + fill: $primary-palette-lighter; } - .primary-500 { - fill: $primary-palette-500; + .primary-default { + fill: $primary-palette-default; } - .primary-contrast-200 { - fill: $canvas-primary-palette-200; + .on-surface { + fill: $on-surface; } - .primary-contrast-300 { - fill: $canvas-primary-palette-300; + .canvas-primary-palette-500 { + fill: $canvas-primary-palette-500; } - .primary-contrast-800 { - fill: $canvas-primary-palette-800; + .surface-darker { + fill: $surface-darker; } - .primary-contrast-900 { - fill: $canvas-primary-palette-900; + .surface { + fill: $surface; } .primary-contrast-A700 { @@ -144,64 +152,70 @@ } g.component rect.body { - fill: $canvas-primary-palette-900; + fill: $surface; } g.component rect.body.unauthorized { - fill: $canvas-primary-palette-700; + fill: $surface-darker; } - g.component rect.border { - stroke: $canvas-primary-palette-50; + g.component rect.border, + g.connections rect.border { + stroke: $on-surface-lighter; + stroke-width: if( + $is-dark, + 2, + 1 + ); // Dark mode gets a wider stroke to provide contrast between the canvas and components } g.component rect.border.unauthorized { - stroke: $warn-palette-400 !important; + stroke: $warn-palette-darker !important; } g.component rect.border.ghost { - stroke: $canvas-primary-palette-400 !important; + stroke: $canvas-primary-palette-500 !important; } g.component rect.processor-icon-container.unauthorized { - fill: $canvas-primary-palette-700 !important; + fill: $surface-darker; } g.component.selected rect.border { - stroke: $accent-palette-A400 !important; + stroke: utils.get-color-on-surface($color-config, $surface); } text.stats-label { - fill: $canvas-primary-palette-A200; + fill: $on-surface-medium; } text.stats-value { - fill: $warn-palette-A400; + fill: utils.get-color-on-surface($color-config, $surface, 'accent'); } text.stats-info { - fill: $primary-palette-500; + fill: utils.get-color-on-surface($color-config, $surface); } text.bulletin-icon { - fill: $canvas-primary-palette-900; + fill: $surface; } rect.bulletin-background { - fill: $warn-palette-400; + fill: utils.get-color-on-surface($color-config, $surface, 'warn'); } text.active-thread-count-icon { - fill: $primary-palette-500; + fill: $primary-palette-default; } text.active-thread-count { - fill: $warn-palette-A400; + fill: $accent-palette-darker; } path.component-comments { - fill: $canvas-primary-palette-50; - stroke: $canvas-primary-palette-50; + fill: $canvas-primary-palette-A200; + stroke: $canvas-primary-palette-A200; } /* @@ -209,49 +223,49 @@ */ g.component.connectable-destination rect.border { - stroke: $canvas-accent-palette-900; + stroke: $canvas-accent-palette-500; } rect.component-selection, rect.drag-selection, rect.label-drag { - stroke: $canvas-primary-palette-A400; + stroke: $on-surface-medium; fill: transparent; } text.add-connect { - fill: $accent-palette-A400; + fill: utils.get-color-on-surface($color-config, $surface); } /* Processor */ #component-drop-shadow feFlood { - flood-color: $canvas-primary-palette-200; + flood-color: $drop-shadow-color; } #connection-full-drop-shadow feFlood { - flood-color: $warn-palette-400; + flood-color: $warn-palette-default; } rect.processor-read-write-stats { - fill: $canvas-primary-palette-900; + fill: $surface; } rect.processor-stats-border { - fill: $primary-palette-200; + fill: $on-surface-lighter; } rect.processor-stats-in-out { - fill: $canvas-primary-palette-800; + fill: $surface-darker; } text.processor-name { - fill: $canvas-primary-palette-A200; + fill: $on-surface-medium; } text.processor-type { - fill: $primary-palette-500; + fill: utils.get-color-on-surface($color-config, $surface, 'accent'); } text.processor-bundle { @@ -259,16 +273,16 @@ } rect.processor-icon-container { - fill: $canvas-primary-palette-900; + fill: $surface; } circle.restricted-background, circle.is-primary-background { - fill: $canvas-primary-palette-900; + fill: $surface; } text.restricted { - fill: $warn-palette-400; + fill: $warn-palette-default; } /* @@ -276,44 +290,44 @@ */ g.connection rect.body { - fill: $canvas-primary-palette-900; + fill: $surface; } g.connection rect.body.unauthorized { - fill: $canvas-primary-palette-700; + fill: $surface-darker; } g.connection rect.border.unauthorized { - stroke: $warn-palette-400; + stroke: $warn-palette-default; } g.connection rect.border.full { - stroke: $warn-palette-A700; + stroke: $accent-palette-darker; } g.connection.selected rect.border { - stroke: $canvas-accent-palette-A700; + stroke: $canvas-warn-palette-A700; } path.connector { - stroke: $warn-palette-800; + stroke: $warn-palette-700; } path.connector.connectable { - stroke: $canvas-accent-palette-900; + stroke: $canvas-accent-palette-500; } g.connection path.connection-path { fill: none; - stroke: $canvas-primary-palette-50; + stroke: $on-surface; } g.connection path.connection-path.full { - stroke: $warn-palette-400; + stroke: $warn-palette-default; } g.connection path.connection-path.unauthorized { - stroke: $warn-palette-400; + stroke: $warn-palette-default; } text.connection-from-run-status, @@ -321,17 +335,16 @@ text.expiration-icon, text.load-balance-icon, text.penalized-icon { - fill: $primary-palette-500; - text-shadow: 0 0 4px $canvas-primary-palette-900; + fill: $primary-palette-default; } text.load-balance-icon-active { - fill: $accent-palette-A200; + fill: $accent-palette-darker; } text.connection-from-run-status.is-missing-port, text.connection-to-run-status.is-missing-port { - fill: $canvas-accent-palette-A200; + fill: $canvas-warn-palette-A200; } g.connection rect.backpressure-tick { @@ -340,12 +353,12 @@ g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down { - fill: $canvas-primary-palette-900; + fill: $surface; } g.connection rect.backpressure-tick.data-size-prediction, g.connection rect.backpressure-tick.object-prediction { - fill: $canvas-primary-palette-50; + fill: $canvas-primary-palette-A200; } g.connection rect.backpressure-tick.data-size-prediction.not-configured, @@ -356,12 +369,12 @@ } g.connection rect.backpressure-tick.not-configured { - fill: $canvas-primary-palette-400; + fill: $canvas-primary-palette-500; } g.connection rect.backpressure-object, g.connection rect.backpressure-data-size { - fill: $canvas-primary-palette-500; + fill: $on-surface-lighter; } g.connection rect.backpressure-object.not-configured, @@ -370,26 +383,26 @@ } g.connection rect.backpressure-percent { - fill: $canvas-accent-palette-300; + fill: $canvas-accent-palette-default; } g.connection rect.backpressure-percent.warning { - fill: $canvas-accent-palette-A400; + fill: $canvas-warn-palette-A400; } g.connection rect.backpressure-percent.error { - fill: $warn-palette-400; + fill: $warn-palette-default; } /* ghost connection */ g.connection.ghost path.connection-path, g.connection.ghost rect.connection-label { - stroke: $canvas-primary-palette-400; + stroke: $canvas-primary-palette-500; } g.connection path.connection-selection-path { - stroke: $canvas-accent-palette-A700; + stroke: $canvas-warn-palette-A700; fill: none; } @@ -398,141 +411,134 @@ } g.connection rect.startpoint { - stroke: $warn-palette-800; - fill: $warn-palette-800; + stroke: $canvas-warn-palette-800; + fill: $canvas-warn-palette-800; } g.connection rect.midpoint { - stroke: $canvas-accent-palette-A700; - fill: $canvas-accent-palette-A700; + stroke: $canvas-warn-palette-A700; + fill: $canvas-warn-palette-A700; } g.connection rect.endpoint { - stroke: $canvas-accent-palette-800; - fill: $canvas-accent-palette-800; + stroke: $canvas-accent-palette-A700; + fill: $canvas-accent-palette-A700; } /* labels */ g.label rect.labelpoint { - stroke: $canvas-accent-palette-A700; - fill: $canvas-accent-palette-A700; + stroke: $canvas-warn-palette-A700; + fill: $canvas-warn-palette-A700; } /* funnels */ text.funnel-icon { - fill: $warn-palette-A200; + fill: $accent-palette-default; } /* ports */ text.port-name { - fill: $canvas-primary-palette-A200; + fill: $on-surface-medium; } text.port-icon { - fill: $warn-palette-A200; + fill: $accent-palette-default; } rect.remote-banner { - fill: $primary-palette-100; + fill: $surface-alt; } text.port-transmission-icon { - fill: $primary-palette-500; + fill: utils.get-color-on-surface($color-config, $surface-alt); } /* process groups */ rect.process-group-stats-in-out { - fill: $canvas-primary-palette-900; + fill: $surface; } rect.process-group-stats-border { - fill: $primary-palette-200; + fill: $on-surface-lighter; } rect.process-group-queued-stats { - fill: $canvas-primary-palette-800; + fill: $surface-darker; } rect.process-group-read-write-stats { - fill: $canvas-primary-palette-800; + fill: $surface-darker; } - rect.process-group-details-banner { - fill: $primary-palette-100; + rect.process-group-details-banner, + rect.remote-process-group-details-banner, + rect.remote-process-group-last-refresh-rect { + fill: $surface-alt; } - text.process-group-name { - fill: $canvas-primary-palette-A200; + text.version-control { + text-shadow: 0 0 4px $surface; } - text.version-control { - text-shadow: 0 0 4px $canvas-primary-palette-900; + $pg-surface: utils.ensure-contrast( + $primary-palette-default, + $primary-palette-default-contrast, + $primary-palette-lighter + ); + rect.process-group-banner, + rect.remote-process-group-banner { + fill: $pg-surface; } - rect.process-group-banner { - fill: $primary-palette-200; + text.process-group-name, + text.remote-process-group-name { + fill: utils.ensure-contrast($primary-palette-default-contrast, $pg-surface, $on-surface-medium); } text.process-group-contents-count { - fill: $warn-palette-A400; + fill: utils.get-color-on-surface($color-config, $surface, 'accent'); } g.process-group.drop rect.border { - stroke: $canvas-accent-palette-800; + stroke: $canvas-accent-palette-A700; } text.process-group-contents-icon { - fill: $primary-palette-500; + fill: $primary-palette-default; } /* remote process group */ rect.remote-process-group-stats-border { - fill: $primary-palette-200; + fill: $on-surface-lighter; } rect.remote-process-group-sent-stats { - fill: $canvas-primary-palette-800; + fill: $surface-darker; } rect.remote-process-group-received-stats { - fill: $canvas-primary-palette-900; - } - - rect.remote-process-group-details-banner { - fill: $primary-palette-100; - } - - rect.remote-process-group-last-refresh-rect { - fill: $primary-palette-100; - } - - text.remote-process-group-name { - fill: $canvas-primary-palette-A200; + fill: $surface; } text.remote-process-group-uri { - fill: $accent-palette-A400; - } - - rect.remote-process-group-banner { - fill: $primary-palette-200; + fill: utils.get-color-on-surface($color-config, $surface); } text.remote-process-group-transmission-status { - fill: $primary-palette-500; + fill: $primary-palette-default-contrast; } text.remote-process-group-transmission-secure { - fill: $accent-palette-A400; + fill: utils.get-color-on-surface($color-config, $surface); } text.remote-process-group-last-refresh { - fill: $primary-palette-500; + fill: utils.get-color-on-surface($color-config, $surface); } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.scss index c52049eb4910..4d8ef1fdb915 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.scss @@ -47,10 +47,6 @@ font-family: Roboto; } - g.component rect.border { - stroke-width: 1; - } - g.component rect.border.unauthorized { stroke-width: 3; stroke-dasharray: 4; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.ts index 37686388219f..ef2ad47503ef 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/canvas.component.ts @@ -339,13 +339,13 @@ export class Canvas implements OnInit, OnDestroy { .attr('orient', 'auto') .attr('class', function (d: string) { if (d === 'ghost') { - return 'primary-contrast-300'; + return 'canvas-primary-palette-500'; } else if (d === 'unauthorized') { return 'warn-400'; } else if (d === 'full') { return 'warn-400'; } else { - return 'primary-contrast-200'; + return 'on-surface'; } }) .append('path') diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/footer/_footer.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/footer/_footer.component-theme.scss index c39b21fd62dc..b55d8333ec88 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/footer/_footer.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/footer/_footer.component-theme.scss @@ -17,26 +17,25 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. - $color-config: mat.get-color-config($material-theme); $canvas-color-config: mat.get-color-config($canvas-theme); // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $primary-palette-600: mat.get-color-from-palette($primary-palette, 600); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); + $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, A200); + $surface: utils.get-surface($canvas-color-config); + $on-surface: utils.get-on-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); .breadcrumb-container { - box-shadow: 0 1px 6px $canvas-primary-palette-50; - background-color: $canvas-primary-palette-900; - border-top: 1px solid $primary-palette-300; - color: $primary-palette-600; + box-shadow: 0 1px 6px $canvas-primary-palette-A200; + background-color: $surface; + border-top: 1px solid $on-surface-lighter; + color: $on-surface; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/_navigation-control.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/_navigation-control.component-theme.scss index 48c907926fad..85bb7748aa54 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/_navigation-control.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/_navigation-control.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -24,39 +25,34 @@ $canvas-color-config: mat.get-color-config($canvas-theme); // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); - $accent-palette: map.get($color-config, 'accent'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-600: mat.get-color-from-palette($canvas-primary-palette, 600); - $canvas-primary-palette-A100: mat.get-color-from-palette($canvas-primary-palette, 'A100'); + $on-surface-medium: utils.get-on-surface($canvas-color-config, medium); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + $surface: utils.get-surface($canvas-color-config); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $surface-highlight: utils.get-on-surface($canvas-color-config, highlight); $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); div.navigation-control { - box-shadow: 0 1px 6px $canvas-primary-palette-50; - background-color: $canvas-primary-palette-600; - border-top: 1px solid $primary-palette-300; - border-right: 1px solid $primary-palette-300; - border-bottom: 1px solid $primary-palette-300; + box-shadow: 0 1px 6px $canvas-primary-palette-A200; + background-color: $surface; + border: 1px solid $on-surface-lighter; .fa, .icon { - color: $accent-palette-A400; + color: utils.get-color-on-surface($color-config, $surface-darker); } .navigation-control-header { &:hover { - background: linear-gradient(90deg, $canvas-primary-palette-A100 252px, $primary-palette-100 34px); + background: linear-gradient(-90deg, $surface-highlight 34px, transparent 35px); } } .navigation-control-title { - color: $canvas-primary-palette-A200; + color: $on-surface-medium; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/birdseye/_birdseye.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/birdseye/_birdseye.component-theme.scss index 20cdc0157963..530f7705049f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/birdseye/_birdseye.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/birdseye/_birdseye.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -24,20 +25,18 @@ $canvas-color-config: mat.get-color-config($canvas-theme); // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-600: mat.get-color-from-palette($primary-palette, 600); - $canvas-primary-palette-700: mat.get-color-from-palette($canvas-primary-palette, 700); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); + $canvas-primary-palette-A100: mat.get-color-from-palette($canvas-primary-palette, A100); + $surface: utils.get-surface($canvas-color-config); #birdseye { - background: $canvas-primary-palette-900; - border: 1px solid $canvas-primary-palette-700; + background: $surface; + border: 1px solid $canvas-primary-palette-A100; rect.birdseye-brush { - stroke: $primary-palette-600; + stroke: utils.get-color-on-surface($color-config, $surface); fill: transparent; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/_operation-control.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/_operation-control.component-theme.scss index a562bf3a09f5..6f85e54422fc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/_operation-control.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/_operation-control.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -24,57 +25,50 @@ $canvas-color-config: mat.get-color-config($canvas-theme); // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); $accent-palette: map.get($color-config, 'accent'); - $warn-palette: map.get($color-config, 'warn'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $primary-palette-700: mat.get-color-from-palette($primary-palette, 700); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); - $warn-palette-A400: mat.get-color-from-palette($warn-palette, 'A400'); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-600: mat.get-color-from-palette($canvas-primary-palette, 600); - $canvas-primary-palette-A100: mat.get-color-from-palette($canvas-primary-palette, 'A100'); + $accent-palette-default: mat.get-color-from-palette($accent-palette, default); $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); + $surface: utils.get-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + $on-surface-medium: utils.get-on-surface($canvas-color-config, medium); + $surface-highlight: utils.get-on-surface($canvas-color-config, highlight); + div.operation-control { - box-shadow: 0 1px 6px $canvas-primary-palette-50; - background-color: $canvas-primary-palette-600; - border-top: 1px solid $primary-palette-300; - border-right: 1px solid $primary-palette-300; - border-bottom: 1px solid $primary-palette-300; + box-shadow: 0 1px 6px $canvas-primary-palette-A200; + background-color: $surface; + border: 1px solid $on-surface-lighter; .fa, .icon { - color: $accent-palette-A400; + color: utils.get-color-on-surface($color-config, $surface); } .operation-control-header { &:hover { - background: linear-gradient(90deg, $canvas-primary-palette-A100 252px, $primary-palette-100 34px); + background: linear-gradient(-90deg, $surface-highlight 34px, transparent 35px); } } .operation-control-title { - color: $canvas-primary-palette-A200; + color: $on-surface-medium; } .operation-context-logo { .icon { - color: $warn-palette-A200; + color: $accent-palette-default; } } .operation-context-name { - color: $canvas-primary-palette-A200; + color: $on-surface-medium; } .operation-context-type { - color: $primary-palette-700; + color: utils.get-color-on-surface($color-config, $surface); } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/_header.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/_header.component-theme.scss index e22bb1e8ff98..bd7feb61676d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/_header.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/_header.component-theme.scss @@ -20,11 +20,4 @@ @mixin nifi-theme($canvas-theme) { // Get the color config from the theme. - $canvas-color-config: mat.get-color-config($canvas-theme); - - // Get the color palette from the color-config. - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); - - // Get hues from palette - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/_flow-status.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/_flow-status.component-theme.scss index 85ae5701350b..45fb803ae5e2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/_flow-status.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/flow-status/_flow-status.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -26,42 +27,47 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); $warn-palette: map.get($color-config, 'warn'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $warn-palette-400: mat.get-color-from-palette($warn-palette, 400); - $warn-palette-A400: mat.get-color-from-palette($warn-palette, 'A400'); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); + $primary-palette-darker: mat.get-color-from-palette($primary-palette, darker); + $primary-palette-darker-contrast: mat.get-color-from-palette($primary-palette, darker-contrast); + $warn-palette-default: mat.get-color-from-palette($warn-palette, default); + $warn-palette-default-contrast: mat.get-color-from-palette($warn-palette, default-constrast); + + $surface: utils.get-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); .flow-status { - border-bottom: 1px solid $primary-palette-300; + border-bottom: 1px solid $on-surface-lighter; + background: $surface; .fa, .icon { - color: $primary-palette-500; + color: utils.get-color-on-surface($color-config, $surface); } .warning { - color: $warn-palette-400; + color: $warn-palette-default; } .status-value { - color: $warn-palette-A400; + color: utils.get-color-on-surface($color-config, $surface, 'accent'); } .controller-bulletins { - border-left: 1px solid $primary-palette-300; - background-color: $primary-palette-500; + border-left: 1px solid $on-surface-lighter; + background-color: $primary-palette-darker; } .controller-bulletins.has-bulletins { - background-color: $warn-palette-400; + background-color: $warn-palette-default; + } + .controller-bulletins.has-bulletins .fa { + background-color: $warn-palette-default-contrast; } .controller-bulletins .fa { - color: $canvas-primary-palette-900; + color: $primary-palette-darker-contrast; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/new-canvas-item/_new-canvas-item.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/new-canvas-item/_new-canvas-item.component-theme.scss index eb7e240fd8f0..0f83eb7e74d2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/new-canvas-item/_new-canvas-item.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/new-canvas-item/_new-canvas-item.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -25,33 +26,47 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $accent-palette: map.get($color-config, 'accent'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $primary-palette-default-contrast: mat.get-color-from-palette($primary-palette, default-contrast); + $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $on-surface-highlight: utils.get-on-surface($canvas-color-config, highlight); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + + // Use the same logic as _navigation.component-theme.scss to determine the bg color for hover + $hover-bg: mat.get-color-from-palette($primary-palette, "navbar"); + @if ($hover-bg) { + // Nothing to do here, we have special color from the palette. + } @else { + // There was not a special value set for the navbar, so we use Angular Material behavior. + $hover-bg: mat.get-color-from-palette($primary-palette, "default"); + } .new-canvas-item { .icon { - color: $accent-palette-A400; - &.hovering { - background-color: $primary-palette-100; - box-shadow: 0 3px 6px $canvas-primary-palette-50; + // This solution re-uses the highlight value used throughout the UI, but because we need to hide the non-hover + // version of the icons, we create a double layered gradient with the matching background color of the + // navigation bar, then put the highlight on top of it. + background: linear-gradient($on-surface-highlight, $on-surface-highlight), + linear-gradient($hover-bg, $hover-bg); .component-button-grip { background: repeating-linear-gradient( 90deg, - $primary-palette-300, - $primary-palette-300 4px, - $primary-palette-300 4px, - $primary-palette-300 6px + $on-surface-lighter, + $on-surface-lighter 4px, + transparent 4px, + transparent 6px ); } } + &.cdk-drag-dragging { + color: utils.get-color-on-surface($color-config, $surface-darker); + mix-blend-mode: difference; // Make sure the dragged icon is always visible + } } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/search/_search.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/search/_search.component-theme.scss index 448531fe62a9..66c6bc6e56c6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/search/_search.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/header/search/_search.component-theme.scss @@ -17,43 +17,38 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. - $color-config: mat.get-color-config($material-theme); $canvas-color-config: mat.get-color-config($canvas-theme); - // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); - // Get hues from palette - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); - $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); - $canvas-primary-palette-A700: mat.get-color-from-palette($canvas-primary-palette, 'A700'); - $canvas-primary-palette-contrast-300: mat.get-color-from-palette($canvas-primary-palette, '300-contrast'); + $surface: utils.get-surface($canvas-color-config); + $on-surface: utils.get-on-surface($canvas-color-config); + $on-surface-medium: utils.get-on-surface($canvas-color-config, medium); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); .search-container { - border-left: 1px solid $primary-palette-300; + border-left: 1px solid $on-surface-lighter; &:hover, &.open { - background-color: $canvas-primary-palette-900; - border-left: 1px solid $canvas-primary-palette-A700; + background-color: $surface; + border-left: 1px solid $on-surface-lighter; } .search-input { - color: $canvas-primary-palette-A200; - background-color: $canvas-primary-palette-900; + color: $on-surface; + background-color: $surface; } .fa { - color: $canvas-primary-palette-A700; + color: $on-surface-medium; } } .search-results { - background-color: $canvas-primary-palette-900; + background-color: $surface; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/prioritizers/_prioritizers.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/prioritizers/_prioritizers.component-theme.scss index 67a5fe6ab417..d1045ad11947 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/prioritizers/_prioritizers.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/connection/prioritizers/_prioritizers.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -28,32 +29,32 @@ $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-100: mat.get-color-from-palette($canvas-primary-palette, 100); - $canvas-primary-palette-400: mat.get-color-from-palette($canvas-primary-palette, 400); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, A200); $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); + $surface: utils.get-surface($canvas-color-config); + $on-surface: utils.get-on-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); .prioritizers { .prioritizers-list { - background: $primary-palette-500; + background: $primary-palette-default; - border: solid 1px $canvas-primary-palette-400; + border: solid 1px $canvas-primary-palette-500; .cdk-drag-disabled { - background: $canvas-primary-palette-500; + background: $on-surface-lighter; } } .prioritizer-draggable-item { - border-bottom: solid 1px $canvas-primary-palette-400; - color: $canvas-primary-palette-100; - background: $canvas-primary-palette-900; + border-bottom: solid 1px $canvas-primary-palette-500; + color: $on-surface; + background: $surface; } .cdk-drag-preview { - box-shadow: 0 3px 6px $canvas-primary-palette-50; + box-shadow: 0 3px 6px $canvas-primary-palette-A200; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/_create-process-group.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/_create-process-group.component-theme.scss index 86883fb2fa44..9d455d90e3a1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/_create-process-group.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/process-group/create-process-group/_create-process-group.component-theme.scss @@ -26,13 +26,11 @@ $accent-palette: map.get($color-config, 'accent'); // Get hues from palette - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - - // Get hues from palette + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); .create-process-group-form { .upload-flow-definition { - color: $accent-palette-A400; + color: $accent-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html index 3f7ba9dff232..58d09c9cb38e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/edit-processor/relationship-settings/relationship-settings.component.html @@ -91,8 +91,8 @@ (change)="handleChanged()" [disabled]="isDisabled" class="flex gap-x-2"> - Penalize - Yield + Penalize + Yield
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/_create-remote-process-group.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/_create-remote-process-group.component-theme.scss index 4fa732184006..a361eca8991b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/_create-remote-process-group.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/items/remote-process-group/create-remote-process-group/_create-remote-process-group.component-theme.scss @@ -26,5 +26,5 @@ $accent-palette: map.get($color-config, 'accent'); // Get hues from palette - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/banner/_banner.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/banner/_banner.component-theme.scss index 71e4d7d64fe3..60728ad27d61 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/banner/_banner.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/common/banner/_banner.component-theme.scss @@ -17,20 +17,18 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($theme) { // Get the color config from the theme. $color-config: mat.get-color-config($theme); - // Get the color palette from the color-config. - $warn-palette: map.get($color-config, 'warn'); - // Get hues from palette - $warn-palette-100: mat.get-color-from-palette($warn-palette, 100); + $warn-surface: utils.get-surface($color-config, default, 'warn'); div { &.error { - background-color: $warn-palette-100; + background-color: $warn-surface; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/_controller-services.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/_controller-services.component-theme.scss index 20858b80fbfc..99367449f0ac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/_controller-services.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/_controller-services.component-theme.scss @@ -26,9 +26,9 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .controller-services-header { - color: $primary-palette-500; + color: $primary-palette-default; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss index 30dd724e1685..c46e1179170b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/_manage-remote-ports.component-theme.scss @@ -25,20 +25,20 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $canvas-accent-palette: map.get($canvas-color-config, 'accent'); + $canvas-warn-palette: map.get($canvas-color-config, 'warn'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $canvas-accent-palette-A200: mat.get-color-from-palette($canvas-accent-palette, 'A200'); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $canvas-warn-palette-A200: mat.get-color-from-palette($canvas-warn-palette, 'A200'); .manage-remote-ports-header { - color: $primary-palette-500; + color: $primary-palette-default; } .manage-remote-ports-table { .listing-table { .fa.fa-warning { - color: $canvas-accent-palette-A200; + color: $canvas-warn-palette-A200; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/feature/_login.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/feature/_login.component-theme.scss index f60260c711a2..332603ac609f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/feature/_login.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/feature/_login.component-theme.scss @@ -17,19 +17,15 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { - $color-config: mat.get-color-config($material-theme); $canvas-color-config: mat.get-color-config($canvas-theme); - // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); - // Get hues from palette - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); + $surface: utils.get-surface($canvas-color-config); .login-background { - background: $canvas-primary-palette-900 url(../../../../assets/icons/bg-error.png) left top no-repeat; + background: $surface url(../../../../assets/icons/bg-error.png) left top no-repeat; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/ui/login-form/_login-form.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/ui/login-form/_login-form.component-theme.scss index a657dbc1c00b..68e8b1fdc307 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/ui/login-form/_login-form.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/login/ui/login-form/_login-form.component-theme.scss @@ -26,11 +26,11 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .login-form { .login-title { - color: $primary-palette-500; + color: $primary-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/feature/_parameter-contexts.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/feature/_parameter-contexts.component-theme.scss index 410954f8b066..9438b0407c1f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/feature/_parameter-contexts.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/feature/_parameter-contexts.component-theme.scss @@ -26,9 +26,9 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .parameter-context-header { - color: $primary-palette-500; + color: $primary-palette-default; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss index 0116242f7b5e..49e398f2de04 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -28,31 +29,33 @@ $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-100: mat.get-color-from-palette($canvas-primary-palette, 100); - $canvas-primary-palette-400: mat.get-color-from-palette($canvas-primary-palette, 400); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, A200); $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); + $surface: utils.get-surface($canvas-color-config); + $on-surface: utils.get-on-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + .parameter-context-inheritance { .parameter-context-inheritance-list { - background: $primary-palette-500; + background: $primary-palette-default; - border: solid 1px $canvas-primary-palette-400; + border: solid 1px $canvas-primary-palette-500; .cdk-drag-disabled { - background: $canvas-primary-palette-500; + background: $on-surface-lighter; } } .parameter-context-draggable-item { - border-bottom: solid 1px $canvas-primary-palette-400; - color: $canvas-primary-palette-100; - background: white; + border-bottom: solid 1px $canvas-primary-palette-500; + color: $on-surface; + background: $surface; } .cdk-drag-preview { - box-shadow: 0 3px 6px $canvas-primary-palette-50; + box-shadow: 0 3px 6px $canvas-primary-palette-A200; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/feature/_provenance.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/feature/_provenance.component-theme.scss index 80e82167c641..369411a6a968 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/feature/_provenance.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/feature/_provenance.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -25,40 +26,43 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $warn-palette: map.get($color-config, 'warn'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); + $accent-palette: map.get($color-config, 'accent'); // Get hues from palette - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $canvas-primary-palette-200: mat.get-color-from-palette($canvas-primary-palette, 200); - $canvas-primary-palette-600: mat.get-color-from-palette($canvas-primary-palette, 600); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); - $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); + $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $accent-palette-default: mat.get-color-from-palette($accent-palette, default); + $surface: utils.get-surface($canvas-color-config); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $on-surface: utils.get-on-surface($canvas-color-config); .provenance-header { - color: $primary-palette-500; + color: $primary-palette-default; } rect.lineage { - fill: $canvas-primary-palette-600; + fill: $surface-darker; } g.flowfile-icon text { - fill: $warn-palette-A200; + fill: $accent-palette-default; // Accent palette } circle.event-circle { - fill: $primary-palette-300; - stroke: $canvas-primary-palette-200; + fill: $primary-palette-lighter; + stroke: $on-surface; } path.link { - stroke: $canvas-primary-palette-200; + stroke: $on-surface; + } + + marker { + fill: $on-surface; } circle.flowfile-link { - fill: $canvas-primary-palette-900; - stroke: $canvas-primary-palette-200; + fill: $surface; + stroke: $on-surface; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/_provenance-event-table.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/_provenance-event-table.component-theme.scss index 6a46994c0903..968ddb064a48 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/_provenance-event-table.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/_provenance-event-table.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -24,23 +25,21 @@ $canvas-color-config: mat.get-color-config($canvas-theme); // Get the color palette from the color-config. - $accent-palette: map.get($color-config, 'accent'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-600: mat.get-color-from-palette($canvas-primary-palette, 600); - $canvas-primary-palette-700: mat.get-color-from-palette($canvas-primary-palette, 700); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $canvas-primary-palette-A100: mat.get-color-from-palette($canvas-primary-palette, A100); .provenance-event-table { .lineage { - border: 1px solid $canvas-primary-palette-700; - background-color: $canvas-primary-palette-600; + border: 1px solid $canvas-primary-palette-A100; + background-color: $surface-darker; .lineage-controls { .fa, .icon { - color: $accent-palette-A400; + color: utils.get-color-on-surface($color-config, $surface-darker); } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/_lineage.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/_lineage.component-theme.scss index 58d8249e4380..1dc455bfc438 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/_lineage.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/_lineage.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -26,42 +27,41 @@ // Get the color palette from the color-config. $accent-palette: map.get($color-config, 'accent'); $warn-palette: map.get($color-config, 'warn'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); - $canvas-accent-palette: map.get($canvas-color-config, 'accent'); + $canvas-warn-palette: map.get($canvas-color-config, 'warn'); // Get hues from palette - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $warn-palette-400: mat.get-color-from-palette($warn-palette, 400); - $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); - $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); - $canvas-accent-palette-A200: mat.get-color-from-palette($canvas-accent-palette, 'A200'); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); + $warn-palette-default: mat.get-color-from-palette($warn-palette, default); + $canvas-warn-palette-A200: mat.get-color-from-palette($canvas-warn-palette, 'A200'); + $on-surface: utils.get-on-surface($canvas-color-config); #lineage { canvas, svg { text.event-type { - fill: $canvas-primary-palette-A200; + fill: $on-surface; } path.link.selected { - stroke: $warn-palette-400; + stroke: $warn-palette-default; + fill: $warn-palette-default; } g.event circle.selected { - fill: $warn-palette-400; + fill: $warn-palette-default; } g.event circle.context { - fill: $canvas-accent-palette-A200; + fill: $canvas-warn-palette-A200; } g.flowfile circle.context, g.event circle.context { - stroke: $accent-palette-A400; + stroke: $accent-palette-default; } .flowfile-icon { - color: $warn-palette-A200; + color: $accent-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/lineage.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/lineage.component.ts index 3ef3754b9584..08c46dd4c69c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/lineage.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/lineage/lineage.component.ts @@ -315,7 +315,7 @@ export class LineageComponent implements OnInit { if (d.indexOf('SELECTED') >= 0) { return 'warn-400'; } else { - return 'primary-contrast-200'; + return 'on-surface'; } }) .append('path') diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/_flowfile-dialog.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/_flowfile-dialog.component-theme.scss index 89456e69d260..ba6d348b6cba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/_flowfile-dialog.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-dialog/_flowfile-dialog.component-theme.scss @@ -26,13 +26,13 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .flowfile { .mdc-dialog__content { .tab-content { .flowfile-header { - color: $primary-palette-500; + color: $primary-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/_flowfile-table.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/_flowfile-table.component-theme.scss index b58a34f0abd3..d008dac19a05 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/_flowfile-table.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/_flowfile-table.component-theme.scss @@ -27,16 +27,16 @@ $warn-palette: map.get($color-config, 'warn'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $warn-palette-400: mat.get-color-from-palette($warn-palette, 400); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $warn-palette-default: mat.get-color-from-palette($warn-palette, default); .flowfile-table { .queue-listing-header { - color: $primary-palette-500; + color: $primary-palette-default; } .listing-message { - color: $warn-palette-400; + color: $warn-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/feature/_settings.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/feature/_settings.component-theme.scss index cf13fc636a23..7de25dba480b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/feature/_settings.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/feature/_settings.component-theme.scss @@ -17,18 +17,17 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../node_modules/@angular/material/core/theming/inspection' as inspection; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($theme) { // Get the color config from the theme. $color-config: mat.get-color-config($theme); - // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); - - // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + // Get the surface color from material so we can ensure best contrast + $surface: inspection.get-theme-color($theme, background, background); .settings-header { - color: $primary-palette-500; + color: utils.get-color-on-surface($color-config, $surface); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/_summary.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/_summary.component-theme.scss index 8ebd57666691..6ef8cedadb22 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/_summary.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/feature/_summary.component-theme.scss @@ -17,18 +17,17 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../node_modules/@angular/material/core/theming/inspection' as inspection; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($theme) { // Get the color config from the theme. $color-config: mat.get-color-config($theme); - // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); - - // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + // Get the surface color from material so we can ensure best contrast + $surface: inspection.get-theme-color($theme, background, background); .summary-header { - color: $primary-palette-500; + color: utils.get-color-on-surface($color-config, $surface); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/feature/_users.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/feature/_users.component-theme.scss index ee08282a7f41..ad525d82e016 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/feature/_users.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/feature/_users.component-theme.scss @@ -26,9 +26,9 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .user-header { - color: $primary-palette-500; + color: $primary-palette-default; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss index 10194ed036e4..8bae82e45cfc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/component-context/_component-context.component-theme.scss @@ -26,24 +26,22 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); $accent-palette: map.get($color-config, 'accent'); - $warn-palette: map.get($color-config, 'warn'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-700: mat.get-color-from-palette($primary-palette, 700); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); + $primary-palette-darker: mat.get-color-from-palette($primary-palette, darker); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); .component-context { .fa, .icon { - color: $accent-palette-A400; + color: $accent-palette-default; } .component-context-logo { .icon { - color: $warn-palette-A200; + color: $accent-palette-default; } } @@ -52,11 +50,11 @@ } .component-context-type { - color: $primary-palette-700; + color: $primary-palette-darker; } .component-context-id { - color: $warn-palette-A200; + color: $accent-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/context-menu/_context-menu.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/context-menu/_context-menu.component-theme.scss index c0738622bdd0..9b00cb14b19a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/context-menu/_context-menu.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/context-menu/_context-menu.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -25,42 +26,41 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $accent-palette: map.get($color-config, 'accent'); $canvas-primary-palette: map.get($canvas-color-config, 'primary'); $canvas-accent-palette: map.get($canvas-color-config, 'accent'); // Get hues from palette - $primary-palette-50: mat.get-color-from-palette($primary-palette, 50); - $primary-palette-200: mat.get-color-from-palette($primary-palette, 200); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-300: mat.get-color-from-palette($canvas-primary-palette, 300); + $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); + $on-surface-medium: utils.get-on-surface($canvas-color-config, medium); + $on-surface-highlight: utils.get-on-surface($canvas-color-config, highlight); $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); - $canvas-accent-palette-800: mat.get-color-from-palette($canvas-accent-palette, 800); + $canvas-accent-palette-A700: mat.get-color-from-palette($canvas-accent-palette, 'A700'); div.context-menu { - background-color: $primary-palette-50; - border: 1px solid $accent-palette-A400; - box-shadow: 0 3px 6px $canvas-primary-palette-50; - color: $accent-palette-A400; + background-color: $surface-darker; + border: 1px solid $primary-palette-default; + box-shadow: 0 3px 6px $canvas-primary-palette-A200; + color: utils.get-color-on-surface($color-config, $surface-darker); .context-menu-item { .context-menu-item-text { - color: $canvas-primary-palette-A200; + color: $on-surface-medium; } } .context-menu-item:hover { - background-color: $primary-palette-200; + background-color: $on-surface-highlight; } .context-menu-item:active { - background-color: $canvas-primary-palette-300; + background-color: $canvas-primary-palette-500; } &.show-focused { .context-menu-item:focus { - outline: $canvas-accent-palette-800 solid 1px; + outline: $canvas-accent-palette-A700 solid 1px; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html index b99f2db7f145..963f10decd76 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-parameter-dialog/edit-parameter-dialog.component.html @@ -41,8 +41,8 @@

{{ isNew ? 'Add' : 'Edit' }} Parameter

- Yes - No + Yes + No
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-tenant/edit-tenant-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-tenant/edit-tenant-dialog.component.html index 5ea13dfb8203..638b109b410d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-tenant/edit-tenant-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/edit-tenant/edit-tenant-dialog.component.html @@ -21,8 +21,8 @@

{{ isNew ? 'Add' : 'Edit' }} {{ isUser ? 'User' : 'User Gro
- Individual - Group + Individual + Group
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/extension-creation/_extension-creation.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/extension-creation/_extension-creation.component-theme.scss index 31f828f5225a..fefc8d1e3224 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/extension-creation/_extension-creation.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/extension-creation/_extension-creation.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -25,19 +26,28 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); + $accent-palette: map.get($color-config, 'accent'); $warn-palette: map.get($color-config, 'warn'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); - $canvas-accent-palette: map.get($canvas-color-config, 'accent'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $canvas-primary-palette-800: mat.get-color-from-palette($canvas-primary-palette, 800); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); - $canvas-primary-palette-A700: mat.get-color-from-palette($canvas-primary-palette, 'A700'); - $canvas-accent-palette-A100: mat.get-color-from-palette($canvas-accent-palette, 'A100'); - $warn-palette-400: mat.get-color-from-palette($warn-palette, 400); - $warn-palette-A400: mat.get-color-from-palette($warn-palette, 'A400'); + $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); + $primary-palette-default-contrast: mat.get-color-from-palette($primary-palette, 'default-contrast'); + $primary-palette-darker: mat.get-color-from-palette($primary-palette, 'darker'); + $primary-palette-darker-contrast: mat.get-color-from-palette($primary-palette, 'darker-contrast'); + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); + $warn-palette-default: mat.get-color-from-palette($warn-palette, default); + + $surface-darker: utils.get-surface($canvas-color-config, darker); + $surface: utils.get-surface($canvas-color-config); + $accent-surface: utils.get-surface($color-config, default, 'accent'); + $header-surface: utils.ensure-contrast($primary-palette-default, $surface, $primary-palette-darker); + $header-on-surface: utils.ensure-contrast( + $primary-palette-default-contrast, + $header-surface, + $primary-palette-darker-contrast + ); + $surface-highlight: utils.get-on-surface($canvas-color-config, 'highlight'); + $on-surface-medium: utils.get-on-surface($canvas-color-config, medium); .extension-creation-dialog { @include mat.button-density(-1); @@ -45,38 +55,38 @@ .type-table { table { th { - background-color: $primary-palette-500; - color: $canvas-primary-palette-900; + background-color: $header-surface; + color: $header-on-surface; + + .mat-sort-header-arrow { + color: $header-on-surface; + } } tr:hover { - background-color: $primary-palette-100 !important; + background-color: $surface-highlight !important; } .selected { - background-color: $canvas-accent-palette-A100 !important; + background-color: $accent-surface !important; } .even { - background-color: $canvas-primary-palette-800; + background-color: $surface-darker; } .fa.fa-shield { - color: $warn-palette-400; + color: $warn-palette-default; } } } .selected-type-name { - color: $warn-palette-A400; + color: $accent-palette-default; } .selected-type-bundle { - color: $canvas-primary-palette-A700; + color: $on-surface-medium; } } - - .mat-sort-header-arrow { - color: $canvas-primary-palette-900; - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/navigation/_navigation.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/navigation/_navigation.component-theme.scss index f6b8bc4489fe..db536e4a1824 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/navigation/_navigation.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/navigation/_navigation.component-theme.scss @@ -17,45 +17,56 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../node_modules/@angular/material/core/theming/inspection' as inspection; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. $color-config: mat.get-color-config($material-theme); - $canvas-color-config: mat.get-color-config($canvas-theme); // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $accent-palette: map.get($color-config, 'accent'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); + $navbar-surface: mat.get-color-from-palette($primary-palette, "navbar"); + $navbar-on-surface: mat.get-color-from-palette($primary-palette, "navbar-contrast"); + @if ($navbar-surface) { + // Nothing to do here, we have special colors from the palette. + } @else { + // There was not a special value set for the navbar, so we use Angular Material behavior. + $navbar-surface: mat.get-color-from-palette($primary-palette, "default"); + $navbar-on-surface: mat.get-color-from-palette($primary-palette, "default-contrast"); + } + + // We need this to get the right color value for the global menu items. + $surface: inspection.get-theme-color($material-theme, background, background); .nifi-navigation { - background-color: $primary-palette-300; + background-color: $navbar-surface; - .icon { - color: $accent-palette-A400; + .icon, + .icon.global-menu { + color: $navbar-on-surface; } .current-user { - color: $canvas-primary-palette-A200; + color: $navbar-on-surface; + } + + a { + color: $navbar-on-surface; + text-decoration-color: $navbar-on-surface; } - .global-menu:hover { - background-color: $primary-palette-100; - box-shadow: 0 3px 6px $canvas-primary-palette-50; + a:hover { + opacity: 0.6; } } button.global-menu-item { .fa, .icon { - color: $accent-palette-A400; + color: utils.get-color-on-surface($color-config, $surface); } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html index 345c35c078cd..f4784cf8eb36 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/new-property-dialog/new-property-dialog.component.html @@ -30,8 +30,8 @@

Add Property

- Yes - No + Yes + No
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/_nf-editor.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/_nf-editor.component-theme.scss index 895d4a1a9e57..4a9ae8626f21 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/_nf-editor.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/editors/nf-editor/_nf-editor.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -28,31 +29,30 @@ $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-300: mat.get-color-from-palette($primary-palette, 300); - $canvas-primary-palette-100: mat.get-color-from-palette($canvas-primary-palette, 100); - $canvas-primary-palette-200: mat.get-color-from-palette($canvas-primary-palette, 200); - $canvas-primary-palette-300: mat.get-color-from-palette($canvas-primary-palette, 300); - $canvas-primary-palette-contrast-200: mat.get-color-from-palette($canvas-primary-palette, '200-contrast'); - $canvas-primary-palette-contrast-900: mat.get-color-from-palette($canvas-primary-palette, '900-contrast'); + $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); + $canvas-primary-palette-500-contrast: mat.get-color-from-palette($canvas-primary-palette, 400-contrast); + $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); + $surface: utils.get-surface($canvas-color-config); + $on-surface: utils.get-on-surface($canvas-color-config); .property-editor { @include mat.button-density(-1); .nf-editor { .CodeMirror { - border: 1px solid $canvas-primary-palette-300; - background-color: $canvas-primary-palette-contrast-900; + border: 1px solid $canvas-primary-palette-500; + background-color: $surface; &.blank { - background: $primary-palette-300; - color: $primary-palette-100; - border: 1px solid $primary-palette-300; + background: $primary-palette-default; + color: $primary-palette-lighter; + border: 1px solid $primary-palette-default; } } .CodeMirror-code { - color: $canvas-primary-palette-contrast-200; + color: $on-surface; } /* @@ -60,8 +60,8 @@ */ .cm-s-default .CodeMirror-matchingbracket { - color: $canvas-primary-palette-200 !important; - background-color: $canvas-primary-palette-300; + color: $canvas-primary-palette-500-contrast !important; + background-color: $canvas-primary-palette-500; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/_provenance-event-dialog.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/_provenance-event-dialog.component-theme.scss index 581d629e0e82..40483370a7f2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/_provenance-event-dialog.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/provenance-event-dialog/_provenance-event-dialog.component-theme.scss @@ -26,7 +26,7 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, '500'); + $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); .provenance-event { @include mat.button-density(-1); @@ -34,7 +34,7 @@ .mdc-dialog__content { .tab-content { .event-header { - color: $primary-palette-500; + color: $primary-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss index fdb567981b1e..3f2efdb8b7aa 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss @@ -18,6 +18,7 @@ @use 'sass:math' as math; @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -26,13 +27,12 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); $handle-size: 15px; - $handle-color: $canvas-primary-palette-500; + $handle-color: $on-surface-lighter; .resizable-triangle { border-right: math.div($handle-size, 2) solid $handle-color; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/_status-history.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/_status-history.component-theme.scss index b5a5a541010e..7cae5e2335b2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/_status-history.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/_status-history.component-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -28,14 +29,14 @@ $canvas-primary-palette: map.get($canvas-color-config, 'primary'); // Get hues from palette - $primary-palette-700: mat.get-color-from-palette($primary-palette, 700); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); + $primary-palette-darker: mat.get-color-from-palette($primary-palette, darker); + $surface: utils.get-surface($canvas-color-config); $canvas-primary-palette-A100: mat.get-color-from-palette($canvas-primary-palette, 'A100'); $canvas-primary-palette-A700: mat.get-color-from-palette($canvas-primary-palette, 'A700'); :host ::ng-deep #status-history-chart-container, :host ::ng-deep #status-history-chart-control-container { - background-color: $canvas-primary-palette-900; + background-color: $surface; .axis path, .axis line { @@ -49,6 +50,6 @@ :host ::ng-deep #status-history-chart-container text, #status-history-chart-control-container text { - fill: $primary-palette-700; + fill: $primary-palette-darker; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/_system-diagnostics-dialog.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/_system-diagnostics-dialog.component-theme.scss index 2551707f9429..92f0404835cf 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/_system-diagnostics-dialog.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/system-diagnostics-dialog/_system-diagnostics-dialog.component-theme.scss @@ -26,12 +26,12 @@ $primary-palette: map.get($color-config, 'primary'); // Get hues from palette - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); + $primary-palette-default: mat.get-color-from-palette($primary-palette, default); .system-diagnostics { .tab-content { .section-header { - color: $primary-palette-500; + color: $primary-palette-default; } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/property-hint-tip/_property-hint-tip.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/property-hint-tip/_property-hint-tip.component-theme.scss index 50de6312892c..ebcda7ac7587 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/property-hint-tip/_property-hint-tip.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/property-hint-tip/_property-hint-tip.component-theme.scss @@ -26,9 +26,9 @@ $accent-palette: map.get($color-config, 'accent'); // Get hues from palette - $accent-palette-A700: mat.get-color-from-palette($accent-palette, 'A700'); + $accent-palette-darker: mat.get-color-from-palette($accent-palette, 'darker'); .hint-pattern { - background-color: $accent-palette-A700; + background-color: $accent-palette-darker; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss index bcc5f6a9f679..7ff0b6488fd4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss @@ -17,6 +17,7 @@ @use 'sass:map'; @use '@angular/material' as mat; +@use '../../assets/utils.scss' as utils; @mixin nifi-theme($material-theme, $canvas-theme) { // Get the color config from the theme. @@ -25,17 +26,22 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $accent-palette: map.get($color-config, 'accent'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); - $canvas-accent-palette: map.get($canvas-color-config, 'accent'); // Get hues from palette - $primary-palette-100: mat.get-color-from-palette($primary-palette, 100); - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-800: mat.get-color-from-palette($canvas-primary-palette, 800); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); - $canvas-accent-palette-A100: mat.get-color-from-palette($canvas-accent-palette, 'A100'); + $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); + $primary-palette-default-contrast: mat.get-color-from-palette($primary-palette, 'default-contrast'); + $primary-palette-darker: mat.get-color-from-palette($primary-palette, 'darker'); + $primary-palette-darker-contrast: mat.get-color-from-palette($primary-palette, 'darker-contrast'); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $surface: utils.get-surface($canvas-color-config); + $accent-surface: utils.get-surface($color-config, default, 'accent'); + $header-surface: utils.ensure-contrast($primary-palette-default, $surface, $primary-palette-darker); + $header-on-surface: utils.ensure-contrast( + $primary-palette-default-contrast, + $header-surface, + $primary-palette-darker-contrast + ); + $surface-highlight: utils.get-on-surface($canvas-color-config, 'highlight'); .listing-table { table { @@ -52,32 +58,36 @@ } th { - background-color: $primary-palette-500 !important; - color: $canvas-primary-palette-900; + background-color: $header-surface !important; + color: $header-on-surface; user-select: none; + + .mat-sort-header-arrow { + color: $header-on-surface; + } } tr:hover { - background-color: $primary-palette-100 !important; + background-color: $surface-highlight !important; } .selected { - background-color: $canvas-accent-palette-A100 !important; + background-color: $accent-surface !important; } .even { - background-color: $canvas-primary-palette-800; + background-color: $surface-darker; } .fa { - color: $accent-palette-A400; + color: utils.get-color-on-surface($color-config, $surface); width: 10px; height: 14px; text-align: center; } .icon { - color: $accent-palette-A400; + color: utils.get-color-on-surface($color-config, $surface); width: 10px; text-align: center; } @@ -100,10 +110,6 @@ } } - .mat-sort-header-arrow { - color: $canvas-primary-palette-900; - } - .mat-sort-header-content { overflow: hidden; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi-canvas.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi-canvas.scss new file mode 100644 index 000000000000..32dae17a2eda --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi-canvas.scss @@ -0,0 +1,186 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Custom Colors following Material Design +// For more information: https://m2.material.io/design/color/the-color-system.html +@use '@angular/material' as mat; + + + +// The $nifi-canvas-light-palette defines the PRIMARY palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi +$nifi-canvas-light-palette: ( + // mat.define-palette($nifi-canvas-light-palette) + 50: #ffffff, // circle.flowfile-link, .processor-read-write-stats, .process-group-stats-in-out, .tooltip, .property-editor, .disabled, .enabled, .stopped, .running, .has-errors, .invalid, .validating, .transmitting, .not-transmitting, .up-to-date, .locally-modified, .sync-failure, .stale, .locally-modified-and-stale, g.component rect.body, text.bulletin-icon, rect.processor-icon-container, circle.restricted-background, circle.is-primary-background, g.connection rect.body, text.connection-to-run-status, text.expiration-icon, text.load-balance-icon, text.penalized-icon, g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down, text.version-control, .breadcrumb-container, #birdseye, .controller-bulletins .fa, .search-container:hover, .search-container.open, .login-background, table th, .mat-sort-header-arrow, .CodeMirror, #status-history-chart-container, #status-history-chart-control-container, #status-history-chart-control-container, + 100: #f9fafb, // .canvas-background, .navigation-control, .operation-control, .lineage + 200: #f4f6f7, // .tooltip, .cm-s-default .CodeMirror-matchingbracket, circle.flowfile-link + 300: #e3e8eb, // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats + 400: #d8d8d8, // g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .cdk-drag-disabled, .resizable-triangle + 500: #acacac, // .unset, .border.ghost, .backpressure-tick.not-configured, g.connection.ghost path.connection-path, g.connection.ghost rect.connection-label, .prioritizers-list, .prioritizer-draggable-item, .parameter-context-inheritance-list, .parameter-context-draggable-item + 600: #666666, // UNUSED + 700: #444444, + 800: #262626, + 900: #000000, + + // some analog colors for headers and hover states, inputs, stats, etc + A100: #e5ebed, // .tooltip, .property-editor, g.component rect.border, .component-comments, g.connection path.connection-path, g.connection rect.backpressure-tick.data-size-prediction, g.connection rect.backpressure-tick.object-prediction, g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .breadcrumb-container, .navigation-control, .operation-control, header.nifi-header, .new-canvas-item.icon.hovering, .cdk-drag-disabled, .cdk-drag-preview, .context-menu, .global-menu:hover, .unauthorized background in light themes + A200: rgba(0, 0, 0, 0.25), // .operation-context-name, text.stats-label, text.processor-name, text.port-name, text.process-group-name, text.remote-process-group-name, .navigation-control-title, .operation-control-title, .operation-context-name, .search-input, .context-menu-item-text, .current-user + A400: rgba(#303030, .75), // rect.component-selection, rect.drag-selection, rect.label-drag + A700: rgba(#000000, 0.87), // .unauthorized background in dark themes + // These are the $nifi-canvas-light-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. + // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(#000000, 0.87) or white. These values are particularly important + // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. + // + // NOTE: Care should be taken here to ensure the values meet accessibility standards. + // + // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-light-palette); + // Since 500 is the default the contrast-500 will be used as the default text color. + contrast: ( + 50: rgba(#000000, 0.87), // This is used for the basis of $on-surface for light themes + 100: rgba(#000000, 0.87), + 200: rgba(#000000, 0.87), + 300: rgba(#000000, 0.87), + 400: rgba(#000000, 0.87), + 500: rgba(#000000, 0.87), + 600: #ffffff, + 700: #ffffff, + 800: #ffffff, + 900: #ffffff, // This is used as the basis for $on-surface for dark themes + A100: rgba(#000000, 0.87), + A200: rgba(#000000, 0.87), + A400: #ffffff, + A700: #ffffff, + ) +); + +// The $nifi-canvas-accent-light-palette defines the ACCENT palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi +$nifi-canvas-accent-light-palette: ( + 50: #e5ebed, // Used for the grid lines on the canvas. Called as the canvas access $surface value. + 100: #c3e8d0, + 200: #9dd9b2, // .running "lighter" hue + 300: #73ca94, + 400: #52bf7e, // .up-to-date,.backpressure-percent "default" hue + 500: #00ff00, //.connectable-destination, .connector.connectable This is an exception to main colors and is explicitly called. + 600: #2cb367, + 700: #1A9964, + 800: #016131, // .version-control, .bulletin-normal "darker" hue for this palette + 900: #0d1411, + + A100: #bfbfff, //.selected + A200: #8080ff, // + A400: #4040ff, //.backpressure-percent-warning, .bulletin-warn, .backpressure-percent.warning, text.run-status-icon + A700: #0000ff, //g.connection.selected rect.border, .connection-selection-path, .midpoint, .labelpoint + + contrast: ( + 50: rgba(#000000, 0.87), + 100: rgba(#000000, 0.87), + 200: rgba(#000000, 0.87), + 300: rgba(#000000, 0.87), + 400: rgba(#000000, 0.87), + 500: rgba(#000000, 0.87), + 600: #ffffff, + 700: #ffffff, + 800: #ffffff, + 900: #ffffff, + A100: rgba(#000000, 0.87), + A200: rgba(#000000, 0.87), + A400: rgba(#000000, 0.87), + A700: rgba(#000000, 0.87), + ) +); + +// The $nifi-canvas-warn-light-palette defines the WARN palette both for all Angular Material components used throughout Apache NiFi and for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi +$nifi-canvas-warn-light-palette: ( + // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #f64e4c + 50: #ffebee, + 100: #ffccd2, // "lighter" hue for this palette. Also .banner-error + 200: #f49999, //.stopped + 300: #eb7071, //.stale + 400: #f64e4c, // color="primary" Default hue for this palette. Also .unauthorized, .restricted, .connection-path.full, .connection-path.unauthorized, .backpressure-percent.error, .controller-bulletins.has-bulletins, .link.selected, circle.selected, .listing-message, .fa-shield + 500: #fa3b30, + 600: #ec3030, //.bulletin-error + 700: #ff1507, + 800: #ff0000, //.connector, .startpoint + 900: #ba554a, // "darker" hue for this palette .bulletin-background + + // A100 - A700 are the COMPLEMENTARY colors defined by the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b which is the primary color of the $material-primary-light-palette. + // These color are used for label values, stats, timestamps, counts, etc. + A100: #ffef85, //.selected + A200: #f8bf47, //.invalid, .is-missing-port, circle.context + A400: #bda500, //.backpressure-percent-warning, .bulletin-warn, .backpressure-percent.warning, text.run-status-icon + A700: #ffcc00, //g.connection.selected rect.border, .connection-selection-path, .midpoint, .labelpoint + + contrast: ( + 50: rgba(#000000, 0.87), + 100: rgba(#000000, 0.87), + 200: rgba(#000000, 0.87), + 300: rgba(#000000, 0.87), + 400: rgba(#000000, 0.87), + 500: #ffffff, + 600: #ffffff, + 700: #ffffff, + 800: #ffffff, + 900: #ffffff, + A100: rgba(#000000, 0.87), + A200: #ffffff, + A400: #ffffff, + A700: #ffffff, + ) +); + +// The $nifi-canvas-dark-palette defines the PRIMARY palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi +$nifi-canvas-dark-palette: $nifi-canvas-light-palette; + +// The $nifi-canvas-dark-palette defines the ACCENT palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi +$nifi-canvas-accent-dark-palette: $nifi-canvas-accent-light-palette; + +// The $nifi-canvas-warn-dark-palette defines the WARN palette both for all Angular Material components used throughout Apache NiFi and for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi +$nifi-canvas-warn-dark-palette: $nifi-canvas-warn-light-palette; + +// Define the palettes for the canvas theme +$nifi-canvas-primary-light: mat.define-palette($nifi-canvas-light-palette); +$nifi-canvas-accent-light: mat.define-palette($nifi-canvas-accent-light-palette, 400, 200, 800); +$nifi-canvas-warn-light: mat.define-palette($nifi-canvas-warn-light-palette, 400, 100, 900); + +// Create the theme objects. We can extract the values we need from the theme passed into the mixins. +$nifi-canvas-theme-light: mat.define-light-theme( + ( + color: ( + primary: $nifi-canvas-primary-light, + accent: $nifi-canvas-accent-light, + warn: $nifi-canvas-warn-light + ), + //typography: mat.define-typography-config(), // TODO: typography + density: -3 + ) +); + +// Create the color palettes used in our dark canvas theme. +$nifi-canvas-primary-dark: mat.define-palette($nifi-canvas-dark-palette); +$nifi-canvas-accent-dark: mat.define-palette($nifi-canvas-accent-dark-palette, 400, 200, 800); +$nifi-canvas-warn-dark: mat.define-palette($nifi-canvas-warn-dark-palette, 600, 200, 800); + +$nifi-canvas-theme-dark: mat.define-dark-theme( + ( + color: ( + primary: $nifi-canvas-primary-dark, + accent: $nifi-canvas-accent-dark, + warn: $nifi-canvas-warn-dark, + ), + //typography: mat.define-typography-config(), // TODO: typography + density: -3 + ) +); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss index 34bf7d8bcc69..d1b925e53859 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss @@ -17,13 +17,18 @@ // Custom Colors following Material Design // For more information: https://m2.material.io/design/color/the-color-system.html +@use 'sass:map'; @use '@angular/material' as mat; +// Define some variables that are re-used throughout the theme. +$on-surface-dark: rgba(#000000, .87); +$on-surface-light: #ffffff; + // The $material-primary-light-palette define the PRIMARY AND ACCENT palettes for all Angular Material components used throughout Apache NiFi $material-primary-light-palette: ( // 50 -> 900 are the PRIMARY colors (mat.define-palette($material-primary-light-palette, 300);) defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b - 50: rgba(249, 250, 251, 0.97), // .context-menu - 100: rgba(233, 239, 243, 1), // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, + 50: rgba(#F9FAFB, 0.97), + 100: #e5ebed, // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, 200: #cbd8dd, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a, button.nifi-button, button.nifi-button:disabled 300: #abbdc5, // .breadcrumb-container, .navigation-control, .operation-control, .flow-status, .controller-bulletins, .component-button-grip, .search-container, .nifi-navigation, .CodeMirror.blank 400: #8aa2ad, // Default hue for this palette (color="primary"). @@ -35,11 +40,10 @@ $material-primary-light-palette: ( // A100 -> A700 are the ACCENT color (mat.define-palette($material-primary-light-palette, A400, A100, A700);). These color are the ANALOGOUS (or possibly the TRIADIC??) colors as defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b // These colors are also used by some custom canvas components that display the ANALOGOUS color for things like buttons, links, borders, info, etc. - A100: #aabec7, // .zero - A200: #44a3cf, // .enabled, .transmitting, .load-balance-icon-active - A400: #004849, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu - A700: rgba(20, 145, 193, 0.12), // .hint-pattern - + A100: rgba(#1491C1, 0.12), // .hint-pattern + A200: #aabec7, // .zero + A400: #44a3cf, // .enabled, .transmitting, .load-balance-icon-active + A700: #004849, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu // These are the $material-primary-light-palette PRIMARY AND ACCENT contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. @@ -49,252 +53,53 @@ $material-primary-light-palette: ( // NOTE: When creating the Material palette definition mat.define-palette($material-primary-light-palette, 300); // Since 300, was set as the default the contrast-300 will be used as the default text color. contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: rgba(black, 0.87), - A400: #ffffff, - A700: #ffffff, - ) -); - -// The $material-primary-dark-palette define the PRIMARY AND ACCENT palettes for all Angular Material components used throughout Apache NiFi -$material-primary-dark-palette: ( - // 50 -> 900 are the PRIMARY colors (mat.define-palette($material-primary-dark-palette, 300);) defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b - 50: rgb(30, 45, 54), // .context-menu - 100: rgba(32, 47, 54, 1), // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, - 200: #30444d, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a, button.nifi-button, button.nifi-button:disabled - 300: #3e5762, // .breadcrumb-container, .navigation-control, .operation-control, .flow-status, .controller-bulletins, .component-button-grip, .search-container, .nifi-navigation, .CodeMirror.blank - 400: #4d6b78, // Default hue for this palette (color="primary"). - 500: #587a89, // .disabled, .not-transmitting, .splash, .access-policies-header, .operation-context-type, .bulletin-board-header, .counter-header, .stats-info, .active-thread-count-icon, .processor-type, .port-transmission-icon, .operation-context-type, .flow-status.fa, .flow-status.icon, .controller-bulletins, .prioritizers-list, .controller-services-header, .login-title, .parameter-context-header, .parameter-context-inheritance-list, .provenance-header, .flowfile-header, .queue-listing-header, .settings-header, .summary-header, .user-header, table th, button.global-menu-item.fa, button.global-menu-item.icon, .event-header, .section-header, - 600: #718d9a, // .breadcrumb-container, .birdseye-brush - 700: #8aa2ad, // "darker" hue for this palette. Also #status-history-chart-container text, #status-history-chart-control-container text - 800: #abbcc5, - 900: #abbcc5, - - // A100 -> A700 are the ACCENT color (mat.define-palette($material-primary-dark-palette, A400, A100, A700);). These color are the ANALOGOUS (or possibly the TRIADIC??) colors as defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b - // These colors are also used by some custom canvas components that display the ANALOGOUS color for things like buttons, links, borders, info, etc. - A100: #aabec7, // .zero - A200: #44a3cf, // .enabled, .transmitting, .load-balance-icon-active - A400: #009b9d, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu - A700: #2cd5d5,//rgba(139, 208, 229, 1),//#aabec7 // .hint-pattern - - // These are the $material-primary-dark-palette PRIMARY AND ACCENT contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($material-primary-dark-palette, 300); - // Since 300, was set as the default the contrast-300 will be used as the default text color. - contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); - -// The $nifi-canvas-light-palette defines the PRIMARY palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-light-palette: ( - // mat.define-palette($nifi-canvas-light-palette) - 50: rgba(0, 0, 0, 0.25), // .tooltip, .property-editor, g.component rect.border, .component-comments, g.connection path.connection-path, g.connection rect.backpressure-tick.data-size-prediction, g.connection rect.backpressure-tick.object-prediction, g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .breadcrumb-container, .navigation-control, .operation-control, header.nifi-header, .new-canvas-item.icon.hovering, .cdk-drag-disabled, .cdk-drag-preview, .context-menu, .global-menu:hover, - 100: rgba(black, 0.87), // .prioritizer-draggable-item, .parameter-context-draggable-item - 200: #000, // .tooltip, .cm-s-default .CodeMirror-matchingbracket, circle.flowfile-link - 300: #aaaaaa, // .context-menu-item:active, .CodeMirror, .cm-s-default .CodeMirror-matchingbracket - 400: #acacac, // .unset, .border.ghost, .backpressure-tick.not-configured, g.connection.ghost path.connection-path, g.connection.ghost rect.connection-label, .prioritizers-list, .prioritizer-draggable-item, .parameter-context-inheritance-list, .parameter-context-draggable-item - 500: #d8d8d8, // g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .cdk-drag-disabled, .resizable-triangle - 600: #f9fafb, // .canvas-background, .navigation-control, .operation-control, .lineage - 700: #e5ebed, // .canvas-background, g.component rect.body.unauthorized, g.component rect.processor-icon-container.unauthorized, g.connection rect.body.unauthorized, #birdseye, .lineage - 800: #f4f6f7, // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats - 900: rgba(255, 255, 255, 1), // circle.flowfile-link, .processor-read-write-stats, .process-group-stats-in-out, .tooltip, .property-editor, .disabled, .enabled, .stopped, .running, .has-errors, .invalid, .validating, .transmitting, .not-transmitting, .up-to-date, .locally-modified, .sync-failure, .stale, .locally-modified-and-stale, g.component rect.body, text.bulletin-icon, rect.processor-icon-container, circle.restricted-background, circle.is-primary-background, g.connection rect.body, text.connection-to-run-status, text.expiration-icon, text.load-balance-icon, text.penalized-icon, g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down, text.version-control, .breadcrumb-container, #birdseye, .controller-bulletins .fa, .search-container:hover, .search-container.open, .login-background, table th, .mat-sort-header-arrow, .CodeMirror, #status-history-chart-container, #status-history-chart-control-container, #status-history-chart-control-container, - - // some analog colors for headers and hover states, inputs, stats, etc - A100: rgba(227, 232, 235, 0), // .navigation-control-header:hover, .operation-control-header:hover, .axis path, .axis line - A200: #262626, // .operation-context-name, text.stats-label, text.processor-name, text.port-name, text.process-group-name, text.remote-process-group-name, .navigation-control-title, .operation-control-title, .operation-context-name, .search-input, .context-menu-item-text, .current-user - A400: #444, // rect.component-selection, rect.drag-selection, rect.label-drag - A700: #666, // text.processor-bundle, .search-container:hover, .search-container.open, .search-container.fa, .selected-type-bundle, .brush .selection - - // These are the $nifi-canvas-light-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-light-palette); - // Since 500 is the default the contrast-500 will be used as the default text color. - contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: rgba(black, 0.87), - 400: #ffffff, - 500: #ffffff, - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: rgba(black, 0.87), - A400: #ffffff, - A700: #ffffff, + 50: $on-surface-dark, + 100: $on-surface-dark, + 200: $on-surface-dark, + 300: $on-surface-dark, + 400: $on-surface-dark, + 500: $on-surface-dark, + 600: $on-surface-light, + 700: $on-surface-light, + 800: $on-surface-light, + 900: $on-surface-light, + A100: $on-surface-dark, + A200: $on-surface-dark, + A400: $on-surface-dark, + A700: $on-surface-light, ) ); -// The $nifi-canvas-dark-palette defines the PRIMARY palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-dark-palette: ( - // mat.define-palette($nifi-canvas-dark-palette) - 50: rgba(255, 255, 255, 1), // .tooltip, .property-editor, g.component rect.border, .component-comments, g.connection path.connection-path, g.connection rect.backpressure-tick.data-size-prediction, g.connection rect.backpressure-tick.object-prediction, g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .breadcrumb-container, .navigation-control, .operation-control, header.nifi-header, .new-canvas-item.icon.hovering, .cdk-drag-disabled, .cdk-drag-preview, .context-menu, .global-menu:hover, - 100: #f4f6f7, //rgba(black, 0.87), // .prioritizer-draggable-item, .parameter-context-draggable-item - 200: #e5ebed, // .tooltip, .cm-s-default .CodeMirror-matchingbracket, circle.flowfile-link - 300: #f9fafb, // .context-menu-item:active, .CodeMirror, .cm-s-default .CodeMirror-matchingbracket - 400: #d8d8d8, // .unset, .border.ghost, .backpressure-tick.not-configured, g.connection.ghost path.connection-path, g.connection.ghost rect.connection-label, .prioritizers-list, .prioritizer-draggable-item, .parameter-context-inheritance-list, .parameter-context-draggable-item - 500: #acacac, // g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .cdk-drag-disabled, .resizable-triangle - 600: #545454, // .canvas-background, .navigation-control, .operation-control, .lineage - 700: #696060, // .canvas-background, g.component rect.body.unauthorized, g.component rect.processor-icon-container.unauthorized, g.connection rect.body.unauthorized, #birdseye, .lineage - 800: rgba(#6b6464, 1), // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats - 900: rgba(#252424, 0.97), // circle.flowfile-link, .processor-read-write-stats, .process-group-stats-in-out, .tooltip, .property-editor, .disabled, .enabled, .stopped, .running, .has-errors, .invalid, .validating, .transmitting, .not-transmitting, .up-to-date, .locally-modified, .sync-failure, .stale, .locally-modified-and-stale, g.component rect.body, text.bulletin-icon, rect.processor-icon-container, circle.restricted-background, circle.is-primary-background, g.connection rect.body, text.connection-to-run-status, text.expiration-icon, text.load-balance-icon, text.penalized-icon, g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down, text.version-control, .breadcrumb-container, #birdseye, .controller-bulletins .fa, .search-container:hover, .search-container.open, .login-background, table th, .mat-sort-header-arrow, .CodeMirror, #status-history-chart-container, #status-history-chart-control-container, #status-history-chart-control-container, - - // some analog colors for headers and hover states, inputs, stats, etc - A100: #000, // .navigation-control-header:hover, .operation-control-header:hover, .axis path, .axis line - A200: #e7e6e6, // .operation-context-name, text.stats-label, text.processor-name, text.port-name, text.process-group-name, text.remote-process-group-name, .navigation-control-title, .operation-control-title, .operation-context-name, .search-input, .context-menu-item-text, .current-user - A400: #b0b0b0, // rect.component-selection, rect.drag-selection, rect.label-drag - A700: #b2b2b2, // text.processor-bundle, .search-container:hover, .search-container.open, .search-container.fa, .selected-type-bundle, .brush .selection - - // These are the $nifi-canvas-dark-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-dark-palette); - // Since 500 is the default the contrast-500 will be used as the default text color. +$material-accent-light-palette: ( + 50: #fff8cc, + 100: #ebe2be, + 200: #d7cbb0, // lighter + 300: #c4b5a2, + 400: #b09e94, + 500: #9c8886, // default + 600: #8f7775, + 700: #836563, + 800: #765452, // darker + 900: #4a3435, + A100: #e7ebbe, + A200: #d7d7b0, + A400: #b0ab94, + A700: #020202, contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); - -// The $nifi-canvas-light-palette defines the ACCENT palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-accent-light-palette: ( - // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #52bf7e - 50: #e6f6ec, - 100: #c3e8d0, // "lighter" hue for this palette. - 200: #9dd9b2, //.running - 300: #73ca94, //.backpressure-percent - 400: #52bf7e, // color="primary" Default hue for this palette. Also .up-to-date - 500: #2cb367, - 600: #1A9964, //.version-control - 700: #016131, // "darker" hue for this palette Also .bulletin-normal - 800: #0000ff, //.endpoint, g.process-group.drop rect.border - 900: #00ff00, //.connectable-destination, .connector.connectable - - // A100 - A700 are the ANALOGOUS colors but are more customized. These colors are used to highlight, warn, denote midpoints and labelpoints, etc - A100: #ffef85, //.selected - A200: #f8bf47, //.invalid, .is-missing-port, circle.context - A400: #bda500, //.backpressure-percent-warning, .bulletin-warn, .backpressure-percent.warning, text.run-status-icon - A700: #ffcc00, //g.connection.selected rect.border, .connection-selection-path, .midpoint, .labelpoint - - // These are the $nifi-canvas-accent-light-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-accent-light-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. - contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: rgba(black, 0.87), - 400: rgba(black, 0.87), - 500: rgba(black, 0.87), - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: rgba(black, 0.87), - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); - -// The $nifi-canvas-dark-palette defines the ACCENT palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-accent-dark-palette: ( - // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #52bf7e - 50: #016131, - 100: #1A9964, // "lighter" hue for this palette. - 200: #2cb367, //.running - 300: #52bf7e, //.backpressure-percent - 400: #73ca94, // color="primary" Default hue for this palette. Also .up-to-date - 500: #9dd9b2, - 600: #c3e8d0, //.version-control - 700: #e6f6ec, // "darker" hue for this palette Also .bulletin-normal - 800: #0000ff, //.endpoint, g.process-group.drop rect.border - 900: #00ff00, //.connectable-destination, .connector.connectable - - // A100 - A700 are the ANALOGOUS colors but are more customized. These colors are used to highlight, warn, denote midpoints and labelpoints, etc - A100: #cbaa09, //.selected - A200: #bda500, //.invalid, .is-missing-port, circle.context - A400: #f8bf47, //.backpressure-percent-warning, .bulletin-warn, .backpressure-percent.warning, text.run-status-icon - A700: #948b4b, //g.connection.selected rect.border, .connection-selection-path, .midpoint, .labelpoint - - // These are the $nifi-canvas-accent-dark-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angu - // lar Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-accent-dark-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. - contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), + 50: $on-surface-dark, + 100: $on-surface-dark, + 200: $on-surface-dark, + 300: $on-surface-dark, + 400: $on-surface-dark, + 500: $on-surface-dark, + 600: $on-surface-dark, + 700: $on-surface-light, + 800: $on-surface-light, + 900: $on-surface-light, + A100: $on-surface-dark, + A200: $on-surface-dark, + A400: $on-surface-dark, + A700: $on-surface-dark, ) ); @@ -302,102 +107,54 @@ $nifi-canvas-accent-dark-palette: ( $warn-light-palette: ( // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #f64e4c 50: #ffebee, - 100: #ffccd2, // "lighter" hue for this palette. Also .banner-error - 200: #f49999, //.stopped - 300: #eb7071, //.stale - 400: #f64e4c, // color="primary" Default hue for this palette. Also .unauthorized, .bulletin-background, .restricted, .connection-path.full, .connection-path.unauthorized, .backpressure-percent.error, .controller-bulletins.has-bulletins, .link.selected, circle.selected, .listing-message, .fa-shield + 100: #ffccd2, + 200: #f49999, // "lighter" hue for this palette..stopped + 300: #f57472, + 400: #f64e4c, // color="primary" Default hue for this palette. Also .unauthorized, .restricted, .connection-path.full, .connection-path.unauthorized, .backpressure-percent.error, .controller-bulletins.has-bulletins, .link.selected, circle.selected, .listing-message, .fa-shield, .stale 500: #fa3b30, - 600: #ec3030, //.bulletin-error - 700: #ff1507, // "darker" hue for this palette - 800: #ff0000, //.connector, .startpoint - 900: #f10000, - - // A100 - A700 are the COMPLEMENTARY colors defined by the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b which is the primary color of the $material-primary-light-palette. - // These color are used for label values, stats, timestamps, counts, etc. - A100: #d0c3c2, //.sync-failure - A200: #9c8886, // .operation-context-logo, .funnel-icon, .port-icon, .flowfile-icon - A400: #765452, // .value, .refresh-timestamp, .stats-value, .active-thread-count, .process-group-contents-count, .operation-context-id, .selected-type-name - A700: #4a3435, // .version-control - - // These are the $nifi-canvas-accent-light-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($warn-light-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. + 600: #ff1507, // .bulletin-error + 700: #ff0000, //.connector, .startpoint + 800: #ec3030, + 900: #ba554a, // "darker" hue for this palette .bulletin-background + A100: #ffef85, //.sync-failure + A200: #f8bf47, //.invalid, .is-missing-port, circle.context + A400: #f29833, // .value, .refresh-timestamp, .stats-value, .active-thread-count, .process-group-contents-count, .operation-context-id, .selected-type-name + A700: #eb711e, // .version-control contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: rgba(black, 0.87), - 400: rgba(black, 0.87), - 500: #ffffff, - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: #ffffff, - A400: #ffffff, - A700: #ffffff, + 50: $on-surface-dark, + 100: $on-surface-dark, + 200: $on-surface-dark, + 300: $on-surface-dark, + 400: $on-surface-dark, + 500: $on-surface-dark, + 600: $on-surface-dark, + 700: $on-surface-dark, + 800: $on-surface-dark, + 900: $on-surface-light, + A100: $on-surface-dark, + A200: $on-surface-dark, + A400: $on-surface-dark, + A700: $on-surface-dark, ) ); -// The $warn-dark-palette defines the WARN palette both for all Angular Material components used throughout Apache NiFi and for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$warn-dark-palette: ( - // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #f64e4c - 50: #ffebee, - 100: #ffccd2, // "lighter" hue for this palette. Also .banner-error - 200: #f49999, //.stopped - 300: #eb7071, //.stale - 400: #f64e4c, // color="primary" Default hue for this palette. Also .unauthorized, .bulletin-background, .restricted, .connection-path.full, .connection-path.unauthorized, .backpressure-percent.error, .controller-bulletins.has-bulletins, .link.selected, circle.selected, .listing-message, .fa-shield - 500: #fa3b30, - 600: #ec3030, //.bulletin-error - 700: #ff1507, // "darker" hue for this palette - 800: #ff0000, //.connector, .startpoint - 900: #f10000, +// Dark and light palettes are identical for NiFi, their "default", "lighter", and "darker" values are different and set below. +$material-primary-dark-palette: $material-primary-light-palette; +$material-accent-dark-palette: $material-accent-light-palette; +$warn-dark-palette: $warn-light-palette; - // A100 - A700 are the COMPLEMENTARY colors defined by the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b which is the primary color of the $material-primary-dark-palette. - // These color are used for label values, stats, timestamps, counts, etc. - A100: #ab777a, //.sync-failure - A200: #b98481, // .operation-context-logo, .funnel-icon, .port-icon, .flowfile-icon - A400: #d5bab7, // .value, .refresh-timestamp, .stats-value, .active-thread-count, .process-group-contents-count, .operation-context-id, .selected-type-name - A700: #b6abaa, // .version-control +// Append a special case to the light palette to preserve the legacy styling of NiFi's navigation bar. +$material-primary-light-palette: map.set($material-primary-light-palette, 'navbar', #aabdc5); - // These are the $nifi-canvas-accent-dark-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($warn-dark-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. - contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: rgba(black, 0.87), - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); +// We need to pull out the contrast palette and add the contrast to it, then restore it. +$contrasts: map.get($material-primary-light-palette, 'contrast'); +$contrasts: map.set($contrasts, 'navbar', #004849); +$material-primary-light-palette: map.set($material-primary-light-palette, 'contrast', $contrasts); // Define the palettes for your theme -$material-primary-light: mat.define-palette($material-primary-light-palette, 600, 100, 900); -$material-accent-light: mat.define-palette($material-primary-light-palette, A400, A100, A700); -$nifi-canvas-primary-light: mat.define-palette($nifi-canvas-light-palette); -$nifi-canvas-accent-light: mat.define-palette($nifi-canvas-accent-light-palette, 400, 100, 700); -$warn-light: mat.define-palette($warn-light-palette, 400, 100, 700); +$material-primary-light: mat.define-palette($material-primary-light-palette, 600, 300, 700); +$material-accent-light: mat.define-palette($material-accent-light-palette, 500, 200, 800); +$warn-light: mat.define-palette($warn-light-palette, 400, 200, 900); // Create the theme objects. We can extract the values we need from the theme passed into the mixins. $material-theme-light: mat.define-light-theme( @@ -412,24 +169,10 @@ $material-theme-light: mat.define-light-theme( ) ); -$nifi-canvas-theme-light: mat.define-light-theme( - ( - color: ( - primary: $nifi-canvas-primary-light, - accent: $nifi-canvas-accent-light, - warn: $warn-light - ), - //typography: mat.define-typography-config(), // TODO: typography - density: -3 - ) -); - // Create the color palettes used in our dark theme. -$material-primary-dark: mat.define-palette($material-primary-dark-palette); -$material-accent-dark: mat.define-palette($material-primary-dark-palette, A400, A100, A700); -$nifi-canvas-primary-dark: mat.define-palette($nifi-canvas-dark-palette); -$nifi-canvas-accent-dark: mat.define-palette($nifi-canvas-accent-dark-palette); -$warn-dark: mat.define-palette($warn-dark-palette, 600, 200, 800); +$material-primary-dark: mat.define-palette($material-primary-dark-palette, 400, 200, 600); +$material-accent-dark: mat.define-palette($material-accent-dark-palette, 400, 300, 700); +$warn-dark: mat.define-palette($warn-dark-palette, 500, 200, 700); $material-theme-dark: mat.define-dark-theme( ( @@ -442,15 +185,3 @@ $material-theme-dark: mat.define-dark-theme( typography: mat.define-typography-config(), ) ); - -$nifi-canvas-theme-dark: mat.define-dark-theme( - ( - color: ( - primary: $nifi-canvas-primary-dark, - accent: $nifi-canvas-accent-dark, - warn: $warn-dark, - ), - //typography: mat.define-typography-config(), // TODO: typography - density: -3 - ) -); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss index 40eb88e23a12..075c973468ad 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss @@ -17,14 +17,18 @@ @use '@angular/material' as mat; +// Define some variables that are re-used throughout the theme. +$on-surface-dark: rgba(black, 0.87); +$on-surface-light: #ffffff; + // Custom Colors following Material Design // For more information: https://m2.material.io/design/color/the-color-system.html // The $material-primary-light-palette define the PRIMARY AND ACCENT palettes for all Angular Material components used throughout Apache NiFi $material-primary-light-palette: ( // 50 -> 900 are the PRIMARY colors (mat.define-palette($material-primary-light-palette, 300);) defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b - 50: rgba(240, 231, 242, 1), // .context-menu - 100: rgba(218, 195, 224, 1), // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, + 50: #f0e7f2, // .context-menu + 100: #dac3e0, // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, 200: #c29dcc, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a, button.nifi-button, button.nifi-button:disabled 300: #aa79b7, // .breadcrumb-container, .navigation-control, .operation-control, .flow-status, .controller-bulletins, .component-button-grip, .search-container, .nifi-navigation, .CodeMirror.blank 400: #985fa7, // Default hue for this palette (color="primary"). @@ -36,10 +40,10 @@ $material-primary-light-palette: ( // A100 -> A700 are the ACCENT color (mat.define-palette($material-primary-light-palette, A400, A100, A700);). These color are the ANALOGOUS (or possibly the TRIADIC??) colors as defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b // These colors are also used by some custom canvas components that display the ANALOGOUS color for things like buttons, links, borders, info, etc. - A100: #aabec7, // .zero - A200: #44a3cf, // .enabled, .transmitting, .load-balance-icon-active - A400: #004849, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu - A700: rgba(20, 145, 193, 0.12), // .hint-pattern + A100: rgba(20, 145, 193, 0.12), // .hint-pattern + A200: #aabec7, // .zero + A400: #44a3cf, // .enabled, .transmitting, .load-balance-icon-active + A700: #004849, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu // These are the $material-primary-light-palette PRIMARY AND ACCENT contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important @@ -50,354 +54,69 @@ $material-primary-light-palette: ( // NOTE: When creating the Material palette definition mat.define-palette($material-primary-light-palette, 300); // Since 300, was set as the default the contrast-300 will be used as the default text color. contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: rgba(black, 0.87), - A400: #ffffff, - A700: #ffffff, + 50: $on-surface-dark, + 100: $on-surface-dark, + 200: $on-surface-dark, + 300: $on-surface-light, + 400: $on-surface-light, + 500: $on-surface-light, + 600: $on-surface-light, + 700: $on-surface-light, + 800: $on-surface-light, + 900: $on-surface-light, + A100: $on-surface-dark, + A200: $on-surface-dark, + A400: $on-surface-light, + A700: $on-surface-light, ) ); // The $material-primary-dark-palette define the PRIMARY AND ACCENT palettes for all Angular Material components used throughout Apache NiFi -$material-primary-dark-palette: ( - // 50 -> 900 are the PRIMARY colors (mat.define-palette($material-primary-dark-palette, 300);) defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b - 50: rgba(69, 47, 101, 1), // .context-menu - 100: rgba(93, 57, 123, 1), // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, - 200: #6b3f86, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a, button.nifi-button, button.nifi-button:disabled - 300: #7b4690, // .breadcrumb-container, .navigation-control, .operation-control, .flow-status, .controller-bulletins, .component-button-grip, .search-container, .nifi-navigation, .CodeMirror.blank - 400: #874b98, // Default hue for this palette (color="primary"). - 500: #985fa7, // .disabled, .not-transmitting, .splash, .access-policies-header, .operation-context-type, .bulletin-board-header, .counter-header, .stats-info, .active-thread-count-icon, .processor-type, .port-transmission-icon, .operation-context-type, .flow-status.fa, .flow-status.icon, .controller-bulletins, .prioritizers-list, .controller-services-header, .login-title, .parameter-context-header, .parameter-context-inheritance-list, .provenance-header, .flowfile-header, .queue-listing-header, .settings-header, .summary-header, .user-header, table th, button.global-menu-item.fa, button.global-menu-item.icon, .event-header, .section-header, - 600: #aa79b7, // .breadcrumb-container, .birdseye-brush - 700: #d3bada, // "darker" hue for this palette. Also #status-history-chart-container text, #status-history-chart-control-container text - 800: #dac3e0, - 900: #f0e7f2, - - // A100 -> A700 are the ACCENT color (mat.define-palette($material-primary-dark-palette, A400, A100, A700);). These color are the ANALOGOUS (or possibly the TRIADIC??) colors as defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b - // These colors are also used by some custom canvas components that display the ANALOGOUS color for things like buttons, links, borders, info, etc. - A100: #2cd5d5, // .zero - A200: #009b9d, // .enabled, .transmitting, .load-balance-icon-active - A400: #44a3cf, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu - A700: rgba(139, 208, 229, 1),//#aabec7 // .hint-pattern - - // These are the $material-primary-dark-palette PRIMARY AND ACCENT contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($material-primary-dark-palette, 300); - // Since 300, was set as the default the contrast-300 will be used as the default text color. - contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); - -// The $nifi-canvas-light-palette defines the PRIMARY palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-light-palette: ( - // mat.define-palette($nifi-canvas-light-palette) - 50: rgba(0, 0, 0, 0.25), // .tooltip, .property-editor, g.component rect.border, .component-comments, g.connection path.connection-path, g.connection rect.backpressure-tick.data-size-prediction, g.connection rect.backpressure-tick.object-prediction, g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .breadcrumb-container, .navigation-control, .operation-control, header.nifi-header, .new-canvas-item.icon.hovering, .cdk-drag-disabled, .cdk-drag-preview, .context-menu, .global-menu:hover, - 100: rgba(black, 0.87), // .prioritizer-draggable-item, .parameter-context-draggable-item - 200: #000, // .tooltip, .cm-s-default .CodeMirror-matchingbracket, circle.flowfile-link - 300: #aaaaaa, // .context-menu-item:active, .CodeMirror, .cm-s-default .CodeMirror-matchingbracket - 400: #acacac, // .unset, .border.ghost, .backpressure-tick.not-configured, g.connection.ghost path.connection-path, g.connection.ghost rect.connection-label, .prioritizers-list, .prioritizer-draggable-item, .parameter-context-inheritance-list, .parameter-context-draggable-item - 500: #d8d8d8, // g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .cdk-drag-disabled, .resizable-triangle - 600: #f9fafb, // .canvas-background, .navigation-control, .operation-control, .lineage - 700: #e5ebed, // .canvas-background, g.component rect.body.unauthorized, g.component rect.processor-icon-container.unauthorized, g.connection rect.body.unauthorized, #birdseye, .lineage - 800: #f4f6f7, // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats - 900: rgba(255, 255, 255, 1), // circle.flowfile-link, .processor-read-write-stats, .process-group-stats-in-out, .tooltip, .property-editor, .disabled, .enabled, .stopped, .running, .has-errors, .invalid, .validating, .transmitting, .not-transmitting, .up-to-date, .locally-modified, .sync-failure, .stale, .locally-modified-and-stale, g.component rect.body, text.bulletin-icon, rect.processor-icon-container, circle.restricted-background, circle.is-primary-background, g.connection rect.body, text.connection-to-run-status, text.expiration-icon, text.load-balance-icon, text.penalized-icon, g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down, text.version-control, .breadcrumb-container, #birdseye, .controller-bulletins .fa, .search-container:hover, .search-container.open, .login-background, table th, .mat-sort-header-arrow, .CodeMirror, #status-history-chart-container, #status-history-chart-control-container, #status-history-chart-control-container, - - // some analog colors for headers and hover states, inputs, stats, etc - A100: rgba(227, 232, 235, 0), // .navigation-control-header:hover, .operation-control-header:hover, .axis path, .axis line - A200: #262626, // .operation-context-name, text.stats-label, text.processor-name, text.port-name, text.process-group-name, text.remote-process-group-name, .navigation-control-title, .operation-control-title, .operation-context-name, .search-input, .context-menu-item-text, .current-user - A400: #444, // rect.component-selection, rect.drag-selection, rect.label-drag - A700: #666, // text.processor-bundle, .search-container:hover, .search-container.open, .search-container.fa, .selected-type-bundle, .brush .selection - - // These are the $nifi-canvas-light-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-light-palette); - // Since 500 is the default the contrast-500 will be used as the default text color. - contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: rgba(black, 0.87), - 400: #ffffff, - 500: #ffffff, - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: rgba(black, 0.87), - A400: #ffffff, - A700: #ffffff, - ) -); - -// The $nifi-canvas-dark-palette defines the PRIMARY palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-dark-palette: ( - // mat.define-palette($nifi-canvas-dark-palette) - 50: rgba(255, 255, 255, 1), // .tooltip, .property-editor, g.component rect.border, .component-comments, g.connection path.connection-path, g.connection rect.backpressure-tick.data-size-prediction, g.connection rect.backpressure-tick.object-prediction, g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .breadcrumb-container, .navigation-control, .operation-control, header.nifi-header, .new-canvas-item.icon.hovering, .cdk-drag-disabled, .cdk-drag-preview, .context-menu, .global-menu:hover, - 100: #f4f6f7, //rgba(black, 0.87), // .prioritizer-draggable-item, .parameter-context-draggable-item - 200: #e5ebed, // .tooltip, .cm-s-default .CodeMirror-matchingbracket, circle.flowfile-link - 300: #f9fafb, // .context-menu-item:active, .CodeMirror, .cm-s-default .CodeMirror-matchingbracket - 400: #d8d8d8, // .unset, .border.ghost, .backpressure-tick.not-configured, g.connection.ghost path.connection-path, g.connection.ghost rect.connection-label, .prioritizers-list, .prioritizer-draggable-item, .parameter-context-inheritance-list, .parameter-context-draggable-item - 500: #acacac, // g.connection rect.backpressure-object, g.connection rect.backpressure-data-size, .cdk-drag-disabled, .resizable-triangle - 600: #545454, // .canvas-background, .navigation-control, .operation-control, .lineage - 700: #696060, // .canvas-background, g.component rect.body.unauthorized, g.component rect.processor-icon-container.unauthorized, g.connection rect.body.unauthorized, #birdseye, .lineage - 800: rgba(#6b6464, 1), // .even, .remote-process-group-sent-stats, .processor-stats-in-out, .process-group-queued-stats, .process-group-read-write-stats - 900: rgba(#252424, 0.97), // circle.flowfile-link, .processor-read-write-stats, .process-group-stats-in-out, .tooltip, .property-editor, .disabled, .enabled, .stopped, .running, .has-errors, .invalid, .validating, .transmitting, .not-transmitting, .up-to-date, .locally-modified, .sync-failure, .stale, .locally-modified-and-stale, g.component rect.body, text.bulletin-icon, rect.processor-icon-container, circle.restricted-background, circle.is-primary-background, g.connection rect.body, text.connection-to-run-status, text.expiration-icon, text.load-balance-icon, text.penalized-icon, g.connection rect.backpressure-tick.data-size-prediction.prediction-down, g.connection rect.backpressure-tick.object-prediction.prediction-down, text.version-control, .breadcrumb-container, #birdseye, .controller-bulletins .fa, .search-container:hover, .search-container.open, .login-background, table th, .mat-sort-header-arrow, .CodeMirror, #status-history-chart-container, #status-history-chart-control-container, #status-history-chart-control-container, - - // some analog colors for headers and hover states, inputs, stats, etc - A100: #000, // .navigation-control-header:hover, .operation-control-header:hover, .axis path, .axis line - A200: #e7e6e6, // .operation-context-name, text.stats-label, text.processor-name, text.port-name, text.process-group-name, text.remote-process-group-name, .navigation-control-title, .operation-control-title, .operation-context-name, .search-input, .context-menu-item-text, .current-user - A400: #b0b0b0, // rect.component-selection, rect.drag-selection, rect.label-drag - A700: #b2b2b2, // text.processor-bundle, .search-container:hover, .search-container.open, .search-container.fa, .selected-type-bundle, .brush .selection - - // These are the $nifi-canvas-dark-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-dark-palette); - // Since 500 is the default the contrast-500 will be used as the default text color. - contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); - -// The $nifi-canvas-light-palette defines the ACCENT palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-accent-light-palette: ( - // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #52bf7e - 50: #e6f6ec, - 100: #c3e8d0, // "lighter" hue for this palette. - 200: #9dd9b2, //.running - 300: #73ca94, //.backpressure-percent - 400: #52bf7e, // color="primary" Default hue for this palette. Also .up-to-date - 500: #2cb367, - 600: #1A9964, //.version-control - 700: #016131, // "darker" hue for this palette Also .bulletin-normal - 800: #0000ff, //.endpoint, g.process-group.drop rect.border - 900: #00ff00, //.connectable-destination, .connector.connectable - - // A100 - A700 are the ANALOGOUS colors but are more customized. These colors are used to highlight, warn, denote midpoints and labelpoints, etc - A100: #ffef85, //.selected - A200: #f8bf47, //.invalid, .is-missing-port, circle.context - A400: #bda500, //.backpressure-percent-warning, .bulletin-warn, .backpressure-percent.warning, text.run-status-icon - A700: #ffcc00, //g.connection.selected rect.border, .connection-selection-path, .midpoint, .labelpoint - - // These are the $nifi-canvas-accent-light-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-accent-light-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. - contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: rgba(black, 0.87), - 400: rgba(black, 0.87), - 500: rgba(black, 0.87), - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: rgba(black, 0.87), - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); - -// The $nifi-canvas-dark-palette defines the ACCENT palette for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$nifi-canvas-accent-dark-palette: ( - // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #52bf7e - 50: #016131, - 100: #1A9964, // "lighter" hue for this palette. - 200: #2cb367, //.running - 300: #52bf7e, //.backpressure-percent - 400: #73ca94, // color="primary" Default hue for this palette. Also .up-to-date - 500: #9dd9b2, - 600: #c3e8d0, //.version-control - 700: #e6f6ec, // "darker" hue for this palette Also .bulletin-normal - 800: #0000ff, //.endpoint, g.process-group.drop rect.border - 900: #00ff00, //.connectable-destination, .connector.connectable - - // A100 - A700 are the ANALOGOUS colors but are more customized. These colors are used to highlight, warn, denote midpoints and labelpoints, etc - A100: #cbaa09, //.selected - A200: #bda500, //.invalid, .is-missing-port, circle.context - A400: #f8bf47, //.backpressure-percent-warning, .bulletin-warn, .backpressure-percent.warning, text.run-status-icon - A700: #948b4b, //g.connection.selected rect.border, .connection-selection-path, .midpoint, .labelpoint - - // These are the $nifi-canvas-accent-dark-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($nifi-canvas-accent-dark-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. - contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: #ffffff, - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); +$material-primary-dark-palette: $material-primary-light-palette; +// The $warn-light-palette defines the WARN palette both for all Angular Material components used throughout Apache NiFi and for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi // The $warn-light-palette defines the WARN palette both for all Angular Material components used throughout Apache NiFi and for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi $warn-light-palette: ( // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #f64e4c 50: #ffebee, - 100: #ffccd2, // "lighter" hue for this palette. Also .banner-error - 200: #f49999, //.stopped - 300: #eb7071, //.stale - 400: #f64e4c, // color="primary" Default hue for this palette. Also .unauthorized, .bulletin-background, .restricted, .connection-path.full, .connection-path.unauthorized, .backpressure-percent.error, .controller-bulletins.has-bulletins, .link.selected, circle.selected, .listing-message, .fa-shield + 100: #ffccd2, + 200: #f49999, // "lighter" hue for this palette..stopped + 300: #f57472, + 400: #f64e4c, // color="primary" Default hue for this palette. Also .unauthorized, .restricted, .connection-path.full, .connection-path.unauthorized, .backpressure-percent.error, .controller-bulletins.has-bulletins, .link.selected, circle.selected, .listing-message, .fa-shield, .stale 500: #fa3b30, - 600: #ec3030, //.bulletin-error - 700: #ff1507, // "darker" hue for this palette - 800: #ff0000, //.connector, .startpoint - 900: #f10000, - - // A100 - A700 are the COMPLEMENTARY colors defined by the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b which is the primary color of the $material-primary-light-palette. - // These color are used for label values, stats, timestamps, counts, etc. - A100: #b6abaa, //.sync-failure - A200: #9c8886, // .operation-context-logo, .funnel-icon, .port-icon, .flowfile-icon - A400: #765452, // .value, .refresh-timestamp, .stats-value, .active-thread-count, .process-group-contents-count, .operation-context-id, .selected-type-name - A700: #4a3435, // .version-control - - // These are the $nifi-canvas-accent-light-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($warn-light-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. + 600: #ff1507, // .bulletin-error + 700: #ff0000, //.connector, .startpoint + 800: #ec3030, + 900: #ba554a, // "darker" hue for this palette .bulletin-background + A100: #ffef85, //.sync-failure + A200: #f8bf47, //.invalid, .is-missing-port, circle.context + A400: #f29833, // .value, .refresh-timestamp, .stats-value, .active-thread-count, .process-group-contents-count, .operation-context-id, .selected-type-name + A700: #eb711e, // .version-control contrast: ( - 50: rgba(black, 0.87), - 100: rgba(black, 0.87), - 200: rgba(black, 0.87), - 300: rgba(black, 0.87), - 400: rgba(black, 0.87), - 500: #ffffff, - 600: #ffffff, - 700: #ffffff, - 800: #ffffff, - 900: #ffffff, - A100: rgba(black, 0.87), - A200: #ffffff, - A400: #ffffff, - A700: #ffffff, + 50: $on-surface-dark, + 100: $on-surface-dark, + 200: $on-surface-dark, + 300: $on-surface-dark, + 400: $on-surface-dark, + 500: $on-surface-dark, + 600: $on-surface-dark, + 700: $on-surface-dark, + 800: $on-surface-dark, + 900: $on-surface-light, + A100: $on-surface-dark, + A200: $on-surface-dark, + A400: $on-surface-dark, + A700: $on-surface-dark, ) ); // The $warn-dark-palette defines the WARN palette both for all Angular Material components used throughout Apache NiFi and for all flow designer canvas components that make up the NiFi canvas theme used throughout Apache NiFi -$warn-dark-palette: ( - // 50 -> 900 are the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #f64e4c - 50: #f10000, - 100: #ff0000, // "lighter" hue for this palette. Also .banner-error - 200: #ff1507, //.stopped - 300: #ec3030, //.stale - 400: #fa3b30, // color="primary" Default hue for this palette. Also .unauthorized, .bulletin-background, .restricted, .connection-path.full, .connection-path.unauthorized, .backpressure-percent.error, .controller-bulletins.has-bulletins, .link.selected, circle.selected, .listing-message, .fa-shield - 500: #f64e4c, - 600: #eb7071, //.bulletin-error - 700: #f49999, // "darker" hue for this palette - 800: #ffccd2, //.connector, .startpoint - 900: #ffebee, - - // A100 - A700 are the COMPLEMENTARY colors defined by the PRIMARY colors defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b which is the primary color of the $material-primary-dark-palette. - // These color are used for label values, stats, timestamps, counts, etc. - A100: #8e77ab, //.sync-failure - A200: #9a81b9, // .operation-context-logo, .funnel-icon, .port-icon, .flowfile-icon - A400: #c5b7d5, // .value, .refresh-timestamp, .stats-value, .active-thread-count, .process-group-contents-count, .operation-context-id, .selected-type-name - A700: #afaab6, // .version-control - - // These are the $nifi-canvas-accent-dark-palette PRIMARY palette contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. - // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important - // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. - // - // NOTE: Care should be taken here to ensure the values meet accessibility standards. - // - // NOTE: When creating the Material palette definition mat.define-palette($warn-dark-palette, 400, 100, 700); - // Since 400 is the default the contrast-400 will be used as the default text color in some cases. - contrast: ( - 50: #ffffff, - 100: #ffffff, - 200: #ffffff, - 300: #ffffff, - 400: #ffffff, - 500: rgba(black, 0.87), - 600: rgba(black, 0.87), - 700: rgba(black, 0.87), - 800: rgba(black, 0.87), - 900: rgba(black, 0.87), - A100: #ffffff, - A200: #ffffff, - A400: rgba(black, 0.87), - A700: rgba(black, 0.87), - ) -); +$warn-dark-palette: $warn-light-palette; // Define the palettes for your theme $material-primary-light: mat.define-palette($material-primary-light-palette); -$material-accent-light: mat.define-palette($material-primary-light-palette, A400, A100, A700); -$nifi-canvas-primary-light: mat.define-palette($nifi-canvas-light-palette); -$nifi-canvas-accent-light: mat.define-palette($nifi-canvas-accent-light-palette, 400, 100, 700); -$warn-light: mat.define-palette($warn-light-palette, 400, 100, 700); +$material-accent-light: mat.define-palette($material-primary-light-palette, A700, A200, A100); +$warn-light: mat.define-palette($warn-light-palette, 400, 100, 600); // Create the theme objects. We can extract the values we need from the theme passed into the mixins. $material-theme-light: mat.define-light-theme( @@ -412,24 +131,10 @@ $material-theme-light: mat.define-light-theme( ) ); -$nifi-canvas-theme-light: mat.define-light-theme( - ( - color: ( - primary: $nifi-canvas-primary-light, - accent: $nifi-canvas-accent-light, - warn: $warn-light - ), - //typography: mat.define-typography-config(), // TODO: typography - density: -3 - ) -); - // Create the color palettes used in our dark theme. $material-primary-dark: mat.define-palette($material-primary-dark-palette); -$material-accent-dark: mat.define-palette($material-primary-dark-palette, A400, A100, A700); -$nifi-canvas-primary-dark: mat.define-palette($nifi-canvas-dark-palette); -$nifi-canvas-accent-dark: mat.define-palette($nifi-canvas-accent-dark-palette); -$warn-dark: mat.define-palette($warn-dark-palette, 600, 200, 800); +$material-accent-dark: mat.define-palette($material-primary-dark-palette, A700, A200, A100); +$warn-dark: mat.define-palette($warn-dark-palette, 800, 200, 700); $material-theme-dark: mat.define-dark-theme( ( @@ -442,16 +147,3 @@ $material-theme-dark: mat.define-dark-theme( typography: mat.define-typography-config(), ) ); - -$nifi-canvas-theme-dark: mat.define-dark-theme( - ( - color: ( - primary: $nifi-canvas-primary-dark, - accent: $nifi-canvas-accent-dark, - warn: $warn-dark, - ), - //typography: mat.define-typography-config(), // TODO: typography - density: -3 - ) -); - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss new file mode 100644 index 000000000000..7f9b8d3546bd --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss @@ -0,0 +1,186 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@use 'sass:map'; +@use '@angular/material' as mat; + +@function ensure-contrast($foreground, $background, $backup: #000000) { + $ratio: contrast($foreground, $background); + $backup-ratio: contrast($backup, $background); + + @if $ratio < 4.5 { + @return $backup; + } @else { + @return $foreground; + } +} + +@function get-surface($theme, $step: default, $palette: 'primary') { + // Check to see if it's a dark theme + $is-dark: map.get($theme, is-dark); + $color-config: mat.get-color-config($theme); + $color-palette: map.get($theme, $palette); + + $value: if( + $is-dark, + ( + default: 900, + darker: 800 + ), + ( + default: 50, + darker: 100 + ) + ); + + //Grab 50 for light themes and 900 for dark themes + $surface: mat.get-color-from-palette($color-palette, map.get($value, $step)); + + @return $surface; +} + +@function get-on-surface($theme, $step: default, $palette: 'primary') { + // Check to see if it's a dark theme + $is-dark: map.get($theme, is-dark); + $color-config: mat.get-color-config($theme); + $color-palette: map.get($theme, 'primary'); + + $value: if($is-dark, '900-contrast', '50-contrast'); + + //Grab 50 for light themes and 900 for dark themes + $on-surface: mat.get-color-from-palette($color-palette, $value); + $highlight: if($is-dark, 0.1, 0.07); + + $list: ( + default: 1, + lighter: if($is-dark, 0.28, 0.2), + highlight: if($is-dark, 0.15, 0.05), + medium: 0.87 + ); + + $alpha: map.get($list, $step); + $alpha: if($alpha, $alpha, 1); + + $on-surface: rgba($on-surface, $alpha); + + @return $on-surface; +} + +@function get-color-on-surface($theme, $surface: #ffffff, $palette: 'primary') { + // Check to see if it's a dark theme + $is-dark: map.get($theme, is-dark); + $color-config: mat.get-color-config($theme); + $color-palette: map.get($theme, $palette); + + $default: mat.get-color-from-palette($color-palette, default); + $high-contrast: mat.get-color-from-palette($color-palette, if($is-dark, lighter, darker)); + + $on-surface: ensure-contrast($default, $surface, $high-contrast); + + @return $on-surface; +} + +@function contrast($foreground, $background) { + // Calculate and modify the luminosity based on the W3C formula. + $foreground-luminosity: luminosity($foreground) + 0.05; + $background-luminosity: luminosity($background) + 0.05; + + $ratio: 0; + // Invert the colors to get the ratio when the background is brighter than the foreground. + @if ($background-luminosity > $foreground-luminosity) { + $ratio: calc($background-luminosity / $foreground-luminosity); + } @else { + $ratio: calc($foreground-luminosity / $background-luminosity); + } + + @return $ratio; +} + +@function luminosity($color) { + // Get the index so we can lookup the pre-calculated values. + // We add 1 because the list index starts at 1, not 0. + $r: red($color) + 1; + $g: green($color) + 1; + $b: blue($color) + 1; + + // Lookup the pre-calculated values. + $r: nth($rgb-lookup, $r); + $g: nth($rgb-lookup, $g); + $b: nth($rgb-lookup, $b); + + // Apply the weights for each color channel, and add them together. + $luminosity: $r * 0.2126 + $g * 0.7152 + $b * 0.0722; + + @return $luminosity; +} + +/* + * This is a lookup list of the outputs of the middle luminosity calculation used by W3C + * to determine contrast ratios. SASS cannot do exponent calculations so this is necessary. + * + * Indexes 1 through 10 are calculated as (8bit value for RGB / 255) / 12.92 + * Indexes 11 through 256 are calculated as ((8bit value for RGB / 255) + 0.055) / 1.055) ^ 2.4 + * + */ +$rgb-lookup: 0 0.0003035269835488375 0.000607053967097675 0.0009105809506465125 0.00121410793419535 + 0.0015176349177441874 0.001821161901293025 0.0021246888848418626 0.0024282158683907 0.0027317428519395373 + 0.003035269835488375 0.003346535763899161 0.003676507324047436 0.004024717018496307 0.004391442037410293 + 0.004776953480693729 0.005181516702338386 0.005605391624202723 0.006048833022857054 0.006512090792594475 + 0.006995410187265387 0.007499032043226175 0.008023192985384994 0.008568125618069307 0.009134058702220787 + 0.00972121732023785 0.010329823029626936 0.010960094006488246 0.011612245179743885 0.012286488356915872 + 0.012983032342173012 0.013702083047289686 0.014443843596092545 0.01520851442291271 0.01599629336550963 + 0.016807375752887384 0.017641954488384078 0.018500220128379697 0.019382360956935723 0.0202885630566524 + 0.021219010376003555 0.022173884793387385 0.02315336617811041 0.024157632448504756 0.02518685962736163 + 0.026241221894849898 0.027320891639074894 0.028426039504420793 0.0295568344378088 0.030713443732993635 + 0.03189603307301153 0.033104766570885055 0.03433980680868217 0.03560131487502034 0.03688945040110004 + 0.0382043715953465 0.03954623527673284 0.04091519690685319 0.042311410620809675 0.043735029256973465 + 0.04518620438567554 0.046665086336880095 0.04817182422688942 0.04970656598412723 0.05126945837404324 + 0.052860647023180246 0.05448027644244237 0.05612849004960009 0.05780543019106723 0.0595112381629812 + 0.06124605423161761 0.06301001765316767 0.06480326669290577 0.06662593864377289 0.06847816984440017 + 0.07036009569659588 0.07227185068231748 0.07421356838014963 0.07618538148130785 0.07818742180518633 + 0.08021982031446832 0.0822827071298148 0.08437621154414882 0.08650046203654976 0.08865558628577294 + 0.09084171118340768 0.09305896284668745 0.0953074666309647 0.09758734714186246 0.09989872824711389 + 0.10224173308810132 0.10461648409110419 0.10702310297826761 0.10946171077829933 0.1119324278369056 + 0.11443537382697373 0.11697066775851084 0.11953842798834562 0.12213877222960187 0.12477181756095049 + 0.12743768043564743 0.1301364766903643 0.13286832155381798 0.13563332965520566 0.13843161503245183 + 0.14126329114027164 0.14412847085805777 0.14702726649759498 0.14995978981060856 0.15292615199615017 + 0.1559264637078274 0.1589608350608804 0.162029375639111 0.1651321945016676 0.16826940018969075 0.1714411007328226 + 0.17464740365558504 0.17788841598362912 0.18116424424986022 0.184474994500441 0.18782077230067787 + 0.19120168274079138 0.1946178304415758 0.19806931955994886 0.20155625379439707 0.20507873639031693 + 0.20863687014525575 0.21223075741405523 0.21586050011389926 0.2195261997292692 0.2232279573168085 + 0.22696587351009836 0.23074004852434915 0.23455058216100522 0.238397573812271 0.24228112246555486 + 0.24620132670783548 0.25015828472995344 0.25415209433082675 0.2581828529215958 0.26225065752969623 + 0.26635560480286247 0.2704977910130658 0.27467731206038465 0.2788942634768104 0.2831487404299921 0.2874408377269175 + 0.29177064981753587 0.2961382707983211 0.3005437944157765 0.3049873140698863 0.30946892281750854 0.31398871337571754 + 0.31854677812509186 0.32314320911295075 0.3277780980565422 0.33245153634617935 0.33716361504833037 + 0.3419144249086609 0.3467040563550296 0.35153259950043936 0.3564001441459435 0.3613067797835095 0.3662525955988395 + 0.3712376804741491 0.3762621229909065 0.38132601143253014 0.386429433787049 0.39157247774972326 0.39675523072562685 + 0.4019777798321958 0.4072402119017367 0.41254261348390375 0.4178850708481375 0.4232676699860717 0.4286904966139066 + 0.43415363617474895 0.4396571738409188 0.44520119451622786 0.45078578283822346 0.45641102318040466 + 0.4620769996544071 0.467783796112159 0.47353149614800955 0.4793201831008268 0.4851499400560704 0.4910208498478356 + 0.4969329950608704 0.5028864580325687 0.5088813208549338 0.5149176653765214 0.5209955732043543 0.5271151257058131 + 0.5332764040105052 0.5394794890121072 0.5457244613701866 0.5520114015120001 0.5583403896342679 0.5647115057049292 + 0.5711248294648731 0.5775804404296506 0.5840784178911641 0.5906188409193369 0.5972017883637634 0.6038273388553378 + 0.6104955708078648 0.6172065624196511 0.6239603916750761 0.6307571363461468 0.6375968739940326 0.6444796819705821 + 0.6514056374198242 0.6583748172794485 0.665387298282272 0.6724431569576875 0.6795424696330938 0.6866853124353135 + 0.6938717612919899 0.7011018919329731 0.7083757798916868 0.7156935005064807 0.7230551289219693 0.7304607400903537 + 0.7379104087727308 0.7454042095403874 0.7529422167760779 0.7605245046752924 0.768151147247507 0.7758222183174236 + 0.7835377915261935 0.7912979403326302 0.799102738014409 0.8069522576692516 0.8148465722161012 0.8227857543962835 + 0.8307698767746546 0.83879901174074 0.846873231509858 0.8549926081242338 0.8631572134541023 0.8713671191987972 + 0.8796223968878317 0.8879231178819663 0.8962693533742664 0.9046611743911496 0.9130986517934192 0.9215818562772946 + 0.9301108583754237 0.938685728457888 0.9473065367331999 0.9559733532492861 0.9646862478944651 0.9734452903984125 + 0.9822505503331171 0.9911020971138298 1; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss index bd9dce75684c..486bc6a41ce5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss @@ -73,8 +73,13 @@ @use 'codemirror/lib/codemirror.css'; @use 'codemirror/addon/hint/show-hint.css'; +@use 'assets/utils.scss' as utils; + // NOTE: for faster developer cycles during theme development the theme can also be changed here. // Ex: @import 'assets/themes/purple'; +// To override the canvas theme, you need to set the variables $nifi-canvas-theme-light and $nifi-canvas-theme-dark +@import 'assets/themes/nifi-canvas'; +// To override the NiFi theme, you need to set the variables $material-theme-light and $material-theme-dark @import 'assets/themes/nifi'; $fontPrimary: 'Roboto', sans-serif; @@ -187,149 +192,146 @@ $appFontPath: '~roboto-fontface/fonts'; $canvas-accent-palette: map.get($canvas-color-config, 'accent'); // Get hues from palette - $primary-palette-50: mat.get-color-from-palette($primary-palette, 50); - $primary-palette-200: mat.get-color-from-palette($primary-palette, 200); - $primary-palette-500: mat.get-color-from-palette($primary-palette, 500); - $accent-palette-A100: mat.get-color-from-palette($accent-palette, 'A100'); - $accent-palette-A200: mat.get-color-from-palette($accent-palette, 'A200'); - $accent-palette-A400: mat.get-color-from-palette($accent-palette, 'A400'); - $canvas-primary-palette-50: mat.get-color-from-palette($canvas-primary-palette, 50); - $canvas-primary-palette-200: mat.get-color-from-palette($canvas-primary-palette, 200); + + // Start with the canvas theme. + $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, A200); $canvas-primary-palette-400: mat.get-color-from-palette($canvas-primary-palette, 400); - $canvas-primary-palette-900: mat.get-color-from-palette($canvas-primary-palette, 900); - $canvas-accent-palette-200: mat.get-color-from-palette($canvas-accent-palette, 200); - $canvas-accent-palette-400: mat.get-color-from-palette($canvas-accent-palette, 400); - $canvas-accent-palette-A200: mat.get-color-from-palette($canvas-accent-palette, 'A200'); - $warn-palette-200: mat.get-color-from-palette($warn-palette, 200); - $warn-palette-300: mat.get-color-from-palette($warn-palette, 300); - $warn-palette-A100: mat.get-color-from-palette($warn-palette, 'A100'); - $warn-palette-A400: mat.get-color-from-palette($warn-palette, 'A400'); + $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); + $canvas-accent-palette-lighter: mat.get-color-from-palette($canvas-accent-palette, lighter); + $canvas-accent-palette-default: mat.get-color-from-palette($canvas-accent-palette, default); + + $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); + $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); + $primary-palette-A400: mat.get-color-from-palette($primary-palette, 'A400'); + + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); + $accent-palette-lighter: mat.get-color-from-palette($accent-palette, 'lighter'); + + $warn-palette-lighter: mat.get-color-from-palette($warn-palette, lighter); + $warn-palette-default: mat.get-color-from-palette($warn-palette, 'default'); + + // Alternative hue for warn colors. + $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); + + $surface: utils.get-surface($canvas-color-config); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $surface-highlight: utils.get-on-surface($canvas-color-config, highlight); + $on-surface: utils.get-on-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + + * { // Tailwind sets a default that doesn't shift with light and dark themes + border-color: $on-surface-lighter; + } a { - color: $accent-palette-A400; - text-decoration-color: $primary-palette-200; + color: utils.get-color-on-surface($color-config, $surface); + text-decoration-color: $primary-palette-lighter; } a:hover { - text-decoration-color: $accent-palette-A400; + text-decoration-color: utils.get-color-on-surface($color-config, $surface); } .tooltip { - background-color: $canvas-primary-palette-900; - border-color: $canvas-primary-palette-200; - box-shadow: 0 2px 5px $canvas-primary-palette-50; - color: $canvas-primary-palette-200; + background-color: $surface; + border-color: $on-surface; + box-shadow: 0 2px 5px $canvas-primary-palette-A200; + color: $on-surface; } .property-editor { - background-color: $canvas-primary-palette-900; - box-shadow: 0 2px 5px $canvas-primary-palette-50; + background-color: $surface; + box-shadow: 0 2px 5px $canvas-primary-palette-A200; } .disabled { - color: $primary-palette-500 !important; - fill: $primary-palette-500 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $primary-palette-default !important; + fill: $primary-palette-default !important; } .enabled { - color: $accent-palette-A200 !important; - fill: $accent-palette-A200 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $primary-palette-A400 !important; + fill: $primary-palette-A400 !important; } .stopped { - color: $warn-palette-200 !important; - fill: $warn-palette-200 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $warn-palette-lighter !important; + fill: $warn-palette-lighter !important; } .running { - color: $canvas-accent-palette-200 !important; - fill: $canvas-accent-palette-200 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $canvas-accent-palette-lighter !important; + fill: $canvas-accent-palette-lighter !important; } .has-errors, .invalid { - color: $canvas-accent-palette-A200 !important; - fill: $canvas-accent-palette-A200 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $warn-palette-A200 !important; + fill: $warn-palette-A200 !important; } .validating { - color: $canvas-primary-palette-400 !important; - fill: $canvas-primary-palette-400 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $canvas-primary-palette-500 !important; + fill: $canvas-primary-palette-500 !important; } .transmitting { - color: $accent-palette-A200 !important; - fill: $accent-palette-A200 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; - } - - .not-transmitting { - color: $primary-palette-500 !important; - fill: $primary-palette-500 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $canvas-accent-palette-default !important; + fill: $canvas-accent-palette-default !important; } .up-to-date { - color: $canvas-accent-palette-400 !important; - fill: $canvas-accent-palette-400 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $canvas-accent-palette-default !important; + fill: $canvas-accent-palette-default !important; } .locally-modified, .sync-failure { - color: $warn-palette-A100 !important; - fill: $warn-palette-A100 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $accent-palette-lighter !important; + fill: $accent-palette-lighter !important; } .stale, .locally-modified-and-stale { - color: $warn-palette-300 !important; - fill: $warn-palette-300 !important; - text-shadow: 0 0 4px $canvas-primary-palette-900; + color: $warn-palette-default !important; + fill: $warn-palette-default !important; } .zero { - color: $accent-palette-A100 !important; - fill: $accent-palette-A100 !important; - text-shadow: none !important; + opacity: 0.5; } .blank, .unset, .sensitive { - color: $canvas-primary-palette-400 !important; + color: $canvas-primary-palette-500 !important; } button.nifi-button { - border: 1px solid $primary-palette-200; - background-color: $primary-palette-50; - color: $accent-palette-A400; + color: utils.get-color-on-surface($color-config, $surface-darker); } button.nifi-button:hover { - border-color: $accent-palette-A400; + background-color: $surface-highlight; } button.nifi-button:disabled { - color: $primary-palette-200 !important; + color: $on-surface-lighter; + background-color: transparent; cursor: not-allowed; - border: 1px solid $primary-palette-200; - + i { - color: $primary-palette-200 !important; + color: $on-surface-lighter; } } .value, .refresh-timestamp { - color: $warn-palette-A400; + color: utils.get-color-on-surface($color-config, $surface, 'accent'); + } + + .accent-palette-default { + color: $accent-palette-default; } ngx-skeleton-loader .skeleton-loader { From 275dcc1e5084908deef8259f7f6d3807d7432463 Mon Sep 17 00:00:00 2001 From: bob Date: Tue, 12 Mar 2024 21:11:06 -0500 Subject: [PATCH 37/73] NIFI-12891: Update ElasticSearchClientService member variables on verify This closes #8496 Signed-off-by: Chris Sampson --- .../ElasticSearchClientServiceImpl.java | 26 ++++++++++++++----- .../ElasticSearchClientService_IT.java | 13 ++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java index 8445a883fd44..e2efb792f4ea 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java @@ -192,19 +192,23 @@ public void onEnabled(final ConfigurationContext context) throws InitializationE responseCharset = Charset.forName(context.getProperty(CHARSET).getValue()); // re-create the ObjectMapper in case the SUPPRESS_NULLS property has changed - the JsonInclude settings aren't dynamic - mapper = new ObjectMapper(); - if (ALWAYS_SUPPRESS.getValue().equals(context.getProperty(SUPPRESS_NULLS).getValue())) { - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - } + createObjectMapper(context); - prettyPrintWriter = mapper.writerWithDefaultPrettyPrinter(); } catch (final Exception ex) { getLogger().error("Could not initialize ElasticSearch client.", ex); throw new InitializationException(ex); } } + private void createObjectMapper(final ConfigurationContext context) { + mapper = new ObjectMapper(); + if (ALWAYS_SUPPRESS.getValue().equals(context.getProperty(SUPPRESS_NULLS).getValue())) { + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + } + prettyPrintWriter = mapper.writerWithDefaultPrettyPrinter(); + } + @OnDisabled public void onDisabled() throws IOException { if (this.sniffer != null) { @@ -217,6 +221,9 @@ public void onDisabled() throws IOException { this.client = null; } this.url = null; + this.mapper = null; + this.prettyPrintWriter = null; + this.responseCharset = null; } @Override @@ -231,6 +238,9 @@ public List verify(final ConfigurationContext context, final ConfigVerificationResult.Builder snifferResult = new ConfigVerificationResult.Builder() .verificationStepName(VERIFICATION_STEP_SNIFFER); + responseCharset = Charset.forName(context.getProperty(CHARSET).getValue()); + createObjectMapper(context); + // configure the Rest Client try (final RestClient verifyClient = setupClient(context)) { clientSetupResult.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL); @@ -252,6 +262,9 @@ public List verify(final ConfigurationContext context, clientSetupResult.outcome(ConfigVerificationResult.Outcome.FAILED) .explanation("Unable to configure, are all mandatory properties set (see logs for details)?"); } finally { + responseCharset = null; + mapper = null; + prettyPrintWriter = null; final ConfigVerificationResult clientSetup = clientSetupResult.build(); if (ConfigVerificationResult.Outcome.SUCCESSFUL != clientSetup.getOutcome()) { connectionResult.outcome(ConfigVerificationResult.Outcome.SKIPPED) @@ -266,6 +279,7 @@ public List verify(final ConfigurationContext context, results.add(connectionResult.build()); results.add(warningsResult.build()); results.add(snifferResult.build()); + } return results; diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java index 867e83717c2e..04a2cb62e6c1 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java @@ -96,6 +96,19 @@ void testVerifySuccess() { assertVerifySnifferSkipped(results); } + @Test + void testVerifyDisabledSuccess() { + //Disable as verify can be run in either state + runner.disableControllerService(service); + final List results = service.verify( + new MockConfigurationContext(service, getClientServiceProperties(), runner.getProcessContext().getControllerServiceLookup(), null), + runner.getLogger(), + Collections.emptyMap() + ); + assertEquals(4, results.size()); + assertEquals(3, results.stream().filter(result -> result.getOutcome() == ConfigVerificationResult.Outcome.SUCCESSFUL).count(), results.toString()); + } + @Test void testVerifySniffer() { runner.disableControllerService(service); From f8d3fcd66cd53668d339ccbcb6d2175defb63393 Mon Sep 17 00:00:00 2001 From: Peter Turcsanyi Date: Mon, 11 Mar 2024 16:45:17 +0100 Subject: [PATCH 38/73] NIFI-10707 Added proxy support in PutBigQuery Bumped GCP client library version Added grpc-* jars in service api nar in order to avoid CNFE warning in io.grpc.LoadBalancerRegistry Dependency clean-up in GCP modules Signed-off-by: Pierre Villard This closes #8491. --- .../nifi-gcp-bundle/nifi-gcp-nar/pom.xml | 196 ++++++++++++++++++ .../nifi-gcp-parameter-providers/pom.xml | 24 --- .../nifi-gcp-processors/pom.xml | 32 --- .../processors/gcp/bigquery/PutBigQuery.java | 41 +++- .../gcp/bigquery/PutBigQueryTest.java | 9 +- .../nifi-gcp-services-api/pom.xml | 29 +-- nifi-nar-bundles/nifi-gcp-bundle/pom.xml | 2 +- 7 files changed, 249 insertions(+), 84 deletions(-) diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-nar/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-nar/pom.xml index b4a71b996e73..d4a17e5b2c6c 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-nar/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-nar/pom.xml @@ -36,11 +36,207 @@ org.apache.nifi nifi-gcp-processors 2.0.0-SNAPSHOT + + + org.codehaus.mojo + animal-sniffer-annotations + + + com.google.android + annotations + + + com.google.auto.value + auto-value-annotations + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + + com.google.guava + failureaccess + + + com.google.auth + google-auth-library-credentials + + + com.google.auth + google-auth-library-oauth2-http + + + com.google.http-client + google-http-client + + + com.google.http-client + google-http-client-gson + + + io.grpc + grpc-api + + + io.grpc + grpc-context + + + io.grpc + grpc-core + + + io.grpc + grpc-util + + + com.google.code.gson + gson + + + com.google.guava + guava + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + com.google.j2objc + j2objc-annotations + + + com.google.code.findbugs + jsr305 + + + com.google.guava + listenablefuture + + + io.opencensus + opencensus-api + + + io.opencensus + opencensus-contrib-http-util + + + io.perfmark + perfmark-api + + org.apache.nifi nifi-gcp-parameter-providers 2.0.0-SNAPSHOT + + + org.codehaus.mojo + animal-sniffer-annotations + + + com.google.android + annotations + + + com.google.auto.value + auto-value-annotations + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + + com.google.guava + failureaccess + + + com.google.auth + google-auth-library-credentials + + + com.google.auth + google-auth-library-oauth2-http + + + com.google.http-client + google-http-client + + + com.google.http-client + google-http-client-gson + + + io.grpc + grpc-api + + + io.grpc + grpc-context + + + io.grpc + grpc-core + + + io.grpc + grpc-util + + + com.google.code.gson + gson + + + com.google.guava + guava + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + com.google.j2objc + j2objc-annotations + + + com.google.code.findbugs + jsr305 + + + com.google.guava + listenablefuture + + + io.opencensus + opencensus-api + + + io.opencensus + opencensus-contrib-http-util + + + io.perfmark + perfmark-api + + diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-parameter-providers/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-parameter-providers/pom.xml index e53128d48d17..d576e15cb66d 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-parameter-providers/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-parameter-providers/pom.xml @@ -42,33 +42,9 @@ 2.0.0-SNAPSHOT provided - - org.slf4j - jcl-over-slf4j - com.google.cloud google-cloud-secretmanager - - - commons-logging - commons-logging - - - - - com.google.auth - google-auth-library-oauth2-http - - - com.google.code.findbugs - jsr305 - - - commons-logging - commons-logging - - org.apache.nifi diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml index d95d101cd1ed..24e2f86ec933 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml @@ -94,12 +94,6 @@ com.google.cloud google-cloud-core - - - com.google.code.findbugs - jsr305 - - com.google.cloud @@ -109,10 +103,6 @@ com.google.cloud google-cloud-bigquery - - commons-logging - commons-logging - org.json json @@ -123,10 +113,6 @@ com.google.cloud google-cloud-bigquerystorage - - commons-logging - commons-logging - org.json json @@ -136,22 +122,10 @@ com.google.cloud google-cloud-pubsub - - - commons-logging - commons-logging - - com.google.cloud google-cloud-pubsublite - - - commons-logging - commons-logging - - com.google.apis @@ -195,12 +169,6 @@ com.google.cloud google-cloud-vision - - - commons-logging - commons-logging - - diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java index a7c27e8a7e95..8536f232c700 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java @@ -21,6 +21,8 @@ import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.rpc.TransportChannelProvider; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; import com.google.cloud.bigquery.storage.v1.BQTableSchemaToProtoDescriptor; @@ -42,6 +44,7 @@ import com.google.cloud.bigquery.storage.v1.stub.BigQueryWriteStubSettings; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; +import io.grpc.HttpConnectProxiedSocketAddress; import io.grpc.Status; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.TriggerSerially; @@ -59,7 +62,9 @@ import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.gcp.ProxyAwareTransportFactory; import org.apache.nifi.processors.gcp.bigquery.proto.ProtoUtils; +import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.serialization.RecordReader; import org.apache.nifi.serialization.RecordReaderFactory; import org.apache.nifi.serialization.record.MapRecord; @@ -67,6 +72,8 @@ import java.io.IOException; import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; @@ -182,7 +189,8 @@ public class PutBigQuery extends AbstractBigQueryProcessor { TRANSFER_TYPE, APPEND_RECORD_COUNT, RETRY_COUNT, - SKIP_INVALID_ROWS + SKIP_INVALID_ROWS, + ProxyConfiguration.createProxyConfigPropertyDescriptor(false, ProxyAwareTransportFactory.PROXY_SPECS) ).collect(collectingAndThen(toList(), Collections::unmodifiableList)); @Override @@ -198,7 +206,7 @@ public void onScheduled(ProcessContext context) { maxRetryCount = context.getProperty(RETRY_COUNT).asInteger(); recordBatchCount = context.getProperty(APPEND_RECORD_COUNT).asInteger(); endpoint = context.getProperty(BIGQUERY_API_ENDPOINT).evaluateAttributeExpressions().getValue(); - writeClient = createWriteClient(getGoogleCredentials(context)); + writeClient = createWriteClient(getGoogleCredentials(context), ProxyConfiguration.getConfiguration(context)); } @OnUnscheduled @@ -225,7 +233,7 @@ public void onTrigger(ProcessContext context, ProcessSession session) { writeStream = createWriteStream(tableName); tableSchema = writeStream.getTableSchema(); protoDescriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); - streamWriter = createStreamWriter(writeStream.getName(), protoDescriptor, getGoogleCredentials(context)); + streamWriter = createStreamWriter(writeStream.getName(), protoDescriptor, getGoogleCredentials(context), ProxyConfiguration.getConfiguration(context)); } catch (Descriptors.DescriptorValidationException | IOException e) { getLogger().error("Failed to create Big Query Stream Writer for writing", e); context.yield(); @@ -395,12 +403,13 @@ private WriteStream createWriteStream(TableName tableName) { return writeClient.createWriteStream(createWriteStreamRequest); } - protected BigQueryWriteClient createWriteClient(GoogleCredentials credentials) { + protected BigQueryWriteClient createWriteClient(GoogleCredentials credentials, ProxyConfiguration proxyConfiguration) { BigQueryWriteClient client; try { BigQueryWriteSettings.Builder builder = BigQueryWriteSettings.newBuilder(); builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); builder.setEndpoint(endpoint); + builder.setTransportChannelProvider(createTransportChannelProvider(proxyConfiguration)); client = BigQueryWriteClient.create(builder.build()); } catch (Exception e) { @@ -410,13 +419,35 @@ protected BigQueryWriteClient createWriteClient(GoogleCredentials credentials) { return client; } - protected StreamWriter createStreamWriter(String streamName, Descriptors.Descriptor descriptor, GoogleCredentials credentials) throws IOException { + protected StreamWriter createStreamWriter(String streamName, Descriptors.Descriptor descriptor, GoogleCredentials credentials, ProxyConfiguration proxyConfiguration) throws IOException { ProtoSchema protoSchema = ProtoSchemaConverter.convert(descriptor); StreamWriter.Builder builder = StreamWriter.newBuilder(streamName); builder.setWriterSchema(protoSchema); builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); builder.setEndpoint(endpoint); + builder.setChannelProvider(createTransportChannelProvider(proxyConfiguration)); + + return builder.build(); + } + + private TransportChannelProvider createTransportChannelProvider(ProxyConfiguration proxyConfiguration) { + InstantiatingGrpcChannelProvider.Builder builder = InstantiatingGrpcChannelProvider.newBuilder(); + + if (proxyConfiguration != null) { + if (proxyConfiguration.getProxyType() == Proxy.Type.HTTP) { + builder.setChannelConfigurator(managedChannelBuilder -> managedChannelBuilder.proxyDetector( + targetServerAddress -> HttpConnectProxiedSocketAddress.newBuilder() + .setTargetAddress((InetSocketAddress) targetServerAddress) + .setProxyAddress(new InetSocketAddress(proxyConfiguration.getProxyServerHost(), proxyConfiguration.getProxyServerPort())) + .setUsername(proxyConfiguration.getProxyUserName()) + .setPassword(proxyConfiguration.getProxyUserPassword()) + .build() + )); + } else if (proxyConfiguration.getProxyType() == Proxy.Type.SOCKS) { + getLogger().warn("Proxy type SOCKS is not supported, the proxy configuration will be ignored"); + } + } return builder.build(); } diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/bigquery/PutBigQueryTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/bigquery/PutBigQueryTest.java index 10b238220dd7..3ff7edf68c63 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/bigquery/PutBigQueryTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/bigquery/PutBigQueryTest.java @@ -47,6 +47,7 @@ import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.Processor; import org.apache.nifi.processors.gcp.credentials.service.GCPCredentialsControllerService; +import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.schema.access.SchemaAccessUtils; import org.apache.nifi.util.TestRunner; @@ -162,12 +163,12 @@ protected BigQuery getCloudService(final ProcessContext context) { } @Override - protected StreamWriter createStreamWriter(String streamName, Descriptors.Descriptor descriptor, GoogleCredentials credentials) { + protected StreamWriter createStreamWriter(String streamName, Descriptors.Descriptor descriptor, GoogleCredentials credentials, ProxyConfiguration proxyConfiguration) { return streamWriter; } @Override - protected BigQueryWriteClient createWriteClient(GoogleCredentials credentials) { + protected BigQueryWriteClient createWriteClient(GoogleCredentials credentials, ProxyConfiguration proxyConfiguration) { return writeClient; } }; @@ -410,12 +411,12 @@ protected BigQuery getCloudService(final ProcessContext context) { } @Override - protected StreamWriter createStreamWriter(String streamName, Descriptors.Descriptor descriptor, GoogleCredentials credentials) throws IOException { + protected StreamWriter createStreamWriter(String streamName, Descriptors.Descriptor descriptor, GoogleCredentials credentials, ProxyConfiguration proxyConfiguration) throws IOException { throw new IOException(); } @Override - protected BigQueryWriteClient createWriteClient(GoogleCredentials credentials) { + protected BigQueryWriteClient createWriteClient(GoogleCredentials credentials, ProxyConfiguration proxyConfiguration) { return writeClient; } }; diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml index 3714d85228d4..354a54896fe0 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-services-api/pom.xml @@ -33,29 +33,22 @@ com.google.auth google-auth-library-oauth2-http - - - com.google.code.findbugs - jsr305 - - - commons-logging - commons-logging - - - org.slf4j - jcl-over-slf4j + io.grpc + grpc-api - com.github.stephenc.findbugs - findbugs-annotations - 1.3.9-1 + io.grpc + grpc-context - com.fasterxml.jackson.core - jackson-databind - + io.grpc + grpc-core + + + io.grpc + grpc-util + diff --git a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml index 5ef800b8ffe6..64568d562d90 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml @@ -27,7 +27,7 @@ pom - 26.25.0 + 26.34.0 From c093ea54b77c816d94a23c7af19f97f96baa7d2f Mon Sep 17 00:00:00 2001 From: tpalfy Date: Tue, 5 Mar 2024 17:48:08 +0100 Subject: [PATCH 39/73] NIFI-12862 When building FlowAnalysisRuleViolationDTO objects (in StandardNiFiServiceFacade), violating component details will be left blank when user has no read permission for that component. NIFI-12862 Expose subjectComponentType to frontend. Signed-off-by: Pierre Villard This closes #8475. --- .../api/dto/FlowAnalysisRuleViolationDTO.java | 13 ++++++++++ .../nifi/web/StandardNiFiServiceFacade.java | 24 ++++++++++++------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleViolationDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleViolationDTO.java index 4e1762f42aa7..0201a0dc24f2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleViolationDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleViolationDTO.java @@ -35,6 +35,8 @@ public FlowAnalysisRuleViolationDTO() { private String issueId; private String violationMessage; + private String subjectComponentType; + private PermissionsDTO subjectPermissionDto; private boolean enabled; @@ -129,6 +131,17 @@ public void setViolationMessage(String violationMessage) { } + /** + * @return the type of the subject that violated the rule + */ + public String getSubjectComponentType() { + return subjectComponentType; + } + + public void setSubjectComponentType(String subjectComponentType) { + this.subjectComponentType = subjectComponentType; + } + /** * @return true if this result should be in effect, false otherwise */ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index 78dbc4caa316..96ec627f6117 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -6478,23 +6478,29 @@ public FlowAnalysisResultEntity createFlowAnalysisResultEntity(Collection { FlowAnalysisRuleViolationDTO ruleViolationDto = new FlowAnalysisRuleViolationDTO(); - ruleViolationDto.setEnforcementPolicy(ruleViolation.getEnforcementPolicy().toString()); + String subjectId = ruleViolation.getSubjectId(); + String groupId = ruleViolation.getGroupId(); + ruleViolationDto.setScope(ruleViolation.getScope()); + ruleViolationDto.setSubjectId(subjectId); ruleViolationDto.setRuleId(ruleViolation.getRuleId()); ruleViolationDto.setIssueId(ruleViolation.getIssueId()); - ruleViolationDto.setViolationMessage(ruleViolation.getViolationMessage()); - String subjectId = ruleViolation.getSubjectId(); - String groupId = ruleViolation.getGroupId(); + ruleViolationDto.setSubjectComponentType(ruleViolation.getSubjectComponentType().name()); + ruleViolationDto.setEnforcementPolicy(ruleViolation.getEnforcementPolicy().toString()); - ruleViolationDto.setSubjectId(subjectId); - ruleViolationDto.setGroupId(groupId); - ruleViolationDto.setSubjectDisplayName(ruleViolation.getSubjectDisplayName()); - ruleViolationDto.setSubjectPermissionDto(createPermissionDto( + PermissionsDTO subjectPermissionDto = createPermissionDto( subjectId, ruleViolation.getSubjectComponentType(), groupId - )); + ); + ruleViolationDto.setSubjectPermissionDto(subjectPermissionDto); + + if (subjectPermissionDto.getCanRead()) { + ruleViolationDto.setGroupId(groupId); + ruleViolationDto.setSubjectDisplayName(ruleViolation.getSubjectDisplayName()); + ruleViolationDto.setViolationMessage(ruleViolation.getViolationMessage()); + } return ruleViolationDto; }) From 69e7a884a3426dd99d7bfaabb014d3a5a2f61a47 Mon Sep 17 00:00:00 2001 From: Scott Aslan Date: Thu, 14 Mar 2024 18:13:10 -0400 Subject: [PATCH 40/73] [NIFI-12822] table, border, theming updates (#8499) * [NIFI-12822] table, border, theming updates * remove nifi-button and replace with mat-icon-button * Update nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss Co-authored-by: James Mingardi-Elliott * Update nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/operation-control.component.scss Co-authored-by: James Mingardi-Elliott * Update nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.scss Co-authored-by: James Mingardi-Elliott * review feedback * final touches --------- Co-authored-by: James Mingardi-Elliott This closes #8499 --- .../policy-table/policy-table.component.html | 2 +- .../component-access-policies.component.html | 6 +- .../component-access-policies.module.ts | 4 +- .../global-access-policies.component.html | 6 +- .../global-access-policies.module.ts | 4 +- .../bulletin-board-list.component.html | 4 +- .../bulletin-board.component.html | 2 +- .../bulletin-board.component.ts | 4 +- .../counter-listing.component.html | 2 +- .../counter-listing/counter-listing.module.ts | 4 +- .../counter-table.component.html | 2 +- ...nfiguration-history-listing.component.html | 4 +- ...configuration-history-table.component.html | 2 +- .../navigation-control.component.html | 10 +- .../navigation-control.component.scss | 6 +- .../navigation-control.component.ts | 3 +- .../operation-control.component.html | 27 +- .../operation-control.component.scss | 6 +- .../operation-control.component.ts | 3 +- .../_new-canvas-item.component-theme.scss | 5 +- .../search/_search.component-theme.scss | 4 +- .../_prioritizers.component-theme.scss | 12 - .../prioritizers/prioritizers.component.html | 8 +- .../prioritizers/prioritizers.component.scss | 19 +- .../import-from-registry.component.html | 2 +- .../controller-services.component.html | 4 +- .../controller-services.module.ts | 4 +- .../manage-remote-ports.component.html | 4 +- .../manage-remote-ports.module.ts | 4 +- ...r-context-inheritance.component-theme.scss | 14 - ...rameter-context-inheritance.component.html | 8 +- ...rameter-context-inheritance.component.scss | 19 +- .../parameter-context-listing.component.html | 4 +- .../parameter-context-listing.module.ts | 4 +- .../parameter-context-table.component.html | 2 +- .../parameter-table.component.html | 186 +++++----- .../parameter-table.component.scss | 21 +- .../provenance-event-table.component.html | 6 +- .../provenance-event-table.component.ts | 4 +- .../flowfile-table.component.html | 2 +- .../queue-listing.component.html | 2 +- .../ui/queue-listing/queue-listing.module.ts | 4 +- .../flow-analysis-rule-table.component.html | 4 +- .../flow-analysis-rule-table.component.scss | 10 +- .../flow-analysis-rules.component.html | 4 +- .../flow-analysis-rules.module.ts | 4 +- ...agement-controller-services.component.html | 4 +- .../management-controller-services.module.ts | 3 +- ...rameter-provider-parameters.component.html | 2 +- .../parameter-groups-table.component.html | 2 +- .../parameter-providers-table.component.html | 2 +- .../parameter-providers.component.html | 4 +- .../parameter-providers.module.ts | 3 +- .../registry-client-table.component.html | 2 +- .../registry-clients.component.html | 4 +- .../registry-clients.module.ts | 10 +- .../reporting-task-table.component.html | 2 +- .../reporting-tasks.component.html | 4 +- .../reporting-tasks/reporting-tasks.module.ts | 4 +- .../cluster-summary-dialog.component.html | 2 +- .../cluster-summary-dialog.component.ts | 5 +- .../connection-cluster-table.component.html | 2 +- .../port-cluster-table.component.html | 2 +- ...process-group-cluster-table.component.html | 2 +- .../processor-cluster-table.component.html | 2 +- ...process-group-cluster-table.component.html | 2 +- .../port-status-table.component.html | 4 +- .../port-status-table.component.ts | 11 +- .../connection-status-table.component.html | 4 +- .../connection-status-table.component.ts | 11 +- .../process-group-status-table.component.html | 4 +- .../process-group-status-table.component.ts | 11 +- .../processor-status-table.component.html | 4 +- .../processor-status-table.component.ts | 11 +- ...-process-group-status-table.component.html | 4 +- ...te-process-group-status-table.component.ts | 11 +- .../user-access-policies.component.html | 2 +- .../user-listing/user-listing.component.html | 2 +- .../ui/user-listing/user-listing.module.ts | 3 +- .../user-table/user-table.component.html | 4 +- .../user-table/user-table.component.ts | 11 +- .../component-state.component.html | 2 +- .../controller-service-table.component.html | 2 +- .../controller-service-table.component.scss | 12 +- .../_extension-creation.component-theme.scss | 41 +-- .../extension-creation.component.html | 6 +- .../extension-creation.component.scss | 19 +- .../_navigation.component-theme.scss | 8 +- .../nf-editor/_nf-editor.component-theme.scss | 2 - .../nf-editor/nf-editor.component.scss | 7 + .../property-table.component.html | 276 +++++++-------- .../property-table.component.scss | 27 +- .../resizable/_resizable.component-theme.scss | 41 --- .../common/resizable/resizable.component.scss | 4 + .../status-history.component.html | 2 +- .../system-diagnostics-dialog.component.html | 2 +- .../src/main/nifi/src/assets/styles/_app.scss | 330 ++++++++++++++++++ ...g-table-theme.scss => _listing-table.scss} | 94 ++--- .../src/main/nifi/src/assets/themes/nifi.scss | 4 +- .../main/nifi/src/assets/themes/purple.scss | 4 +- .../src/main/nifi/src/styles.scss | 325 +---------------- 101 files changed, 907 insertions(+), 926 deletions(-) delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss rename nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/{_listing-table-theme.scss => _listing-table.scss} (87%) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/policy-table/policy-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/policy-table/policy-table.component.html index 8c5a845f2769..60fe975412d1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/policy-table/policy-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/common/policy-table/policy-table.component.html @@ -15,7 +15,7 @@ ~ limitations under the License. -->
-
+
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/component-access-policies.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/component-access-policies.module.ts index 4b058d80243b..94e7b6d48176 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/component-access-policies.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/component-access-policies/component-access-policies.module.ts @@ -27,6 +27,7 @@ import { MatSelectModule } from '@angular/material/select'; import { ComponentAccessPoliciesRoutingModule } from './component-access-policies-routing.module'; import { NifiTooltipDirective } from '../../../../ui/common/tooltips/nifi-tooltip.directive'; import { PolicyTable } from '../common/policy-table/policy-table.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [ComponentAccessPolicies], @@ -41,7 +42,8 @@ import { PolicyTable } from '../common/policy-table/policy-table.component'; ReactiveFormsModule, MatSelectModule, NifiTooltipDirective, - PolicyTable + PolicyTable, + MatButtonModule ] }) export class ComponentAccessPoliciesModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.component.html index c5a0228845b7..c4195adc8d34 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.component.html @@ -108,14 +108,14 @@ @if (flowConfiguration.supportsConfigurableAuthorizer) {
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.module.ts index 8a95b78bfdad..b0dcd699c2c2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/access-policies/ui/global-access-policies/global-access-policies.module.ts @@ -27,6 +27,7 @@ import { MatSelectModule } from '@angular/material/select'; import { GlobalAccessPoliciesRoutingModule } from './global-access-policies-routing.module'; import { NifiTooltipDirective } from '../../../../ui/common/tooltips/nifi-tooltip.directive'; import { PolicyTable } from '../common/policy-table/policy-table.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [GlobalAccessPolicies], @@ -41,7 +42,8 @@ import { PolicyTable } from '../common/policy-table/policy-table.component'; ReactiveFormsModule, MatSelectModule, NifiTooltipDirective, - PolicyTable + PolicyTable, + MatButtonModule ] }) export class GlobalAccessPoliciesModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/bulletin-board-list.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/bulletin-board-list.component.html index f37b82a0410e..ab03775bc2d0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/bulletin-board-list.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board-list/bulletin-board-list.component.html @@ -41,7 +41,7 @@
-
+
    @for (item of bulletinBoardItems; track item) { @@ -72,7 +72,7 @@ } @else { @if (asBulletinEvent(item); as event) {
  • -
    +
    {{ event.message }}
    • diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html index 1b5df94263d1..53569af2f46b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.html @@ -34,7 +34,7 @@ >Auto-refresh
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.ts index d1dfbd371160..8aceb1e11081 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/bulletins/ui/bulletin-board/bulletin-board.component.ts @@ -42,6 +42,7 @@ import { ReactiveFormsModule } from '@angular/forms'; import { BulletinBoardList } from './bulletin-board-list/bulletin-board-list.component'; import { MatSlideToggleChange, MatSlideToggleModule } from '@angular/material/slide-toggle'; import { CommonModule } from '@angular/common'; +import { MatButtonModule } from '@angular/material/button'; @Component({ selector: 'bulletin-board', @@ -55,7 +56,8 @@ import { CommonModule } from '@angular/common'; ReactiveFormsModule, BulletinBoardList, MatSlideToggleModule, - CommonModule + CommonModule, + MatButtonModule ], templateUrl: './bulletin-board.component.html', styleUrls: ['./bulletin-board.component.scss'] diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.component.html index 4209ca0fb764..b6688c0ba771 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.component.html @@ -32,7 +32,7 @@ }
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.module.ts index cf85491aef3b..051c35d43f58 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-listing.module.ts @@ -25,6 +25,7 @@ import { MatSortModule } from '@angular/material/sort'; import { MatInputModule } from '@angular/material/input'; import { ReactiveFormsModule } from '@angular/forms'; import { MatSelectModule } from '@angular/material/select'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [CounterListing, CounterTable], @@ -36,7 +37,8 @@ import { MatSelectModule } from '@angular/material/select'; MatSortModule, MatInputModule, ReactiveFormsModule, - MatSelectModule + MatSelectModule, + MatButtonModule ] }) export class CounterListingModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-table/counter-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-table/counter-table.component.html index 9c5ce0fefa61..7fc5fb32abc1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-table/counter-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/counters/ui/counter-listing/counter-table/counter-table.component.html @@ -38,7 +38,7 @@
-
+
@if ((currentUser$ | async)?.controllerPermissions?.canWrite) {
-
@@ -111,7 +111,7 @@
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/flow-configuration-history-table/flow-configuration-history-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/flow-configuration-history-table/flow-configuration-history-table.component.html index 76962ace1a60..e0b7c33f0f16 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/flow-configuration-history-table/flow-configuration-history-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-configuration-history/ui/flow-configuration-history-listing/flow-configuration-history-table/flow-configuration-history-table.component.html @@ -16,7 +16,7 @@ -->
-
+
- - - -
@if (isNotRootGroup()) { - } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.scss index 6449d8cdccbe..0d8578ea5493 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.scss @@ -18,9 +18,9 @@ div.navigation-control { margin-bottom: 2px; - .fa, - .icon { - font-size: 18px; + .mat-mdc-icon-button { + --mdc-icon-button-state-layer-size: 34px; + --mdc-icon-button-icon-size: 17px; } .navigation-control-header { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.ts index 11bce0878c74..e8013e6cf35c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/navigation-control/navigation-control.component.ts @@ -25,12 +25,13 @@ import { initialState } from '../../../../state/flow/flow.reducer'; import { Storage } from '../../../../../../service/storage.service'; import { Birdseye } from './birdseye/birdseye.component'; +import { MatButtonModule } from '@angular/material/button'; @Component({ selector: 'navigation-control', standalone: true, templateUrl: './navigation-control.component.html', - imports: [Birdseye], + imports: [Birdseye, MatButtonModule], styleUrls: ['./navigation-control.component.scss'] }) export class NavigationControl { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/operation-control.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/operation-control.component.html index 797a696b575d..fdf54a425e5d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/operation-control.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/canvas/graph-controls/operation-control/operation-control.component.html @@ -48,7 +48,7 @@
@if (supportsManagedAuthorizer()) { }
Controller Services - @@ -57,7 +57,7 @@

Controller Services

}
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.module.ts index 1184d8d1d135..f211a8aab736 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/controller-service/controller-services.module.ts @@ -23,6 +23,7 @@ import { ControllerServiceTable } from '../../../../ui/common/controller-service import { ControllerServicesRoutingModule } from './controller-services-routing.module'; import { Breadcrumbs } from '../common/breadcrumbs/breadcrumbs.component'; import { Navigation } from '../../../../ui/common/navigation/navigation.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [ControllerServices], @@ -33,7 +34,8 @@ import { Navigation } from '../../../../ui/common/navigation/navigation.componen ControllerServicesRoutingModule, ControllerServiceTable, Breadcrumbs, - Navigation + Navigation, + MatButtonModule ] }) export class ControllerServicesModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html index a4647849b599..a249621e1bad 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.component.html @@ -51,7 +51,7 @@

Manage Remote Ports

-
+
Manage Remote Ports
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts index 9578ec9ce8ee..bc7eec7da187 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/flow-designer/ui/manage-remote-ports/manage-remote-ports.module.ts @@ -31,6 +31,7 @@ import { EffectsModule } from '@ngrx/effects'; import { ManageRemotePortsEffects } from '../../state/manage-remote-ports/manage-remote-ports.effects'; import { remotePortsFeatureKey } from '../../state/manage-remote-ports'; import { manageRemotePortsReducer } from '../../state/manage-remote-ports/manage-remote-ports.reducer'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [ManageRemotePorts], @@ -46,7 +47,8 @@ import { manageRemotePortsReducer } from '../../state/manage-remote-ports/manage Navigation, MatTableModule, MatSortModule, - NifiTooltipDirective + NifiTooltipDirective, + MatButtonModule ] }) export class ManageRemotePortsModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss index 49e398f2de04..4d5c7f5d85e4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/_parameter-context-inheritance.component-theme.scss @@ -30,32 +30,18 @@ // Get hues from palette $primary-palette-default: mat.get-color-from-palette($primary-palette, default); - $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, A200); - $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); $surface: utils.get-surface($canvas-color-config); $on-surface: utils.get-on-surface($canvas-color-config); - $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); .parameter-context-inheritance { .parameter-context-inheritance-list { background: $primary-palette-default; - - border: solid 1px $canvas-primary-palette-500; - - .cdk-drag-disabled { - background: $on-surface-lighter; - } } .parameter-context-draggable-item { - border-bottom: solid 1px $canvas-primary-palette-500; color: $on-surface; background: $surface; } - - .cdk-drag-preview { - box-shadow: 0 3px 6px $canvas-primary-palette-A200; - } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.html index 893a86794240..a5847bffe5d5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.html @@ -19,7 +19,7 @@
Available Parameter Contexts
@for (item of availableParameterContexts; track item; let i = $index) {
@@ -45,14 +45,14 @@
Selected Parameter Contexts
@for (item of selectedParameterContexts; track item; let i = $index) {
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.scss index be70e461cc37..823fb0e9b774 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-inheritance/parameter-context-inheritance.component.scss @@ -17,14 +17,10 @@ .parameter-context-inheritance { .parameter-context-inheritance-list { - min-height: 66px; + min-height: 74px; border-radius: 4px; overflow: hidden; display: block; - - .cdk-drag-disabled { - cursor: not-allowed; - } } .parameter-context-draggable-item { @@ -41,17 +37,4 @@ border: none; } } - - .cdk-drag-preview { - box-sizing: border-box; - border-radius: 4px; - } - - .cdk-drag-placeholder { - opacity: 0; - } - - .cdk-drop-list-dragging { - cursor: grabbing; - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-listing.component.html index 4ac84182c5dc..8b71ef4949a9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-context-listing.component.html @@ -23,7 +23,7 @@ } @else {
-
@@ -39,7 +39,7 @@
-
-
-
+
+
@if (canAddParameters) {
-
} -
-
- - - - - - - - - - + +
Name -
-
- {{ item.entity.parameter.name }} -
-
- @if (hasDescription(item)) { -
- } -
-
- 		Value - @if (isNull(item.entity.parameter.value)) { -
No value set
- } @else { - - -
Sensitive value set
- - - - - -
Empty string set
- - -
-
- {{ value }} -
- @if (hasExtraWhitespace(value)) { +
+
+ + + + + - + + + - - - - + - + + + + + + + + - - -
Name +
+
+ {{ item.entity.parameter.name }} +
+
+ @if (hasDescription(item)) {
+ [tooltipInputData]="getDescriptionTipData(item)">
}
- - } - 		-
- @if (canGoToParameter(item)) { -
- } - @if (canEdit(item)) { -
- } - @if (canDelete(item)) { -
+ + +
Value + @if (isNull(item.entity.parameter.value)) { +
No value set
+ } @else { + + +
Sensitive value set
+ + + + + +
Empty string set
+ + +
+
+ {{ value }} +
+ @if (hasExtraWhitespace(value)) { +
+ } +
+ } - - 		+
+ @if (canGoToParameter(item)) { +
+ } + @if (canEdit(item)) { +
+ } + @if (canDelete(item)) { +
+ } +
+
+
+
-
+
Parameter
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-table/parameter-table.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-table/parameter-table.component.scss index 345a672e6af6..ed0d2a33b887 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-table/parameter-table.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/parameter-contexts/ui/parameter-context-listing/parameter-table/parameter-table.component.scss @@ -17,20 +17,15 @@ @use '@angular/material' as mat; -.parameter-table.listing-table { - @include mat.table-density(-4); - min-width: 740px; +.parameter-table { + .listing-table { + @include mat.table-density(-4); + width: 506px; - table { - width: auto; - - td, - th { - cursor: default; - } - - .mat-column-actions { - width: 75px; + table { + .mat-column-actions { + width: 75px; + } } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.html index c490cbf3853f..0a0324eec5fb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.html @@ -55,14 +55,14 @@
-
-
+
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.ts index ab2b3a8fe8ca..0b3ae62147a8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/ui/provenance-event-listing/provenance-event-table/provenance-event-table.component.ts @@ -40,6 +40,7 @@ import { MatSliderModule } from '@angular/material/slider'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ErrorBanner } from '../../../../../ui/common/error-banner/error-banner.component'; import { ClusterSummary } from '../../../../../state/cluster-summary'; +import { MatButtonModule } from '@angular/material/button'; @Component({ selector: 'provenance-event-table', @@ -59,7 +60,8 @@ import { ClusterSummary } from '../../../../../state/cluster-summary'; MatPaginatorModule, LineageComponent, MatSliderModule, - ErrorBanner + ErrorBanner, + MatButtonModule ], styleUrls: ['./provenance-event-table.component.scss'] }) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/flowfile-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/flowfile-table.component.html index 26ff0b9bac31..ada74f6eecb3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/flowfile-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/flowfile-table/flowfile-table.component.html @@ -39,7 +39,7 @@

{{ connectionLabel }}

-
+
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.component.html index 022c2043863b..a84fcab85869 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.component.html @@ -36,7 +36,7 @@
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.module.ts index 5e77a8f6ab95..848fe88553ba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/ui/queue-listing/queue-listing.module.ts @@ -27,6 +27,7 @@ import { EffectsModule } from '@ngrx/effects'; import { queueFeatureKey, reducers } from '../../state'; import { QueueListingEffects } from '../../state/queue-listing/queue-listing.effects'; import { ErrorBanner } from '../../../../ui/common/error-banner/error-banner.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [QueueListing], @@ -39,7 +40,8 @@ import { ErrorBanner } from '../../../../ui/common/error-banner/error-banner.com FlowFileTable, StoreModule.forFeature(queueFeatureKey, reducers), EffectsModule.forFeature(QueueListingEffects), - ErrorBanner + ErrorBanner, + MatButtonModule ] }) export class QueueListingModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rule-table/flow-analysis-rule-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rule-table/flow-analysis-rule-table.component.html index 579c18998a5a..a6c21535982c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rule-table/flow-analysis-rule-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rule-table/flow-analysis-rule-table.component.html @@ -15,8 +15,8 @@ ~ limitations under the License. --> -
-
+
+
@if (currentUser.controllerPermissions.canWrite) {
-
@@ -45,7 +45,7 @@
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rules.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rules.module.ts index 43378325804d..6354fac243df 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rules.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/flow-analysis-rules/flow-analysis-rules.module.ts @@ -24,6 +24,7 @@ import { MatSortModule } from '@angular/material/sort'; import { MatTableModule } from '@angular/material/table'; import { NifiTooltipDirective } from '../../../../ui/common/tooltips/nifi-tooltip.directive'; import { PropertyTable } from '../../../../ui/common/property-table/property-table.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [FlowAnalysisRules], @@ -35,7 +36,8 @@ import { PropertyTable } from '../../../../ui/common/property-table/property-tab MatTableModule, NifiTooltipDirective, FlowAnalysisRuleTable, - PropertyTable + PropertyTable, + MatButtonModule ] }) export class FlowAnalysisRulesModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.component.html index daf29d4257d3..61485af17a72 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.component.html @@ -25,7 +25,7 @@
@if (currentUser.controllerPermissions.canWrite) {
-
@@ -49,7 +49,7 @@
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.module.ts index 815f85abb2eb..dc00b3a6a2b1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/management-controller-services/management-controller-services.module.ts @@ -21,10 +21,11 @@ import { ManagementControllerServices } from './management-controller-services.c import { NgxSkeletonLoaderModule } from 'ngx-skeleton-loader'; import { ControllerServiceTable } from '../../../../ui/common/controller-service/controller-service-table/controller-service-table.component'; import { ErrorBanner } from '../../../../ui/common/error-banner/error-banner.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [ManagementControllerServices], exports: [ManagementControllerServices], - imports: [CommonModule, NgxSkeletonLoaderModule, ControllerServiceTable, ErrorBanner] + imports: [CommonModule, NgxSkeletonLoaderModule, ControllerServiceTable, ErrorBanner, MatButtonModule] }) export class ManagementControllerServicesModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html index 48814d22b761..579d6fc0c8ff 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/fetch-parameter-provider-parameters/fetch-parameter-provider-parameters.component.html @@ -202,7 +202,7 @@

Fetch Parameters

-
+
-
+
-
+
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/parameter-providers.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/parameter-providers.module.ts index ec080ddc10d6..5887d4e4045a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/parameter-providers.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/parameter-providers/parameter-providers.module.ts @@ -20,10 +20,11 @@ import { CommonModule } from '@angular/common'; import { ParameterProviders } from './parameter-providers.component'; import { NgxSkeletonLoaderModule } from 'ngx-skeleton-loader'; import { ParameterProvidersTable } from './parameter-providers-table/parameter-providers-table.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [ParameterProviders], exports: [ParameterProviders], - imports: [CommonModule, NgxSkeletonLoaderModule, ParameterProvidersTable] + imports: [CommonModule, NgxSkeletonLoaderModule, ParameterProvidersTable, MatButtonModule] }) export class ParameterProvidersModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-client-table/registry-client-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-client-table/registry-client-table.component.html index 349d942b08a5..1302e6dfcd5d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-client-table/registry-client-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-client-table/registry-client-table.component.html @@ -15,7 +15,7 @@ ~ limitations under the License. --> -
+
@if (currentUser.controllerPermissions.canWrite) {
-
@@ -40,7 +40,7 @@
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-clients.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-clients.module.ts index 020a0c8e4109..3bbd38df7f8f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-clients.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/registry-clients/registry-clients.module.ts @@ -23,10 +23,18 @@ import { RegistryClientTable } from './registry-client-table/registry-client-tab import { MatTableModule } from '@angular/material/table'; import { MatSortModule } from '@angular/material/sort'; import { NifiTooltipDirective } from '../../../../ui/common/tooltips/nifi-tooltip.directive'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [RegistryClients, RegistryClientTable], exports: [RegistryClients], - imports: [CommonModule, NgxSkeletonLoaderModule, MatTableModule, MatSortModule, NifiTooltipDirective] + imports: [ + CommonModule, + NgxSkeletonLoaderModule, + MatTableModule, + MatSortModule, + NifiTooltipDirective, + MatButtonModule + ] }) export class RegistryClientsModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-task-table/reporting-task-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-task-table/reporting-task-table.component.html index b19f5a2fb60d..771c0234bab6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-task-table/reporting-task-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-task-table/reporting-task-table.component.html @@ -15,7 +15,7 @@ ~ limitations under the License. --> -
+
@if (currentUser.controllerPermissions.canWrite) {
-
@@ -46,7 +46,7 @@
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-tasks.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-tasks.module.ts index 7652d8c55dec..0d85cc2e71ba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-tasks.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/settings/ui/reporting-tasks/reporting-tasks.module.ts @@ -25,6 +25,7 @@ import { NifiTooltipDirective } from '../../../../ui/common/tooltips/nifi-toolti import { ReportingTaskTable } from './reporting-task-table/reporting-task-table.component'; import { ControllerServiceTable } from '../../../../ui/common/controller-service/controller-service-table/controller-service-table.component'; import { RouterLink } from '@angular/router'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [ReportingTasks, ReportingTaskTable], @@ -36,7 +37,8 @@ import { RouterLink } from '@angular/router'; MatTableModule, NifiTooltipDirective, ControllerServiceTable, - RouterLink + RouterLink, + MatButtonModule ] }) export class ReportingTasksModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html index d1ca40d32a65..5c87a1fa95cb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.html @@ -72,7 +72,7 @@

Cluster {{ componentType }} Summary

-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.ts index 979887c1a367..7de7c562eda3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/cluster-summary-dialog.component.ts @@ -23,7 +23,7 @@ import { MatDialogContent, MatDialogTitle } from '@angular/material/dialog'; -import { MatButton } from '@angular/material/button'; +import { MatButton, MatButtonModule } from '@angular/material/button'; import { ComponentType, isDefinedAndNotNull } from '../../../../../state/shared'; import { ComponentContext } from '../../../../../ui/common/component-context/component-context.component'; import { @@ -68,7 +68,8 @@ interface Helper { PortClusterTable, RemoteProcessGroupClusterTable, ConnectionClusterTable, - ProcessGroupClusterTable + ProcessGroupClusterTable, + MatButtonModule ], templateUrl: './cluster-summary-dialog.component.html', styleUrl: './cluster-summary-dialog.component.scss' diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.html index c22b86e94947..786742a2805a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/cluster-summary-dialog/connection-cluster-table/connection-cluster-table.component.html @@ -16,7 +16,7 @@ -->
-
+
-
+
-
+
-
+
-
+
-
+
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts index e42698c244f3..72cc0c0fa247 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/common/port-status-table/port-status-table.component.ts @@ -27,13 +27,22 @@ import { NiFiCommon } from '../../../../../service/nifi-common.service'; import { MatPaginatorModule } from '@angular/material/paginator'; import { PortStatusSnapshot, PortStatusSnapshotEntity } from '../../../state'; import { ComponentStatusTable } from '../component-status-table/component-status-table.component'; +import { MatButtonModule } from '@angular/material/button'; export type SupportedColumns = 'name' | 'runStatus' | 'in' | 'out'; @Component({ selector: 'port-status-table', standalone: true, - imports: [CommonModule, SummaryTableFilterModule, MatSortModule, MatTableModule, RouterLink, MatPaginatorModule], + imports: [ + CommonModule, + SummaryTableFilterModule, + MatSortModule, + MatTableModule, + RouterLink, + MatPaginatorModule, + MatButtonModule + ], templateUrl: './port-status-table.component.html', styleUrls: ['./port-status-table.component.scss'] }) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html index 7706656ae83c..1675dd1644d6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.html @@ -31,7 +31,7 @@ (filterChanged)="applyFilter($event)">
-
+
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts index df0e1f83abbb..8300e8c47e9d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/connection-status-listing/connection-status-table/connection-status-table.component.ts @@ -27,13 +27,22 @@ import { RouterLink } from '@angular/router'; import { MatPaginatorModule } from '@angular/material/paginator'; import { ConnectionStatusSnapshot, ConnectionStatusSnapshotEntity } from '../../../state'; import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; +import { MatButtonModule } from '@angular/material/button'; export type SupportedColumns = 'name' | 'queue' | 'in' | 'out' | 'threshold' | 'sourceName' | 'destinationName'; @Component({ selector: 'connection-status-table', standalone: true, - imports: [CommonModule, SummaryTableFilterModule, MatSortModule, RouterLink, MatTableModule, MatPaginatorModule], + imports: [ + CommonModule, + SummaryTableFilterModule, + MatSortModule, + RouterLink, + MatTableModule, + MatPaginatorModule, + MatButtonModule + ], templateUrl: './connection-status-table.component.html', styleUrls: ['./connection-status-table.component.scss'] }) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html index 362e65487ce4..bd1dd3e91826 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.html @@ -30,7 +30,7 @@ (filterChanged)="applyFilter($event)">
-
+
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts index 99fc53ed288a..e5e48f3d1747 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/process-group-status-listing/process-group-status-table/process-group-status-table.component.ts @@ -26,6 +26,7 @@ import { RouterLink } from '@angular/router'; import { MatPaginatorModule } from '@angular/material/paginator'; import { ProcessGroupStatusSnapshot, ProcessGroupStatusSnapshotEntity } from '../../../state'; import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; +import { MatButtonModule } from '@angular/material/button'; export type SupportedColumns = | 'name' @@ -42,7 +43,15 @@ export type SupportedColumns = @Component({ selector: 'process-group-status-table', standalone: true, - imports: [CommonModule, MatSortModule, MatTableModule, SummaryTableFilterModule, RouterLink, MatPaginatorModule], + imports: [ + CommonModule, + MatSortModule, + MatTableModule, + SummaryTableFilterModule, + RouterLink, + MatPaginatorModule, + MatButtonModule + ], templateUrl: './process-group-status-table.component.html', styleUrls: ['./process-group-status-table.component.scss'] }) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html index 833c487863b8..d1372ef4de58 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.html @@ -30,7 +30,7 @@ (filterChanged)="applyFilter($event)">
-
+
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts index f83e659039fb..0129643d9e82 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/processor-status-listing/processor-status-table/processor-status-table.component.ts @@ -27,6 +27,7 @@ import { NiFiCommon } from '../../../../../service/nifi-common.service'; import { MatPaginatorModule } from '@angular/material/paginator'; import { ProcessorStatusSnapshot, ProcessorStatusSnapshotEntity } from '../../../state'; import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; +import { MatButtonModule } from '@angular/material/button'; export type SupportedColumns = 'name' | 'type' | 'processGroup' | 'runStatus' | 'in' | 'out' | 'readWrite' | 'tasks'; @@ -35,7 +36,15 @@ export type SupportedColumns = 'name' | 'type' | 'processGroup' | 'runStatus' | templateUrl: './processor-status-table.component.html', styleUrls: ['./processor-status-table.component.scss'], standalone: true, - imports: [RouterLink, SummaryTableFilterModule, MatTableModule, MatSortModule, NgClass, MatPaginatorModule] + imports: [ + RouterLink, + SummaryTableFilterModule, + MatTableModule, + MatSortModule, + NgClass, + MatPaginatorModule, + MatButtonModule + ] }) export class ProcessorStatusTable extends ComponentStatusTable { filterableColumns: SummaryTableFilterColumn[] = [ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html index a2754c9d7cf7..32167af54f6a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.html @@ -31,7 +31,7 @@ (filterChanged)="applyFilter($event)">
-
+
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts index 554c564bcb77..9d8a3d48c250 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/summary/ui/remote-process-group-status-listing/remote-process-group-status-table/remote-process-group-status-table.component.ts @@ -27,13 +27,22 @@ import { RouterLink } from '@angular/router'; import { MatPaginatorModule } from '@angular/material/paginator'; import { RemoteProcessGroupStatusSnapshot, RemoteProcessGroupStatusSnapshotEntity } from '../../../state'; import { ComponentStatusTable } from '../../common/component-status-table/component-status-table.component'; +import { MatButtonModule } from '@angular/material/button'; export type SupportedColumns = 'name' | 'uri' | 'transmitting' | 'sent' | 'received'; @Component({ selector: 'remote-process-group-status-table', standalone: true, - imports: [CommonModule, SummaryTableFilterModule, MatSortModule, MatTableModule, RouterLink, MatPaginatorModule], + imports: [ + CommonModule, + SummaryTableFilterModule, + MatSortModule, + MatTableModule, + RouterLink, + MatPaginatorModule, + MatButtonModule + ], templateUrl: './remote-process-group-status-table.component.html', styleUrls: ['./remote-process-group-status-table.component.scss'] }) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html index fc28e7178c45..7074af5b8448 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-access-policies/user-access-policies.component.html @@ -24,7 +24,7 @@

User Policies

{{ request.identity }}
-
+
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-listing.module.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-listing.module.ts index 5d07bd3029c5..5c05c1a6fddb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-listing.module.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-listing.module.ts @@ -20,10 +20,11 @@ import { UserListing } from './user-listing.component'; import { CommonModule } from '@angular/common'; import { NgxSkeletonLoaderModule } from 'ngx-skeleton-loader'; import { UserTable } from './user-table/user-table.component'; +import { MatButtonModule } from '@angular/material/button'; @NgModule({ declarations: [UserListing], exports: [UserListing], - imports: [CommonModule, NgxSkeletonLoaderModule, UserTable] + imports: [CommonModule, NgxSkeletonLoaderModule, UserTable, MatButtonModule] }) export class UserListingModule {} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-table/user-table.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-table/user-table.component.html index 960c36d33abb..9757d724fab2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-table/user-table.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/users/ui/user-listing/user-table/user-table.component.html @@ -40,14 +40,14 @@
@if (canModifyTenants(currentUser)) { - }
-
+
Component State } -
+
-
+
Add {{ componentType }}Filter types -
-
+
+
-
+
Required field
-
-
-
- - - - + +
Property -
-
- {{ item.descriptor.displayName }} +
+
+ + + + + - + + - - - - + - + + } + + + - - - - - + + + + + - - -
Property +
+
+ {{ item.descriptor.displayName }} +
+
+ @if (hasInfo(item.descriptor)) { +
+ } +
-
- @if (hasInfo(item.descriptor)) { -
- } -
- - 		Value -
- @if (isNull(item.value)) { -
No value set
- } @else { - - -
Sensitive value set
- - + + +
Value +
+ @if (isNull(item.value)) { +
No value set
+ } @else { - - -
Empty string set
- - -
-
- {{ resolvedValue }} + +
Sensitive value set
+ + + + + +
Empty string set
+ + +
+
+ {{ resolvedValue }} +
+ @if (hasExtraWhitespace(resolvedValue)) { +
+ }
- @if (hasExtraWhitespace(resolvedValue)) { -
- } -
- - } -
-
-
- @if (item.descriptor.identifiesControllerService) { -
- } - @if (canConvertToParameter(item)) { -
- } - @if (canGoToParameter(item)) { -
- } - @if (canGoToService(item)) { -
- } - @if (item.type == 'userDefined') { -
- } -
- 		+
+ @if (item.descriptor.identifiesControllerService) { +
+ } + @if (canConvertToParameter(item)) { +
+ } + @if (canGoToParameter(item)) { +
+ } + @if (canGoToService(item)) { +
+ } + @if (item.type == 'userDefined') { +
+ } +
+
- - @if (hasAllowableValues(editorItem)) { - - } @else { - - } - +
+ + @if (hasAllowableValues(editorItem)) { + + } @else { + + } + +
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/property-table.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/property-table.component.scss index 67d151216e13..9fbb08d6d185 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/property-table.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/property-table/property-table.component.scss @@ -17,28 +17,9 @@ @use '@angular/material' as mat; -.property-table.listing-table { - @include mat.table-density(-4); - min-width: 740px; - - table { - width: auto; - - td, - th { - cursor: default; - } - - .mat-column-property { - min-width: 230px; - } - - .mat-column-value { - min-width: 230px; - } - - .mat-column-actions { - min-width: 50px; - } +.property-table { + .listing-table { + @include mat.table-density(-4); + width: 740px; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss deleted file mode 100644 index 3f2efdb8b7aa..000000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/_resizable.component-theme.scss +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -@use 'sass:math' as math; -@use 'sass:map'; -@use '@angular/material' as mat; -@use '../../../../assets/utils.scss' as utils; - -@mixin nifi-theme($material-theme, $canvas-theme) { - // Get the color config from the theme. - $color-config: mat.get-color-config($material-theme); - $canvas-color-config: mat.get-color-config($canvas-theme); - - // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); - - // Get hues from palette - $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); - - $handle-size: 15px; - $handle-color: $on-surface-lighter; - - .resizable-triangle { - border-right: math.div($handle-size, 2) solid $handle-color; - border-bottom: math.div($handle-size, 2) solid $handle-color; - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/resizable.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/resizable.component.scss index ad1a0c770979..0c3c3ddf0d35 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/resizable.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/resizable/resizable.component.scss @@ -41,6 +41,10 @@ $handle-size: 15px; height: 0; border-left: math.div($handle-size, 2) solid transparent; border-top: math.div($handle-size, 2) solid transparent; + border-right-width: math.div($handle-size, 2); + border-right-style: solid; + border-bottom-width: math.div($handle-size, 2); + border-bottom-style: solid; bottom: 0; right: 0; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html index a5e980cb9b66..f35271263333 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/status-history/status-history.component.html @@ -161,7 +161,7 @@

Status History

@if (instances.length > 0) {
-
Last updated:
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss new file mode 100644 index 000000000000..f5223e339594 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss @@ -0,0 +1,330 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@use 'sass:map'; +@use '@angular/material' as mat; +@use '../utils.scss' as utils; + +@mixin styles() { + html, + body { + height: 100%; + } + + body { + margin: 0; + font-family: Roboto, 'Helvetica Neue', sans-serif; + } + + a { + font-size: 13px; + cursor: pointer; + font-weight: normal; + display: inline-block; + font-family: Roboto; + text-decoration: underline; + text-underline-offset: 3px; + } + + //Icon for styling mat-icon in forms throughout the application. + .info-icon { + font-size: 14px; + height: 14px !important; + width: 14px !important; + } + + //This style is needed due to an incompatibility between material and tailwind. + .mdc-notched-outline__notch { + border-right: none; + } + + .small-dialog { + min-width: 320px; + } + + .medium-dialog { + min-width: 470px; + } + + .medium-short-dialog { + max-height: 32%; + max-width: 34%; + min-height: 250px; + min-width: 440px; + font-size: 13px; + } + + .large-dialog { + max-height: 72%; + max-width: 55%; + min-height: 520px; + min-width: 760px; + } + + .xl-dialog { + max-height: 72%; + max-width: 85%; + min-height: 560px; + min-width: 1024px; + } + + .edit-parameter-context-dialog { + max-height: 72%; + max-width: 55%; + min-width: 850px; + min-height: 575px; + } + + .tooltip { + position: fixed; + max-width: 500px; + padding: 10px; + border-radius: 2px; + border-width: 1px; + font-size: 13px; + font-family: Roboto; + font-weight: 400; + word-wrap: break-word; + white-space: normal; + z-index: 1; + + ul { + list-style: disc outside; + margin-left: 1em; + } + } + + .property-editor { + font-size: 13px; + font-family: Roboto; + font-weight: 400; + } + + .CodeMirror-hints { + font-size: 13px !important; + z-index: 1000 !important; + overflow-y: scroll !important; + } + + .blank, + .unset, + .sensitive { + font-weight: normal !important; + } + + .mat-mdc-icon-button { + --mdc-icon-button-state-layer-size: 28px; + --mdc-icon-button-icon-size: 14px; + line-height: var(--mdc-icon-button-state-layer-size); + padding: 0; + + &.mat-mdc-button-base.mdc-icon-button { + padding: 0; + } + } + + .pointer { + cursor: pointer; + } + + .disabled { + cursor: not-allowed; + } + + .value, + .refresh-timestamp { + font-weight: 500; + } +} + +@mixin colors($material-theme, $canvas-theme) { + // Get the color config from the theme. + $color-config: mat.get-color-config($material-theme); + $canvas-color-config: mat.get-color-config($canvas-theme); + + // Get the color palette from the color-config. + $primary-palette: map.get($color-config, 'primary'); + $accent-palette: map.get($color-config, 'accent'); + $warn-palette: map.get($color-config, 'warn'); + $canvas-primary-palette: map.get($canvas-color-config, 'primary'); + $canvas-accent-palette: map.get($canvas-color-config, 'accent'); + + // Get hues from palette + + // Start with the canvas theme. + $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, A200); + $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); + $canvas-accent-palette-lighter: mat.get-color-from-palette($canvas-accent-palette, lighter); + $canvas-accent-palette-default: mat.get-color-from-palette($canvas-accent-palette, default); + + $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); + $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); + $primary-palette-A400: mat.get-color-from-palette($primary-palette, 'A400'); + + $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); + $accent-palette-lighter: mat.get-color-from-palette($accent-palette, 'lighter'); + + $warn-palette-lighter: mat.get-color-from-palette($warn-palette, lighter); + $warn-palette-default: mat.get-color-from-palette($warn-palette, 'default'); + + // Alternative hue for warn colors. + $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); + + $surface: utils.get-surface($canvas-color-config); + $surface-darker: utils.get-surface($canvas-color-config, darker); + $surface-highlight: utils.get-on-surface($canvas-color-config, highlight); + $on-surface: utils.get-on-surface($canvas-color-config); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + + * { + // Tailwind sets a default that doesn't shift with light and dark themes + border-color: $on-surface-lighter; + } + + .cdk-drag-disabled { + cursor: not-allowed !important; + background: $on-surface-lighter !important; + } + + .cdk-drag-preview { + box-sizing: border-box; + border-radius: 4px; + box-shadow: 0 3px 6px $canvas-primary-palette-A200; + } + + .cdk-drag-placeholder { + opacity: 0; + } + + .cdk-drop-list-dragging { + cursor: grabbing; + } + + a { + color: utils.get-color-on-surface($color-config, $surface); + text-decoration-color: $primary-palette-lighter; + } + + a:hover { + text-decoration-color: utils.get-color-on-surface($color-config, $surface); + } + + .tooltip { + background-color: $surface; + border-color: $on-surface; + box-shadow: 0 2px 5px $canvas-primary-palette-A200; + color: $on-surface; + } + + .property-editor { + background-color: $surface; + box-shadow: 0 2px 5px $canvas-primary-palette-A200; + } + + .disabled { + color: $primary-palette-default !important; + fill: $primary-palette-default !important; + } + + .enabled { + color: $primary-palette-A400 !important; + fill: $primary-palette-A400 !important; + } + + .stopped { + color: $warn-palette-lighter !important; + fill: $warn-palette-lighter !important; + } + + .running { + color: $canvas-accent-palette-lighter !important; + fill: $canvas-accent-palette-lighter !important; + } + + .has-errors, + .invalid { + color: $warn-palette-A200 !important; + fill: $warn-palette-A200 !important; + } + + .validating { + color: $canvas-primary-palette-500 !important; + fill: $canvas-primary-palette-500 !important; + } + + .transmitting { + color: $canvas-accent-palette-default !important; + fill: $canvas-accent-palette-default !important; + } + + .up-to-date { + color: $canvas-accent-palette-default !important; + fill: $canvas-accent-palette-default !important; + } + + .locally-modified, + .sync-failure { + color: $accent-palette-lighter !important; + fill: $accent-palette-lighter !important; + } + + .stale, + .locally-modified-and-stale { + color: $warn-palette-default !important; + fill: $warn-palette-default !important; + } + + .zero { + opacity: 0.5; + } + + .blank, + .unset, + .sensitive { + color: $canvas-primary-palette-500 !important; + } + + .mat-mdc-icon-button { + color: utils.get-color-on-surface($color-config, $surface-darker) !important; + } + + .mat-mdc-icon-button:hover { + background-color: $surface-highlight !important; + } + + .mat-mdc-icon-button:disabled { + color: $on-surface-lighter !important; + background-color: transparent !important; + + i { + color: $on-surface-lighter !important; + } + } + + .value, + .refresh-timestamp { + color: utils.get-color-on-surface($color-config, $surface, 'accent'); + } + + h3.page-header { + color: $primary-palette-default; + } + + ngx-skeleton-loader .skeleton-loader { + background: $on-surface-lighter; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table.scss similarity index 87% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss rename to nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table.scss index 7ff0b6488fd4..1fe69b156b24 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_listing-table.scss @@ -17,9 +17,53 @@ @use 'sass:map'; @use '@angular/material' as mat; -@use '../../assets/utils.scss' as utils; +@use '../utils.scss' as utils; -@mixin nifi-theme($material-theme, $canvas-theme) { +@mixin styles() { + .listing-table { + border-width: 1px; + border-style: solid; + + table { + width: 100%; + table-layout: fixed; + + td, + th { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + padding: 0 8px; + cursor: default; + } + + .fa { + width: 10px; + height: 14px; + text-align: center; + } + + .icon { + width: 10px; + text-align: center; + } + + .mat-column-moreDetails { + min-width: 30px; + } + + .mat-column-actions { + min-width: 115px; + } + } + } + + .mat-sort-header-content { + overflow: hidden; + } +} + +@mixin colors($material-theme, $canvas-theme) { // Get the color config from the theme. $color-config: mat.get-color-config($material-theme); $canvas-color-config: mat.get-color-config($canvas-theme); @@ -42,21 +86,12 @@ $primary-palette-darker-contrast ); $surface-highlight: utils.get-on-surface($canvas-color-config, 'highlight'); + $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); .listing-table { - table { - width: 100%; - table-layout: fixed; - - td, - th { - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - padding: 0 8px; - cursor: default; - } + border-color: $on-surface-lighter; + table { th { background-color: $header-surface !important; color: $header-on-surface; @@ -67,6 +102,12 @@ } } + th, + td { + border-bottom-width: 1px; + border-bottom-style: solid; + } + tr:hover { background-color: $surface-highlight !important; } @@ -81,36 +122,11 @@ .fa { color: utils.get-color-on-surface($color-config, $surface); - width: 10px; - height: 14px; - text-align: center; } .icon { color: utils.get-color-on-surface($color-config, $surface); - width: 10px; - text-align: center; - } - - .mat-column-moreDetails { - min-width: 30px; } - - .mat-column-actions { - min-width: 115px; - } - } - - .mat-mdc-table .mdc-data-table__header-row { - height: 35px; - } - - .mat-mdc-table .mdc-data-table__row { - height: 35px; } } - - .mat-sort-header-content { - overflow: hidden; - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss index d1b925e53859..5f71d72017ab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/nifi.scss @@ -29,7 +29,7 @@ $material-primary-light-palette: ( // 50 -> 900 are the PRIMARY colors (mat.define-palette($material-primary-light-palette, 300);) defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b 50: rgba(#F9FAFB, 0.97), 100: #e5ebed, // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, - 200: #cbd8dd, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a, button.nifi-button, button.nifi-button:disabled + 200: #cbd8dd, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a 300: #abbdc5, // .breadcrumb-container, .navigation-control, .operation-control, .flow-status, .controller-bulletins, .component-button-grip, .search-container, .nifi-navigation, .CodeMirror.blank 400: #8aa2ad, // Default hue for this palette (color="primary"). 500: #728e9b, // .disabled, .not-transmitting, .splash, .access-policies-header, .operation-context-type, .bulletin-board-header, .counter-header, .stats-info, .active-thread-count-icon, .processor-type, .port-transmission-icon, .operation-context-type, .flow-status.fa, .flow-status.icon, .controller-bulletins, .prioritizers-list, .controller-services-header, .login-title, .parameter-context-header, .parameter-context-inheritance-list, .provenance-header, .flowfile-header, .queue-listing-header, .settings-header, .summary-header, .user-header, table th, button.global-menu-item.fa, button.global-menu-item.icon, .event-header, .section-header, @@ -43,7 +43,7 @@ $material-primary-light-palette: ( A100: rgba(#1491C1, 0.12), // .hint-pattern A200: #aabec7, // .zero A400: #44a3cf, // .enabled, .transmitting, .load-balance-icon-active - A700: #004849, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu + A700: #004849, // a, a:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu // These are the $material-primary-light-palette PRIMARY AND ACCENT contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important // for light mode and dark mode as these values set the colors for the text when displayed against the primary background on a button, badge, chip, etc. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss index 075c973468ad..84be81e597a2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/themes/purple.scss @@ -29,7 +29,7 @@ $material-primary-light-palette: ( // 50 -> 900 are the PRIMARY colors (mat.define-palette($material-primary-light-palette, 300);) defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors for primary color #728e9b 50: #f0e7f2, // .context-menu 100: #dac3e0, // "lighter" hue for this palette. Also .global-menu:hover, .navigation-control-header:hover, .operation-control-header:hover, .new-canvas-item.icon.hovering, table tr:hover, .CodeMirror.blank, .remote-banner, .process-group-details-banner, .process-group-details-banner, remote-process-group-details-banner, .remote-process-group-last-refresh-rect, - 200: #c29dcc, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a, button.nifi-button, button.nifi-button:disabled + 200: #c29dcc, // .processor-stats-border, .process-group-stats-border, .context-menu-item:hover, .process-group-banner, .remote-process-group-banner, .a 300: #aa79b7, // .breadcrumb-container, .navigation-control, .operation-control, .flow-status, .controller-bulletins, .component-button-grip, .search-container, .nifi-navigation, .CodeMirror.blank 400: #985fa7, // Default hue for this palette (color="primary"). 500: #874b98, // .disabled, .not-transmitting, .splash, .access-policies-header, .operation-context-type, .bulletin-board-header, .counter-header, .stats-info, .active-thread-count-icon, .processor-type, .port-transmission-icon, .operation-context-type, .flow-status.fa, .flow-status.icon, .controller-bulletins, .prioritizers-list, .controller-services-header, .login-title, .parameter-context-header, .parameter-context-inheritance-list, .provenance-header, .flowfile-header, .queue-listing-header, .settings-header, .summary-header, .user-header, table th, button.global-menu-item.fa, button.global-menu-item.icon, .event-header, .section-header, @@ -43,7 +43,7 @@ $material-primary-light-palette: ( A100: rgba(20, 145, 193, 0.12), // .hint-pattern A200: #aabec7, // .zero A400: #44a3cf, // .enabled, .transmitting, .load-balance-icon-active - A700: #004849, // a, a:hover, button.nifi-button, button.nifi-button:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu + A700: #004849, // a, a:hover, .add-tenant-to-policy-form.fa, .component.selected rect.border, .add-connect, .remote-process-group-uri, .remote-process-group-transmission-secure, .navigation-control.fa, .operation-control.fa, .new-canvas-item.icon, .upload-flow-definition, .lineage-controls.fa, .event circle.context, .nifi-navigation.icon, .listing-table.fa, .listing-table.icon, .context-menu // These are the $material-primary-light-palette PRIMARY AND ACCENT contrast colors. These color do not really get defined by https://m2.material.io/design/color/the-color-system.html#tools-for-picking-colors. // Instead if we look to the Angular Material provided palettes we see that these fields are typically rgba(black, 0.87) or white. These values are particularly important diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss index 486bc6a41ce5..188168ac550d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss @@ -19,7 +19,8 @@ // For more information: https://v16.material.angular.io/guide/theming-your-components @use '@angular/material' as mat; @use 'sass:map'; -@use 'assets/styles/listing-table-theme' as listing-table; +@use 'assets/styles/app' as app; +@use 'assets/styles/listing-table' as listing-table; @use 'app/app.component-theme' as app-component; @use 'app/pages/access-policies/feature/access-policies.component-theme' as access-policies; @use 'app/pages/access-policies/ui/common/add-tenant-to-policy-dialog/add-tenant-to-policy-dialog.component-theme' as add-tenant-to-policy; @@ -58,7 +59,6 @@ @use 'app/ui/common/extension-creation/extension-creation.component-theme' as extension-creation; @use 'app/ui/common/navigation/navigation.component-theme' as navigation; @use 'app/ui/common/property-table/editors/nf-editor/nf-editor.component-theme' as nf-editor; -@use 'app/ui/common/resizable/resizable.component-theme' as resizable; @use 'app/ui/common/system-diagnostics-dialog/system-diagnostics-dialog.component-theme' as system-diagnostics-dialog; @use 'app/ui/common/status-history/status-history.component-theme' as status-history; @use 'app/ui/common/tooltips/property-hint-tip/property-hint-tip.component-theme' as property-hint-tip; @@ -73,10 +73,6 @@ @use 'codemirror/lib/codemirror.css'; @use 'codemirror/addon/hint/show-hint.css'; -@use 'assets/utils.scss' as utils; - -// NOTE: for faster developer cycles during theme development the theme can also be changed here. -// Ex: @import 'assets/themes/purple'; // To override the canvas theme, you need to set the variables $nifi-canvas-theme-light and $nifi-canvas-theme-dark @import 'assets/themes/nifi-canvas'; // To override the NiFi theme, you need to set the variables $material-theme-light and $material-theme-dark @@ -178,310 +174,15 @@ $appFontPath: '~roboto-fontface/fonts'; @tailwind components; @tailwind utilities; -//General status styles. TODO - Reconsider this... separating canvas/style styles from html styles... -@mixin nifi-colors($material-theme, $canvas-theme) { - // Get the color config from the theme. - $color-config: mat.get-color-config($material-theme); - $canvas-color-config: mat.get-color-config($canvas-theme); - - // Get the color palette from the color-config. - $primary-palette: map.get($color-config, 'primary'); - $accent-palette: map.get($color-config, 'accent'); - $warn-palette: map.get($color-config, 'warn'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); - $canvas-accent-palette: map.get($canvas-color-config, 'accent'); - - // Get hues from palette - - // Start with the canvas theme. - $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, A200); - $canvas-primary-palette-400: mat.get-color-from-palette($canvas-primary-palette, 400); - $canvas-primary-palette-500: mat.get-color-from-palette($canvas-primary-palette, 500); - $canvas-accent-palette-lighter: mat.get-color-from-palette($canvas-accent-palette, lighter); - $canvas-accent-palette-default: mat.get-color-from-palette($canvas-accent-palette, default); - - $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); - $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); - $primary-palette-A400: mat.get-color-from-palette($primary-palette, 'A400'); - - $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); - $accent-palette-lighter: mat.get-color-from-palette($accent-palette, 'lighter'); - - $warn-palette-lighter: mat.get-color-from-palette($warn-palette, lighter); - $warn-palette-default: mat.get-color-from-palette($warn-palette, 'default'); - - // Alternative hue for warn colors. - $warn-palette-A200: mat.get-color-from-palette($warn-palette, 'A200'); - - $surface: utils.get-surface($canvas-color-config); - $surface-darker: utils.get-surface($canvas-color-config, darker); - $surface-highlight: utils.get-on-surface($canvas-color-config, highlight); - $on-surface: utils.get-on-surface($canvas-color-config); - $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); - - * { // Tailwind sets a default that doesn't shift with light and dark themes - border-color: $on-surface-lighter; - } - - a { - color: utils.get-color-on-surface($color-config, $surface); - text-decoration-color: $primary-palette-lighter; - } - - a:hover { - text-decoration-color: utils.get-color-on-surface($color-config, $surface); - } - - .tooltip { - background-color: $surface; - border-color: $on-surface; - box-shadow: 0 2px 5px $canvas-primary-palette-A200; - color: $on-surface; - } - - .property-editor { - background-color: $surface; - box-shadow: 0 2px 5px $canvas-primary-palette-A200; - } - - .disabled { - color: $primary-palette-default !important; - fill: $primary-palette-default !important; - } - - .enabled { - color: $primary-palette-A400 !important; - fill: $primary-palette-A400 !important; - } - - .stopped { - color: $warn-palette-lighter !important; - fill: $warn-palette-lighter !important; - } - - .running { - color: $canvas-accent-palette-lighter !important; - fill: $canvas-accent-palette-lighter !important; - } - - .has-errors, - .invalid { - color: $warn-palette-A200 !important; - fill: $warn-palette-A200 !important; - } - - .validating { - color: $canvas-primary-palette-500 !important; - fill: $canvas-primary-palette-500 !important; - } - - .transmitting { - color: $canvas-accent-palette-default !important; - fill: $canvas-accent-palette-default !important; - } - - .up-to-date { - color: $canvas-accent-palette-default !important; - fill: $canvas-accent-palette-default !important; - } - - .locally-modified, - .sync-failure { - color: $accent-palette-lighter !important; - fill: $accent-palette-lighter !important; - } - - .stale, - .locally-modified-and-stale { - color: $warn-palette-default !important; - fill: $warn-palette-default !important; - } - - .zero { - opacity: 0.5; - } - - .blank, - .unset, - .sensitive { - color: $canvas-primary-palette-500 !important; - } - - button.nifi-button { - color: utils.get-color-on-surface($color-config, $surface-darker); - } - - button.nifi-button:hover { - background-color: $surface-highlight; - } - - button.nifi-button:disabled { - color: $on-surface-lighter; - background-color: transparent; - cursor: not-allowed; - - i { - color: $on-surface-lighter; - } - } - - .value, - .refresh-timestamp { - color: utils.get-color-on-surface($color-config, $surface, 'accent'); - } - - .accent-palette-default { - color: $accent-palette-default; - } - - ngx-skeleton-loader .skeleton-loader { - background: $canvas-primary-palette-400; - } -} - -@mixin nifi-styles() { - html, - body { - height: 100%; - } - - body { - margin: 0; - font-family: Roboto, 'Helvetica Neue', sans-serif; - } - - a { - font-size: 13px; - cursor: pointer; - font-weight: normal; - display: inline-block; - font-family: Roboto; - text-decoration: underline; - text-underline-offset: 3px; - } - - //Icon for styling mat-icon in forms throughout the application. - .info-icon { - font-size: 14px; - height: 14px !important; - width: 14px !important; - } - - //This style is needed due to an incompatibility between material and tailwind. - .mdc-notched-outline__notch { - border-right: none; - } - - .small-dialog { - min-width: 320px; - } - - .medium-dialog { - min-width: 470px; - } - - .medium-short-dialog { - max-height: 32%; - max-width: 34%; - min-height: 250px; - min-width: 440px; - font-size: 13px; - } - - .large-dialog { - max-height: 72%; - max-width: 55%; - min-height: 520px; - min-width: 760px; - } - - .xl-dialog { - max-height: 72%; - max-width: 85%; - min-height: 560px; - min-width: 1024px; - } - - .edit-parameter-context-dialog { - max-height: 72%; - max-width: 55%; - min-width: 850px; - min-height: 575px; - } - - .tooltip { - position: fixed; - max-width: 500px; - padding: 10px; - border-radius: 2px; - border-width: 1px; - font-size: 13px; - font-family: Roboto; - font-weight: 400; - word-wrap: break-word; - white-space: normal; - z-index: 1; - - ul { - list-style: disc outside; - margin-left: 1em; - } - } - - .property-editor { - font-size: 13px; - font-family: Roboto; - font-weight: 400; - } - - .CodeMirror-hints { - font-size: 13px !important; - z-index: 1000 !important; - overflow-y: scroll !important; - } - - .blank, - .unset, - .sensitive { - font-weight: normal !important; - } - - button.nifi-button { - height: 28px; - width: 28px; - line-height: 28px; - } - - button.nifi-button:hover { - cursor: pointer; - } - - button.nifi-button:disabled { - cursor: not-allowed; - } - - .pointer { - cursor: pointer; - } - - .disabled { - cursor: not-allowed; - } - - .value, - .refresh-timestamp { - font-weight: 500; - } -} - // only include this once (not needed for dark mode) -@include nifi-styles(); +@include app.styles(); +@include listing-table.styles(); -// generate light mode nifi stylesheets +// generate light mode stylesheets @include mat.all-component-themes($material-theme-light); @include app-component.nifi-theme($material-theme-light, $nifi-canvas-theme-light); -@include nifi-colors($material-theme-light, $nifi-canvas-theme-light); -@include listing-table.nifi-theme($material-theme-light, $nifi-canvas-theme-light); +@include app.colors($material-theme-light, $nifi-canvas-theme-light); +@include listing-table.colors($material-theme-light, $nifi-canvas-theme-light); @include access-policies.nifi-theme($material-theme-light); @include add-tenant-to-policy.nifi-theme($material-theme-light); @include component-access-policies.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @@ -499,7 +200,7 @@ $appFontPath: '~roboto-fontface/fonts'; @include flow-status.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include header.nifi-theme($nifi-canvas-theme-light); @include new-canvas-item.nifi-theme($material-theme-light, $nifi-canvas-theme-light); -@include search.nifi-theme($material-theme-light, $nifi-canvas-theme-light); +@include search.nifi-theme($nifi-canvas-theme-light); @include prioritizers.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include create-process-group.nifi-theme($material-theme-light); @include create-remote-process-group.nifi-theme($material-theme-light); @@ -519,7 +220,6 @@ $appFontPath: '~roboto-fontface/fonts'; @include extension-creation.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include navigation.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include nf-editor.nifi-theme($material-theme-light, $nifi-canvas-theme-light); -@include resizable.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include system-diagnostics-dialog.nifi-theme($material-theme-light); @include status-history.nifi-theme($material-theme-light, $nifi-canvas-theme-light); @include property-hint-tip.nifi-theme($material-theme-light); @@ -531,10 +231,10 @@ $appFontPath: '~roboto-fontface/fonts'; // Include the dark theme color styles. @include mat.all-component-colors($material-theme-dark); - // generate dark mode nifi stylesheets + // generate dark mode stylesheets @include app-component.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); - @include nifi-colors($material-theme-dark, $nifi-canvas-theme-dark); - @include listing-table.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); + @include app.colors($material-theme-dark, $nifi-canvas-theme-dark); + @include listing-table.colors($material-theme-dark, $nifi-canvas-theme-dark); @include access-policies.nifi-theme($material-theme-dark); @include add-tenant-to-policy.nifi-theme($material-theme-dark); @include component-access-policies.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @@ -552,7 +252,7 @@ $appFontPath: '~roboto-fontface/fonts'; @include flow-status.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include header.nifi-theme($nifi-canvas-theme-dark); @include new-canvas-item.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); - @include search.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); + @include search.nifi-theme($nifi-canvas-theme-dark); @include prioritizers.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include create-process-group.nifi-theme($material-theme-dark); @include create-remote-process-group.nifi-theme($material-theme-dark); @@ -572,7 +272,6 @@ $appFontPath: '~roboto-fontface/fonts'; @include extension-creation.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include navigation.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include nf-editor.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); - @include resizable.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include system-diagnostics-dialog.nifi-theme($material-theme-dark); @include status-history.nifi-theme($material-theme-dark, $nifi-canvas-theme-dark); @include property-hint-tip.nifi-theme($material-theme-dark); From 33a28dfa48e10025d02a79e7ddbd69d865cb717a Mon Sep 17 00:00:00 2001 From: Joseph Witt Date: Thu, 14 Mar 2024 15:19:11 -0700 Subject: [PATCH 41/73] NIFI-12906 Upgraded ZooKeeper from 3.9.1 to 3.9.2 This closes #8506 Signed-off-by: David Handermann --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 774b43500742..e3f1b4df8bdb 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ 6.2.2 2.2.20 2.2.224 - 3.9.1 + 3.9.2 3.1.8 2.5.1 From 8c264d22f82efea0efd81715371c1437ac57e565 Mon Sep 17 00:00:00 2001 From: Joseph Witt Date: Thu, 14 Mar 2024 15:28:46 -0700 Subject: [PATCH 42/73] NIFI-12907 Upgraded Groovy from 4.0.19 to 4.0.20 This closes #8508 Signed-off-by: David Handermann --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e3f1b4df8bdb..df42bb275172 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ 4.0.1 3.1.0 2.5.0 - 4.0.19 + 4.0.20 3.2.5 3.3.6 1.2.1 From 27fa595b25c848bc06579748c17be2b0bf6dcd2e Mon Sep 17 00:00:00 2001 From: EndzeitBegins <16666115+EndzeitBegins@users.noreply.github.com> Date: Thu, 14 Mar 2024 19:29:38 +0100 Subject: [PATCH 43/73] NIFI-12901 Removed time units in description of time period properties This closes #8504 Signed-off-by: David Handermann --- .../main/java/org/apache/nifi/dbcp/utils/DBCPProperties.java | 4 ++-- .../java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java | 4 ++-- .../nifi/processors/standard/ReplaceTextWithMapping.java | 2 +- .../java/org/apache/nifi/dbcp/HikariCPConnectionPool.java | 2 +- .../java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-dbcp-base/src/main/java/org/apache/nifi/dbcp/utils/DBCPProperties.java b/nifi-nar-bundles/nifi-extension-utils/nifi-dbcp-base/src/main/java/org/apache/nifi/dbcp/utils/DBCPProperties.java index 21d6cdde251a..0b2bd7d3798b 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-dbcp-base/src/main/java/org/apache/nifi/dbcp/utils/DBCPProperties.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-dbcp-base/src/main/java/org/apache/nifi/dbcp/utils/DBCPProperties.java @@ -136,7 +136,7 @@ private DBCPProperties() { public static final PropertyDescriptor MAX_CONN_LIFETIME = new PropertyDescriptor.Builder() .displayName("Max Connection Lifetime") .name("dbcp-max-conn-lifetime") - .description("The maximum lifetime in milliseconds of a connection. After this time is exceeded the " + + .description("The maximum lifetime of a connection. After this time is exceeded the " + "connection will fail the next activation, passivation or validation test. A value of zero or less " + "means the connection has an infinite lifetime.") .defaultValue(DefaultDataSourceValues.MAX_CONN_LIFETIME.getValue()) @@ -148,7 +148,7 @@ private DBCPProperties() { public static final PropertyDescriptor EVICTION_RUN_PERIOD = new PropertyDescriptor.Builder() .displayName("Time Between Eviction Runs") .name("dbcp-time-between-eviction-runs") - .description("The number of milliseconds to sleep between runs of the idle connection evictor thread. When " + + .description("The time period to sleep between runs of the idle connection evictor thread. When " + "non-positive, no idle connection evictor thread will be run.") .defaultValue(DefaultDataSourceValues.EVICTION_RUN_PERIOD.getValue()) .required(false) diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java index 94cc1755f89b..3f2bbff59597 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/nifi/dbcp/hive/Hive3ConnectionPool.java @@ -198,7 +198,7 @@ public class Hive3ConnectionPool extends AbstractControllerService implements Hi public static final PropertyDescriptor MAX_CONN_LIFETIME = new PropertyDescriptor.Builder() .displayName("Max Connection Lifetime") .name("dbcp-max-conn-lifetime") - .description("The maximum lifetime in milliseconds of a connection. After this time is exceeded the " + + .description("The maximum lifetime of a connection. After this time is exceeded the " + "connection will fail the next activation, passivation or validation test. A value of zero or less " + "means the connection has an infinite lifetime.") .defaultValue(DEFAULT_MAX_CONN_LIFETIME) @@ -210,7 +210,7 @@ public class Hive3ConnectionPool extends AbstractControllerService implements Hi public static final PropertyDescriptor EVICTION_RUN_PERIOD = new PropertyDescriptor.Builder() .displayName("Time Between Eviction Runs") .name("dbcp-time-between-eviction-runs") - .description("The number of milliseconds to sleep between runs of the idle connection evictor thread. When " + + .description("The time period to sleep between runs of the idle connection evictor thread. When " + "non-positive, no idle connection evictor thread will be run.") .defaultValue(DEFAULT_EVICTION_RUN_PERIOD) .required(false) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java index fe8c55a509ff..d62401061c80 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceTextWithMapping.java @@ -99,7 +99,7 @@ public class ReplaceTextWithMapping extends AbstractProcessor { .build(); public static final PropertyDescriptor MAPPING_FILE_REFRESH_INTERVAL = new PropertyDescriptor.Builder() .name("Mapping File Refresh Interval") - .description("The polling interval in seconds to check for updates to the mapping file. The default is 60s.") + .description("The polling interval to check for updates to the mapping file. The default is 60s.") .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) .required(true) .defaultValue("60s") diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-hikari-dbcp-service/src/main/java/org/apache/nifi/dbcp/HikariCPConnectionPool.java b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-hikari-dbcp-service/src/main/java/org/apache/nifi/dbcp/HikariCPConnectionPool.java index 128701a610ac..230df5a6f3b0 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-hikari-dbcp-service/src/main/java/org/apache/nifi/dbcp/HikariCPConnectionPool.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-hikari-dbcp-service/src/main/java/org/apache/nifi/dbcp/HikariCPConnectionPool.java @@ -179,7 +179,7 @@ public class HikariCPConnectionPool extends AbstractControllerService implements public static final PropertyDescriptor MAX_CONN_LIFETIME = new PropertyDescriptor.Builder() .name("hikaricp-max-conn-lifetime") .displayName("Max Connection Lifetime") - .description("The maximum lifetime in milliseconds of a connection. After this time is exceeded the " + + .description("The maximum lifetime of a connection. After this time is exceeded the " + "connection will fail the next activation, passivation or validation test. A value of zero or less " + "means the connection has an infinite lifetime.") .defaultValue(DEFAULT_MAX_CONN_LIFETIME) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java index a031f15efb1b..5d75daf2caed 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/src/main/java/org/apache/nifi/dbcp/HadoopDBCPConnectionPool.java @@ -215,7 +215,7 @@ public class HadoopDBCPConnectionPool extends AbstractControllerService implemen public static final PropertyDescriptor MAX_CONN_LIFETIME = new PropertyDescriptor.Builder() .displayName("Max Connection Lifetime") .name("dbcp-max-conn-lifetime") - .description("The maximum lifetime in milliseconds of a connection. After this time is exceeded the " + + .description("The maximum lifetime of a connection. After this time is exceeded the " + "connection will fail the next activation, passivation or validation test. A value of zero or less " + "means the connection has an infinite lifetime.") .defaultValue(DEFAULT_MAX_CONN_LIFETIME) @@ -227,7 +227,7 @@ public class HadoopDBCPConnectionPool extends AbstractControllerService implemen public static final PropertyDescriptor EVICTION_RUN_PERIOD = new PropertyDescriptor.Builder() .displayName("Time Between Eviction Runs") .name("dbcp-time-between-eviction-runs") - .description("The number of milliseconds to sleep between runs of the idle connection evictor thread. When " + + .description("The time period to sleep between runs of the idle connection evictor thread. When " + "non-positive, no idle connection evictor thread will be run.") .defaultValue(DEFAULT_EVICTION_RUN_PERIOD) .required(false) From 42bd5243bb3e32b21559f628126d1795e11c5c30 Mon Sep 17 00:00:00 2001 From: tpalfy Date: Tue, 12 Mar 2024 13:33:01 +0100 Subject: [PATCH 44/73] NIFI-12887 Added Binary String Format property to PutDatabaseRecord - Supports handling Strings as hexadecimal character sequences or base64-encoded binary data when inserting into a binary type column This closes #8493 Signed-off-by: David Handermann --- .../standard/PutDatabaseRecord.java | 43 ++++++++- .../standard/PutDatabaseRecordTest.java | 89 ++++++++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java index 1f60208f4034..26b4281cd099 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutDatabaseRecord.java @@ -77,10 +77,12 @@ import java.sql.Statement; import java.sql.Types; import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.Optional; @@ -238,6 +240,34 @@ public class PutDatabaseRecord extends AbstractProcessor { .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); + static final AllowableValue BINARY_STRING_FORMAT_UTF8 = new AllowableValue( + "UTF-8", + "UTF-8", + "String values for binary columns contain the original value as text via UTF-8 character encoding" + ); + + static final AllowableValue BINARY_STRING_FORMAT_HEXADECIMAL = new AllowableValue( + "Hexadecimal", + "Hexadecimal", + "String values for binary columns contain the original value in hexadecimal format" + ); + + static final AllowableValue BINARY_STRING_FORMAT_BASE64 = new AllowableValue( + "Base64", + "Base64", + "String values for binary columns contain the original value in Base64 encoded format" + ); + + static final PropertyDescriptor BINARY_STRING_FORMAT = new Builder() + .name("put-db-record-binary-format") + .displayName("Binary String Format") + .description("The format to be applied when decoding string values to binary.") + .required(true) + .expressionLanguageSupported(FLOWFILE_ATTRIBUTES) + .allowableValues(BINARY_STRING_FORMAT_UTF8, BINARY_STRING_FORMAT_HEXADECIMAL, BINARY_STRING_FORMAT_BASE64) + .defaultValue(BINARY_STRING_FORMAT_UTF8.getValue()) + .build(); + static final PropertyDescriptor TRANSLATE_FIELD_NAMES = new Builder() .name("put-db-record-translate-field-names") .displayName("Translate Field Names") @@ -388,6 +418,7 @@ public class PutDatabaseRecord extends AbstractProcessor { pds.add(CATALOG_NAME); pds.add(SCHEMA_NAME); pds.add(TABLE_NAME); + pds.add(BINARY_STRING_FORMAT); pds.add(TRANSLATE_FIELD_NAMES); pds.add(UNMATCHED_FIELD_BEHAVIOR); pds.add(UNMATCHED_COLUMN_BEHAVIOR); @@ -606,6 +637,8 @@ private void executeDML(final ProcessContext context, final ProcessSession sessi final int maxBatchSize = context.getProperty(MAX_BATCH_SIZE).evaluateAttributeExpressions(flowFile).asInteger(); final int timeoutMillis = context.getProperty(QUERY_TIMEOUT).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS).intValue(); + final String binaryStringFormat = context.getProperty(BINARY_STRING_FORMAT).evaluateAttributeExpressions(flowFile).getValue(); + // Ensure the table name has been set, the generated SQL statements (and TableSchema cache) will need it if (StringUtils.isEmpty(tableName)) { throw new IllegalArgumentException(format("Cannot process %s because Table Name is null or empty", flowFile)); @@ -764,7 +797,15 @@ private void executeDML(final ProcessContext context, final ProcessSession sessi } currentValue = dest; } else if (currentValue instanceof String) { - currentValue = ((String) currentValue).getBytes(StandardCharsets.UTF_8); + final String stringValue = (String) currentValue; + + if (BINARY_STRING_FORMAT_BASE64.getValue().equals(binaryStringFormat)) { + currentValue = Base64.getDecoder().decode(stringValue); + } else if (BINARY_STRING_FORMAT_HEXADECIMAL.getValue().equals(binaryStringFormat)) { + currentValue = HexFormat.of().parseHex(stringValue); + } else { + currentValue = stringValue.getBytes(StandardCharsets.UTF_8); + } } else if (currentValue != null && !(currentValue instanceof byte[])) { throw new IllegalTypeConversionException("Cannot convert value " + currentValue + " to BLOB/BINARY/VARBINARY/LONGVARBINARY"); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/PutDatabaseRecordTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/PutDatabaseRecordTest.java index e6483efc7fe6..70a908b377d3 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/PutDatabaseRecordTest.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/PutDatabaseRecordTest.java @@ -61,6 +61,7 @@ import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -1686,6 +1687,88 @@ void testInsertWithBlobClob() throws Exception { conn.close(); } + @Test + void testInsertHexStringIntoBinary() throws Exception { + runner.setProperty(PutDatabaseRecord.BINARY_STRING_FORMAT, PutDatabaseRecord.BINARY_STRING_FORMAT_HEXADECIMAL); + + String tableName = "HEX_STRING_TEST"; + String createTable = "CREATE TABLE " + tableName + " (id integer primary key, binary_data blob)"; + String hexStringData = "abCDef"; + + recreateTable(tableName, createTable); + final MockRecordParser parser = new MockRecordParser(); + runner.addControllerService("parser", parser); + runner.enableControllerService(parser); + + parser.addSchemaField("id", RecordFieldType.INT); + parser.addSchemaField("binaryData", RecordFieldType.STRING); + + parser.addRecord(1, hexStringData); + + runner.setProperty(PutDatabaseRecord.RECORD_READER_FACTORY, "parser"); + runner.setProperty(PutDatabaseRecord.STATEMENT_TYPE, PutDatabaseRecord.INSERT_TYPE); + runner.setProperty(PutDatabaseRecord.TABLE_NAME, tableName); + + runner.enqueue(new byte[0]); + runner.run(); + + runner.assertTransferCount(PutDatabaseRecord.REL_SUCCESS, 1); + final Connection conn = dbcp.getConnection(); + final Statement stmt = conn.createStatement(); + + final ResultSet resultSet = stmt.executeQuery("SELECT * FROM " + tableName); + assertTrue(resultSet.next()); + + assertEquals(1, resultSet.getInt(1)); + + Blob blob = resultSet.getBlob(2); + assertArrayEquals(new byte[]{(byte)171, (byte)205, (byte)239}, blob.getBytes(1, (int)blob.length())); + + stmt.close(); + conn.close(); + } + + @Test + void testInsertBase64StringIntoBinary() throws Exception { + runner.setProperty(PutDatabaseRecord.BINARY_STRING_FORMAT, PutDatabaseRecord.BINARY_STRING_FORMAT_BASE64); + + String tableName = "BASE64_STRING_TEST"; + String createTable = "CREATE TABLE " + tableName + " (id integer primary key, binary_data blob)"; + byte[] binaryData = {(byte) 10, (byte) 103, (byte) 234}; + + recreateTable(tableName, createTable); + final MockRecordParser parser = new MockRecordParser(); + runner.addControllerService("parser", parser); + runner.enableControllerService(parser); + + parser.addSchemaField("id", RecordFieldType.INT); + parser.addSchemaField("binaryData", RecordFieldType.STRING); + + parser.addRecord(1, Base64.getEncoder().encodeToString(binaryData)); + + runner.setProperty(PutDatabaseRecord.RECORD_READER_FACTORY, "parser"); + runner.setProperty(PutDatabaseRecord.STATEMENT_TYPE, PutDatabaseRecord.INSERT_TYPE); + runner.setProperty(PutDatabaseRecord.TABLE_NAME, tableName); + + runner.enqueue(new byte[0]); + runner.run(); + + runner.assertTransferCount(PutDatabaseRecord.REL_SUCCESS, 1); + final Connection conn = dbcp.getConnection(); + final Statement stmt = conn.createStatement(); + + final ResultSet resultSet = stmt.executeQuery("SELECT * FROM " + tableName); + assertTrue(resultSet.next()); + + assertEquals(1, resultSet.getInt(1)); + + Blob blob = resultSet.getBlob(2); + assertArrayEquals(binaryData, blob.getBytes(1, (int)blob.length())); + + stmt.close(); + conn.close(); + } + @Test void testInsertWithBlobClobObjectArraySource() throws Exception { String createTableWithBlob = "CREATE TABLE PERSONS (id integer primary key, name clob," + @@ -1959,10 +2042,14 @@ private int getTableSize() throws SQLException { } private void recreateTable(String createSQL) throws ProcessException, SQLException { + recreateTable("PERSONS", createSQL); + } + + private void recreateTable(String tableName, String createSQL) throws ProcessException, SQLException { final Connection conn = dbcp.getConnection(); final Statement stmt = conn.createStatement(); try { - stmt.execute("drop table PERSONS"); + stmt.execute("drop table " + tableName); } catch (SQLException ignore) { // Do nothing, may not have existed } From 3719fddf84ffcf18ed29d87ee931aa1acc1699d3 Mon Sep 17 00:00:00 2001 From: emiliosetiadarma Date: Mon, 22 Jan 2024 01:28:09 -0800 Subject: [PATCH 45/73] NIFI-12700: refactored PutKudu to optimize memory handling for AUTO_FLUSH_SYNC flush mode (unbatched flush) NIFI-12700: made changes based on PR comments. Simplified statements involving determination of whether or not there are flowfile failures/rowErrors. Separated out getting rowErrors from OperationResponses into its own function Signed-off-by: Matt Burgess This closes #8322 --- .../kudu/AbstractKuduProcessor.java | 40 ++++- .../kudu/AutoFlushSyncPutKuduResult.java | 78 ++++++++++ .../apache/nifi/processors/kudu/PutKudu.java | 110 +++++++------ .../nifi/processors/kudu/PutKuduResult.java | 144 ++++++++++++++++++ .../kudu/StandardPutKuduResult.java | 83 ++++++++++ 5 files changed, 388 insertions(+), 67 deletions(-) create mode 100644 nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AutoFlushSyncPutKuduResult.java create mode 100644 nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKuduResult.java create mode 100644 nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/StandardPutKuduResult.java diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java index 47c1a3490387..b44f2330ee15 100644 --- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java +++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AbstractKuduProcessor.java @@ -22,6 +22,7 @@ import java.sql.Timestamp; import java.time.LocalDate; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; @@ -35,6 +36,8 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; +import java.util.stream.Collectors; + import org.apache.kudu.ColumnSchema; import org.apache.kudu.ColumnTypeAttributes; import org.apache.kudu.Schema; @@ -217,19 +220,33 @@ protected void executeOnKuduClient(Consumer actionOnKuduClient) { } } - protected void flushKuduSession(final KuduSession kuduSession, boolean close, final List rowErrors) throws KuduException { - final List responses = close ? kuduSession.close() : kuduSession.flush(); - + /** + * Get the pending errors from the active {@link KuduSession}. This will only be applicable if the flushMode is + * {@code SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND}. + * @return a {@link List} of pending {@link RowError}s + */ + protected List getPendingRowErrorsFromKuduSession(final KuduSession kuduSession) { if (kuduSession.getFlushMode() == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { - rowErrors.addAll(Arrays.asList(kuduSession.getPendingErrors().getRowErrors())); + return Arrays.asList(kuduSession.getPendingErrors().getRowErrors()); } else { - responses.stream() - .filter(OperationResponse::hasRowError) - .map(OperationResponse::getRowError) - .forEach(rowErrors::add); + return Collections.EMPTY_LIST; } } + protected List flushKuduSession(final KuduSession kuduSession) throws KuduException { + final List responses = kuduSession.flush(); + // RowErrors will only be present in the OperationResponses in this case if the flush mode + // selected is MANUAL_FLUSH. It will be empty otherwise. + return getRowErrors(responses); + } + + protected List closeKuduSession(final KuduSession kuduSession) throws KuduException { + final List responses = kuduSession.close(); + // RowErrors will only be present in the OperationResponses in this case if the flush mode + // selected is MANUAL_FLUSH, since the underlying implementation of kuduSession.close() returns + // the OperationResponses from a flush() call. + return getRowErrors(responses); + } @OnStopped public void shutdown() throws Exception { @@ -410,4 +427,11 @@ private String getName() { return String.format("PutKudu[%s]-client-%d", identifier, threadCount.getAndIncrement()); } } + + private List getRowErrors(final List responses) { + return responses.stream() + .filter(OperationResponse::hasRowError) + .map(OperationResponse::getRowError) + .collect(Collectors.toList()); + } } diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AutoFlushSyncPutKuduResult.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AutoFlushSyncPutKuduResult.java new file mode 100644 index 000000000000..d502741688be --- /dev/null +++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/AutoFlushSyncPutKuduResult.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.kudu; + +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.RowError; +import org.apache.nifi.flowfile.FlowFile; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class AutoFlushSyncPutKuduResult extends PutKuduResult { + private final Map> flowFileRowErrorsMap; + + public AutoFlushSyncPutKuduResult() { + super(); + this.flowFileRowErrorsMap = new HashMap<>(); + } + + @Override + public void recordOperation(final Operation operation) { + // this should be a no-op because we don't need to record Operation's origins + // for buffered flush when using AUTO_FLUSH_SYNC + return; + } + + @Override + public void addError(final RowError rowError) { + final List rowErrors = flowFileRowErrorsMap.getOrDefault(flowFile, new ArrayList<>()); + rowErrors.add(rowError); + flowFileRowErrorsMap.put(flowFile, rowErrors); + } + + @Override + public void addErrors(final List rowErrors) { + // This is a no-op because we would never be in a situation where we'd have to add a collection of RowError + // using this Flush Mode. Since we do not keep Operation to FlowFile mapping, it will also be impossible to resolve + // RowErrors to the FlowFile that caused them, hence this method should never be implemented for AUTO_FLUSH_SYNC + return; + } + + @Override + public boolean hasRowErrorsOrFailures() { + if (!flowFileFailures.isEmpty()) { + return true; + } + + for (final Map.Entry> entry : flowFileRowErrorsMap.entrySet()) { + if (!entry.getValue().isEmpty()) { + return true; + } + } + + return false; + } + + @Override + public List getRowErrorsForFlowFile(final FlowFile flowFile) { + return flowFileRowErrorsMap.getOrDefault(flowFile, Collections.EMPTY_LIST); + } +} diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java index eaa07617c21d..a1317b3fd8e1 100644 --- a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java +++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKudu.java @@ -22,10 +22,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -357,53 +355,52 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } private void processFlowFiles(final ProcessContext context, final ProcessSession session, final List flowFiles, final KuduClient kuduClient) { - final Map processedRecords = new HashMap<>(); - final Map flowFileFailures = new HashMap<>(); - final Map operationFlowFileMap = new HashMap<>(); - final List pendingRowErrors = new ArrayList<>(); - final KuduSession kuduSession = createKuduSession(kuduClient); + final PutKuduResult putKuduResult = flushMode == SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC + ? new AutoFlushSyncPutKuduResult() : new StandardPutKuduResult(); try { processRecords(flowFiles, - processedRecords, - flowFileFailures, - operationFlowFileMap, - pendingRowErrors, session, context, kuduClient, - kuduSession); + kuduSession, + putKuduResult); } finally { try { - flushKuduSession(kuduSession, true, pendingRowErrors); + final List rowErrors = closeKuduSession(kuduSession); + if (flushMode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { + putKuduResult.addErrors(getPendingRowErrorsFromKuduSession(kuduSession)); + } else { + putKuduResult.addErrors(rowErrors); + } } catch (final KuduException|RuntimeException e) { getLogger().error("KuduSession.close() Failed", e); } } - if (isRollbackOnFailure() && (!pendingRowErrors.isEmpty() || !flowFileFailures.isEmpty())) { - logFailures(pendingRowErrors, operationFlowFileMap); + putKuduResult.resolveFlowFileToRowErrorAssociations(); + + if (isRollbackOnFailure() && putKuduResult.hasRowErrorsOrFailures()) { + logFailures(putKuduResult); session.rollback(); context.yield(); } else { - transferFlowFiles(flowFiles, processedRecords, flowFileFailures, operationFlowFileMap, pendingRowErrors, session); + transferFlowFiles(flowFiles, session, putKuduResult); } } private void processRecords(final List flowFiles, - final Map processedRecords, - final Map flowFileFailures, - final Map operationFlowFileMap, - final List pendingRowErrors, - final ProcessSession session, - final ProcessContext context, - final KuduClient kuduClient, - final KuduSession kuduSession) { + final ProcessSession session, + final ProcessContext context, + final KuduClient kuduClient, + final KuduSession kuduSession, + final PutKuduResult putKuduResult) { final RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); int bufferedRecords = 0; OperationType prevOperationType = OperationType.INSERT; for (FlowFile flowFile : flowFiles) { + putKuduResult.setFlowFile(flowFile); try (final InputStream in = session.read(flowFile); final RecordReader recordReader = recordReaderFactory.createRecordReader(flowFile, in, getLogger())) { @@ -472,7 +469,12 @@ private void processRecords(final List flowFiles, // ignore operations. if (!supportsInsertIgnoreOp && prevOperationType != operationType && (prevOperationType == OperationType.INSERT_IGNORE || operationType == OperationType.INSERT_IGNORE)) { - flushKuduSession(kuduSession, false, pendingRowErrors); + final List rowErrors = flushKuduSession(kuduSession); + if (flushMode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { + putKuduResult.addErrors(getPendingRowErrorsFromKuduSession(kuduSession)); + } else { + putKuduResult.addErrors(rowErrors); + } kuduSession.setIgnoreAllDuplicateRows(operationType == OperationType.INSERT_IGNORE); } prevOperationType = operationType; @@ -481,34 +483,35 @@ private void processRecords(final List flowFiles, Operation operation = createKuduOperation(operationType, dataRecord, fieldNames, ignoreNull, lowercaseFields, kuduTable); // We keep track of mappings between Operations and their origins, // so that we know which FlowFiles should be marked failure after buffered flush. - operationFlowFileMap.put(operation, flowFile); + putKuduResult.recordOperation(operation); // Flush mutation buffer of KuduSession to avoid "MANUAL_FLUSH is enabled // but the buffer is too big" error. This can happen when flush mode is // MANUAL_FLUSH and a FlowFile has more than one records. if (bufferedRecords == batchSize && flushMode == SessionConfiguration.FlushMode.MANUAL_FLUSH) { bufferedRecords = 0; - flushKuduSession(kuduSession, false, pendingRowErrors); + final List rowErrors = flushKuduSession(kuduSession); + putKuduResult.addErrors(rowErrors); } // OperationResponse is returned only when flush mode is set to AUTO_FLUSH_SYNC - OperationResponse response = kuduSession.apply(operation); + final OperationResponse response = kuduSession.apply(operation); if (response != null && response.hasRowError()) { // Stop processing the records on the first error. // Note that Kudu does not support rolling back of previous operations. - flowFileFailures.put(flowFile, response.getRowError()); + putKuduResult.addFailure(response.getRowError()); break recordReaderLoop; } bufferedRecords++; - processedRecords.merge(flowFile, 1, Integer::sum); + putKuduResult.incrementProcessedRecordsForFlowFile(); } record = recordSet.next(); } } catch (Exception ex) { getLogger().error("Failed to push {} to Kudu", flowFile, ex); - flowFileFailures.put(flowFile, ex); + putKuduResult.addFailure(ex); } } } @@ -575,38 +578,28 @@ private boolean handleSchemaDrift(final KuduTable kuduTable, final KuduClient ku } private void transferFlowFiles(final List flowFiles, - final Map processedRecords, - final Map flowFileFailures, - final Map operationFlowFileMap, - final List pendingRowErrors, - final ProcessSession session) { - // Find RowErrors for each FlowFile - final Map> flowFileRowErrors = pendingRowErrors.stream() - .filter(e -> operationFlowFileMap.get(e.getOperation()) != null) - .collect( - Collectors.groupingBy(e -> operationFlowFileMap.get(e.getOperation())) - ); - + final ProcessSession session, + final PutKuduResult putKuduResult) { long totalCount = 0L; for (FlowFile flowFile : flowFiles) { - final int count = processedRecords.getOrDefault(flowFile, 0); + final int count = putKuduResult.getProcessedRecordsForFlowFile(flowFile); totalCount += count; - final List rowErrors = flowFileRowErrors.get(flowFile); + final List rowErrors = putKuduResult.getRowErrorsForFlowFile(flowFile); - if (rowErrors != null) { + if (rowErrors != null && !rowErrors.isEmpty()) { rowErrors.forEach(rowError -> getLogger().error("Failed to write due to {}", rowError.toString())); flowFile = session.putAttribute(flowFile, RECORD_COUNT_ATTR, Integer.toString(count - rowErrors.size())); - totalCount -= rowErrors.size(); // Don't include error rows in the the counter. + totalCount -= rowErrors.size(); // Don't include error rows in the counter. session.transfer(flowFile, REL_FAILURE); } else { flowFile = session.putAttribute(flowFile, RECORD_COUNT_ATTR, String.valueOf(count)); - if (flowFileFailures.containsKey(flowFile)) { - getLogger().error("Failed to write due to {}", flowFileFailures.get(flowFile)); - session.transfer(flowFile, REL_FAILURE); - } else { + if (putKuduResult.isFlowFileProcessedSuccessfully(flowFile)) { session.transfer(flowFile, REL_SUCCESS); session.getProvenanceReporter().send(flowFile, "Successfully added FlowFile to Kudu"); + } else { + getLogger().error("Failed to write due to {}", putKuduResult.getFailureForFlowFile(flowFile)); + session.transfer(flowFile, REL_FAILURE); } } } @@ -614,15 +607,14 @@ private void transferFlowFiles(final List flowFiles, session.adjustCounter("Records Inserted", totalCount, false); } - private void logFailures(final List pendingRowErrors, final Map operationFlowFileMap) { - final Map> flowFileRowErrors = pendingRowErrors.stream().collect( - Collectors.groupingBy(e -> operationFlowFileMap.get(e.getOperation()))); - - for (final Map.Entry> entry : flowFileRowErrors.entrySet()) { - final FlowFile flowFile = entry.getKey(); - final List errors = entry.getValue(); + private void logFailures(final PutKuduResult putKuduResult) { + final Set processedFlowFiles = putKuduResult.getProcessedFlowFiles(); + for (final FlowFile flowFile : processedFlowFiles) { + final List errors = putKuduResult.getRowErrorsForFlowFile(flowFile); + if (!errors.isEmpty()) { + getLogger().error("Could not write {} to Kudu due to: {}", flowFile, errors); + } - getLogger().error("Could not write {} to Kudu due to: {}", flowFile, errors); } } diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKuduResult.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKuduResult.java new file mode 100644 index 000000000000..f46a65d90dd1 --- /dev/null +++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/PutKuduResult.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.kudu; + +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.RowError; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.serialization.record.Record; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class PutKuduResult { + protected FlowFile flowFile; + protected final Map flowFileFailures; + private final Set processedFlowFiles; + private final Map processedRecords; + + public PutKuduResult() { + this.flowFile = null; + + this.flowFileFailures = new HashMap<>(); + this.processedFlowFiles = new HashSet<>(); + this.processedRecords = new HashMap<>(); + } + + public void setFlowFile(final FlowFile flowFile) { + this.flowFile = flowFile; + processedFlowFiles.add(flowFile); + } + + public Set getProcessedFlowFiles() { + return this.processedFlowFiles; + } + + public int getProcessedRecordsForFlowFile(final FlowFile flowFile) { + return this.processedRecords.getOrDefault(flowFile, 0); + } + + /** + * Increments the number of {@link Record}s that has been successfully processed for this {@link FlowFile} + */ + public void incrementProcessedRecordsForFlowFile() { + final int newCount = this.processedRecords.getOrDefault(flowFile, 0) + 1; + this.processedRecords.put(flowFile, newCount); + } + + /** + * Records an {@link Operation} being processed for a specific {@link FlowFile} + * @param operation the {@link Operation} to record + */ + public abstract void recordOperation(final Operation operation); + + /** + * Records a {@link RowError} for the particular {@link FlowFile} that's being processed + * @param rowError the {@link RowError} to add + */ + public abstract void addError(final RowError rowError); + + /** + * Records a {@link List} of {@link RowError}s for the particular {@link FlowFile} that's being processed + * @param rowErrors the {@link List} of {@link RowError}s to add + */ + public void addErrors(final List rowErrors) { + for (final RowError rowError : rowErrors) { + addError(rowError); + } + } + + /** + * Records a failure (an {@link Exception} or a {@link RowError}) for the particular {@link FlowFile} that's being processed. + * A failure is defined as anything that stops the processing of the records in a {@link FlowFile} + * @param failure the {@link Exception} or {@link RowError} to add + */ + public void addFailure(final Object failure) { + if (flowFileFailures.containsKey(flowFile)) { + throw new IllegalStateException("A failure has already previously occurred while processing FlowFile."); + } + flowFileFailures.put(flowFile, failure); + } + + + /** + * Resolves the associations between {@link FlowFile} and the {@link RowError}s that occurred + * while processing them. This is only applicable in batch sesssion flushes, namely when + * using the {@code SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND} and + * {@code SessionConfiguration.FlushMode.MANUAL_FLUSH} flush modes. Otherwise, this + * function should be a no-op. This function should only be called once finished with processing + * all {@link FlowFile}s in a batch. + */ + public void resolveFlowFileToRowErrorAssociations() { + return; + } + + /** + * Checks whether there was a failure (i.e. either an {@link Exception} or {@link RowError} that happened during processing) + * @return {@code true} if there was a {@link Exception} or a {@link RowError} that happened during processing, {@code false} otherwise + */ + public abstract boolean hasRowErrorsOrFailures(); + + /** + * Checks whether the {@link FlowFile} was processed successfully (i.e. no {@link Exception}s or + * {@link RowError}s occurred while processing the {@link FlowFile}). + * + * @param flowFile {@link FlowFile} to check + * @return {@code true} if the processing the {@link FlowFile} did not incur any exceptions, {@code false} otherwise + */ + public boolean isFlowFileProcessedSuccessfully(final FlowFile flowFile) { + return !flowFileFailures.containsKey(flowFile); + } + + /** + * Returns the failure ({@link Exception} or {@link RowError}) that occurred while processing the {@link FlowFile} + * @param flowFile the {@link FlowFile} to check + * @return the {@link Exception} or {@link RowError} if one occurred while processing the given {@link FlowFile} or {@code null} + */ + public Object getFailureForFlowFile(final FlowFile flowFile) { + return flowFileFailures.get(flowFile); + } + + /** + * Retrieves the {@link RowError}s that have occurred when processing a {@link FlowFile} + * @param flowFile the {@link FlowFile} to retrieve the {@link RowError}s of + * @return a {@link List} of {@link RowError}s for the {@link FlowFile} or an {@code Collections.EMPTY_LIST} if no errors + */ + public abstract List getRowErrorsForFlowFile(final FlowFile flowFile); +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/StandardPutKuduResult.java b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/StandardPutKuduResult.java new file mode 100644 index 000000000000..7b4a61119b2a --- /dev/null +++ b/nifi-nar-bundles/nifi-kudu-bundle/nifi-kudu-processors/src/main/java/org/apache/nifi/processors/kudu/StandardPutKuduResult.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.kudu; + +import org.apache.kudu.client.Operation; +import org.apache.kudu.client.RowError; +import org.apache.nifi.flowfile.FlowFile; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class StandardPutKuduResult extends PutKuduResult { + private final Map operationFlowFileMap; + private final List pendingRowErrors; + private final Map> flowFileRowErrorsMap; + + public StandardPutKuduResult() { + super(); + this.operationFlowFileMap = new HashMap<>(); + this.pendingRowErrors = new ArrayList<>(); + this.flowFileRowErrorsMap = new HashMap<>(); + } + + @Override + public void recordOperation(final Operation operation) { + operationFlowFileMap.put(operation, flowFile); + } + + @Override + public void addError(final RowError rowError) { + // When this class is used to store results from processing FlowFiles, the FlushMode + // is set to AUTO_FLUSH_BACKGROUND or MANUAL_FLUSH. In either case, we won't know which + // FlowFile/Record we are currently processing as the RowErrors are obtained from the KuduSession + // post-processing of the FlowFile/Record + this.pendingRowErrors.add(rowError); + } + + @Override + public void resolveFlowFileToRowErrorAssociations() { + flowFileRowErrorsMap.putAll(pendingRowErrors.stream() + .filter(e -> operationFlowFileMap.get(e.getOperation()) != null) + .collect( + Collectors.groupingBy(e -> operationFlowFileMap.get(e.getOperation())) + ) + ); + + pendingRowErrors.clear(); + } + + @Override + public boolean hasRowErrorsOrFailures() { + if (!flowFileFailures.isEmpty()) { + return true; + } + + return flowFileRowErrorsMap.entrySet() + .stream() + .anyMatch(entry -> !entry.getValue().isEmpty()); + } + + @Override + public List getRowErrorsForFlowFile(final FlowFile flowFile) { + return flowFileRowErrorsMap.getOrDefault(flowFile, Collections.EMPTY_LIST); + } +} From d97c7946967ea57602b48b96d7e6312834bacf2e Mon Sep 17 00:00:00 2001 From: Matt Gilman Date: Fri, 15 Mar 2024 16:15:48 -0400 Subject: [PATCH 46/73] NIFI-12902: Migrating to @angular-devkit/build-angular:application (#8507) * NIFI-12902: - Migrating to @angular-devkit/build-angular:application to avoid and avoid sporadic webpack build issues. - Updating roboto dependencies. * NIFI-12902: - Clean up. This closes #8507 --- .../src/main/nifi/angular.json | 7 +- .../src/main/nifi/package-lock.json | 18 ++-- .../src/main/nifi/package.json | 3 +- .../src/main/nifi/proxy.config.mjs | 18 +++- .../provenance-event-listing.effects.ts | 3 +- .../queue-listing/queue-listing.effects.ts | 2 +- .../parameter-tip.component.scss | 2 +- .../src/main/nifi/src/styles.scss | 92 +------------------ 8 files changed, 40 insertions(+), 105 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json index 84c1979e8237..b45bff79b4ed 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/angular.json @@ -15,11 +15,11 @@ "prefix": "app", "architect": { "build": { - "builder": "@angular-devkit/build-angular:browser", + "builder": "@angular-devkit/build-angular:application", "options": { "outputPath": "dist/nifi", "index": "src/index.html", - "main": "src/main.ts", + "browser": "src/main.ts", "polyfills": ["zone.js"], "tsConfig": "tsconfig.app.json", "inlineStyleLanguage": "scss", @@ -38,13 +38,10 @@ } ], "outputHashing": "all", - "buildOptimizer": false, "optimization": true }, "development": { - "buildOptimizer": false, "optimization": false, - "vendorChunk": true, "extractLicenses": false, "sourceMap": true, "namedChunks": true, diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json index 178f33516b13..969f25b8e9cc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json @@ -19,6 +19,8 @@ "@angular/platform-browser-dynamic": "^17.1.3", "@angular/router": "^17.1.3", "@ctrl/ngx-codemirror": "^7.0.0", + "@fontsource/roboto": "^5.0.12", + "@fontsource/roboto-slab": "^5.0.19", "@ngrx/effects": "^17.1.0", "@ngrx/router-store": "^17.1.0", "@ngrx/store": "^17.1.0", @@ -29,7 +31,6 @@ "humanize-duration": "^3.31.0", "ngrx-immer": "^2.1.1", "ngx-skeleton-loader": "^8.1.0", - "roboto-fontface": "0.10.0", "rxjs": "~7.8.1", "tslib": "^2.6.2", "webfontloader": "^1.6.28", @@ -3288,6 +3289,16 @@ "node": ">=14" } }, + "node_modules/@fontsource/roboto": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/@fontsource/roboto/-/roboto-5.0.12.tgz", + "integrity": "sha512-x0o17jvgoSSbS9OZnUX2+xJmVRvVCfeaYJjkS7w62iN7CuJWtMf5vJj8LqgC7ibqIkitOHVW+XssRjgrcHn62g==" + }, + "node_modules/@fontsource/roboto-slab": { + "version": "5.0.19", + "resolved": "https://registry.npmjs.org/@fontsource/roboto-slab/-/roboto-slab-5.0.19.tgz", + "integrity": "sha512-rqZ+XbhNhLj8L0OnOFioQmd9ElpolZlOGqJSRCZHdjJPk4C6kmbQKO4rGvLrs18eIFCyoqc3dV8i4l2BGZswUQ==" + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.13", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", @@ -14762,11 +14773,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/roboto-fontface": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/roboto-fontface/-/roboto-fontface-0.10.0.tgz", - "integrity": "sha512-OlwfYEgA2RdboZohpldlvJ1xngOins5d7ejqnIBWr9KaMxsnBqotpptRXTyfNRLnFpqzX6sTDt+X+a+6udnU8g==" - }, "node_modules/robust-predicates": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package.json index 30cfe4971a09..fdfe6a72cf4e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package.json +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package.json @@ -27,6 +27,8 @@ "@angular/platform-browser-dynamic": "^17.1.3", "@angular/router": "^17.1.3", "@ctrl/ngx-codemirror": "^7.0.0", + "@fontsource/roboto": "^5.0.12", + "@fontsource/roboto-slab": "^5.0.19", "@ngrx/effects": "^17.1.0", "@ngrx/router-store": "^17.1.0", "@ngrx/store": "^17.1.0", @@ -37,7 +39,6 @@ "humanize-duration": "^3.31.0", "ngrx-immer": "^2.1.1", "ngx-skeleton-loader": "^8.1.0", - "roboto-fontface": "0.10.0", "rxjs": "~7.8.1", "tslib": "^2.6.2", "webfontloader": "^1.6.28", diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/proxy.config.mjs b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/proxy.config.mjs index 36c612c22b14..a2377fe6fd4e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/proxy.config.mjs +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/proxy.config.mjs @@ -5,9 +5,25 @@ const target = { changeOrigin: true, headers: { 'X-ProxyPort': 4200 + }, + configure: (proxy, _options) => { + proxy.on('error', (err, _req, _res) => { + console.log('proxy error', err); + }); + proxy.on('proxyReq', (proxyReq, req, _res) => { + console.log('Sending Request to the Target:', req.method, req.url); + }); + proxy.on('proxyRes', (proxyRes, req, _res) => { + console.log('Received Response from the Target:', proxyRes.statusCode, req.url); + }); + }, + bypass: function(req, res, proxyOptions) { + if (req.url.startsWith('/nifi/')) { + return req.url; + } } }; export default { - '/': target + '/**': target }; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/state/provenance-event-listing/provenance-event-listing.effects.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/state/provenance-event-listing/provenance-event-listing.effects.ts index 6693677d048e..557846b02c85 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/state/provenance-event-listing/provenance-event-listing.effects.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/provenance/state/provenance-event-listing/provenance-event-listing.effects.ts @@ -328,8 +328,7 @@ export class ProvenanceEventListingEffects { panelClass: 'large-dialog' }); - dialogReference.componentInstance.contentViewerAvailable = - about?.contentViewerUrl != null ?? false; + dialogReference.componentInstance.contentViewerAvailable = about?.contentViewerUrl != null; dialogReference.componentInstance.downloadContent .pipe(takeUntil(dialogReference.afterClosed())) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/state/queue-listing/queue-listing.effects.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/state/queue-listing/queue-listing.effects.ts index cb5598b0d019..830b0ae31a1d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/state/queue-listing/queue-listing.effects.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/pages/queue/state/queue-listing/queue-listing.effects.ts @@ -268,7 +268,7 @@ export class QueueListingEffects { panelClass: 'large-dialog' }); - dialogReference.componentInstance.contentViewerAvailable = about?.contentViewerUrl != null ?? false; + dialogReference.componentInstance.contentViewerAvailable = about?.contentViewerUrl != null; dialogReference.componentInstance.downloadContent .pipe(takeUntil(dialogReference.afterClosed())) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/parameter-tip/parameter-tip.component.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/parameter-tip/parameter-tip.component.scss index 0e1912a1af28..61df9f13bb92 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/parameter-tip/parameter-tip.component.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/ui/common/tooltips/parameter-tip/parameter-tip.component.scss @@ -16,7 +16,7 @@ */ .parameter-name { - font-weight: bold; + font-weight: 700; font-family: monospace; font-size: 16px; margin-bottom: 10px; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss index 188168ac550d..f596ec68991f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/styles.scss @@ -67,7 +67,10 @@ @use 'app/ui/common/component-context/component-context.component-theme' as component-context; // Plus imports for other components in your app. -@use 'roboto-fontface/css/roboto/roboto-fontface.css'; +@use '@fontsource/roboto/latin.css' as roboto-normal; +@use '@fontsource/roboto/latin-300-italic.css' as roboto-light-italic; +@use '@fontsource/roboto/latin-400-italic.css' as roboto-normal-italic; +@use '@fontsource/roboto-slab/latin.css' as roboto-slab-normal; @use 'assets/fonts/flowfont/flowfont.css'; @use 'font-awesome/css/font-awesome.min.css'; @use 'codemirror/lib/codemirror.css'; @@ -78,93 +81,6 @@ // To override the NiFi theme, you need to set the variables $material-theme-light and $material-theme-dark @import 'assets/themes/nifi'; -$fontPrimary: 'Roboto', sans-serif; -$fontSecondary: 'Robot Slab', sans-serif; -$fontMedium: 'Roboto Medium', sans-serif; -$fontLight: 'Roboto Light', sans-serif; - -$appFontPath: '~roboto-fontface/fonts'; - -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: 300; - src: - local('Roboto Light'), - local('Roboto-Light'), - url('#{$appFontPath}/roboto/Roboto-Light.woff2') format('woff2'); -} - -@font-face { - font-family: 'Roboto'; - font-style: italic; - font-weight: 300; - src: - local('Roboto LightItalic'), - local('Roboto-LightItalic'), - url('#{$appFontPath}/roboto/Roboto-LightItalic.woff2') format('woff2'); -} - -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: normal; - src: - local('Roboto Regular'), - local('Roboto-Regular'), - url('#{$appFontPath}/roboto/Roboto-Regular.woff2') format('woff2'); -} - -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: 500; - src: - local('Roboto Medium'), - local('Roboto-Medium'), - url('#{$appFontPath}/roboto/Roboto-Medium.woff2') format('woff2'); -} - -@font-face { - font-family: 'Roboto'; - font-style: normal; - font-weight: bold; - src: - local('Roboto Bold'), - local('Roboto-Bold'), - url('#{$appFontPath}/roboto/Roboto-Bold.woff2') format('woff2'); -} - -@font-face { - font-family: 'Roboto'; - font-style: italic; - font-weight: normal; - src: - local('Roboto Italic'), - local('Roboto-Italic'), - url('#{$appFontPath}/roboto/Roboto-RegularItalic.woff2') format('woff2'); -} - -@font-face { - font-family: 'Roboto Slab'; - font-style: normal; - font-weight: normal; - src: - local('RobotoSlab Regular'), - local('RobotoSlab-Regular'), - url('#{$appFontPath}/roboto-slab/Roboto-Slab-Regular.woff2') format('woff2'); -} - -@font-face { - font-family: 'Roboto Slab'; - font-style: normal; - font-weight: bold; - src: - local('RobotoSlab Bold'), - local('RobotoSlab-Bold'), - url('#{$appFontPath}/roboto-slab/Roboto-Slab-Bold.woff2') format('woff2'); -} - // Include the common styles for Angular Material. We include this here so that you only // have to load a single css file for Angular Material in your app. // Be sure that you only ever include this mixin once! From 82e11df8352f94c6450d19d37858d005f22269db Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Fri, 15 Mar 2024 15:49:57 +0100 Subject: [PATCH 47/73] NIFI-12909 Upgraded Nimbus JOSE+JWT from 9.33.0 to 9.37.3 This closes #8518 Signed-off-by: David Handermann --- nifi-nar-bundles/nifi-accumulo-bundle/pom.xml | 2 +- nifi-nar-bundles/nifi-azure-bundle/pom.xml | 2 +- nifi-nar-bundles/nifi-framework-bundle/pom.xml | 2 +- nifi-nar-bundles/nifi-hadoop-libraries-bundle/pom.xml | 2 +- nifi-nar-bundles/nifi-hive-bundle/pom.xml | 2 +- nifi-nar-bundles/nifi-iceberg-bundle/pom.xml | 2 +- .../nifi-hbase_2-client-service-bundle/pom.xml | 2 +- nifi-registry/nifi-registry-core/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nifi-nar-bundles/nifi-accumulo-bundle/pom.xml b/nifi-nar-bundles/nifi-accumulo-bundle/pom.xml index 9183e65560f4..5fde3178d724 100644 --- a/nifi-nar-bundles/nifi-accumulo-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-accumulo-bundle/pom.xml @@ -46,7 +46,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 diff --git a/nifi-nar-bundles/nifi-azure-bundle/pom.xml b/nifi-nar-bundles/nifi-azure-bundle/pom.xml index 61b0502a4668..1353d479f6ae 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-azure-bundle/pom.xml @@ -66,7 +66,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 diff --git a/nifi-nar-bundles/nifi-framework-bundle/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/pom.xml index 215d499c7aa0..01a5e8d01e9c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/pom.xml @@ -254,7 +254,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 org.codehaus.jettison diff --git a/nifi-nar-bundles/nifi-hadoop-libraries-bundle/pom.xml b/nifi-nar-bundles/nifi-hadoop-libraries-bundle/pom.xml index 99d00666e3b5..cae3a617b6dd 100644 --- a/nifi-nar-bundles/nifi-hadoop-libraries-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-hadoop-libraries-bundle/pom.xml @@ -37,7 +37,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 diff --git a/nifi-nar-bundles/nifi-hive-bundle/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/pom.xml index 5cafb6b90c31..48c41e008998 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-hive-bundle/pom.xml @@ -102,7 +102,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml index 5991e614b10a..313daffe3caf 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml @@ -80,7 +80,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml index 3dae76fc5a7b..aa1dcaf2aef1 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml @@ -66,7 +66,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 diff --git a/nifi-registry/nifi-registry-core/pom.xml b/nifi-registry/nifi-registry-core/pom.xml index 573df5d7d77b..bdd885354413 100644 --- a/nifi-registry/nifi-registry-core/pom.xml +++ b/nifi-registry/nifi-registry-core/pom.xml @@ -109,7 +109,7 @@ com.nimbusds nimbus-jose-jwt - 9.33 + 9.37.3 com.google.guava From 82ea7a284c164ab1de4fecdc41209018a3c208c7 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Fri, 15 Mar 2024 15:54:11 -0500 Subject: [PATCH 48/73] NIFI-12910 Upgraded Spring Framework from 6.0.17 to 6.0.18 This closes #8520. - Upgraded Spring Boot from 3.2.2 to 3.2.3 for Registry - Upgraded Spring Framework from 6.1.4 to 6.1.5 for Registry Signed-off-by: Joseph Witt --- nifi-registry/pom.xml | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nifi-registry/pom.xml b/nifi-registry/pom.xml index 6dea181c9b3b..5bdeeb5d3ac1 100644 --- a/nifi-registry/pom.xml +++ b/nifi-registry/pom.xml @@ -35,8 +35,8 @@ nifi-registry-docker-maven - 6.1.4 - 3.2.2 + 6.1.5 + 3.2.3 9.22.3 9.5.0 3.12.0 diff --git a/pom.xml b/pom.xml index df42bb275172..67f63cf42e8a 100644 --- a/pom.xml +++ b/pom.xml @@ -151,7 +151,7 @@ 2.2 4.1.106.Final 6.0.0 - 6.0.17 + 6.0.18 6.2.2 2.2.20 2.2.224 From d62c8054d016bbbd96b949e804fa471c6b24d274 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Fri, 15 Mar 2024 16:01:47 -0500 Subject: [PATCH 49/73] NIFI-12911 Upgraded Jagged from 0.3.0 to 0.3.2 This closes #8522. Signed-off-by: Joseph Witt --- nifi-nar-bundles/nifi-cipher-bundle/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-cipher-bundle/pom.xml b/nifi-nar-bundles/nifi-cipher-bundle/pom.xml index b3e6233c0fba..049d23b0de6d 100644 --- a/nifi-nar-bundles/nifi-cipher-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-cipher-bundle/pom.xml @@ -36,7 +36,7 @@ com.exceptionfactory.jagged jagged-bom - 0.3.0 + 0.3.2 import pom From 495a7dd7f5b8cb2e365c25dcf4fbb37884ecdac3 Mon Sep 17 00:00:00 2001 From: bob Date: Thu, 14 Mar 2024 17:45:02 -0500 Subject: [PATCH 50/73] NIFI-12514 Added Windows support for Python venv This closes #8510 Signed-off-by: David Handermann --- .../org/apache/nifi/py4j/PythonProcess.java | 28 ++++- .../apache/nifi/py4j/PythonProcessTest.java | 109 ++++++++++++++++++ 2 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessTest.java diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java index f34199b5bcab..3cfb2b30f4ce 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java @@ -227,10 +227,7 @@ private String generateAuthToken() { private Process launchPythonProcess(final int listeningPort, final String authToken) throws IOException { final File pythonFrameworkDirectory = processConfig.getPythonFrameworkDirectory(); final File pythonApiDirectory = new File(pythonFrameworkDirectory.getParentFile(), "api"); - final File pythonCmdFile = new File(processConfig.getPythonCommand()); - final String pythonCmd = pythonCmdFile.getName(); - final File pythonCommandFile = new File(virtualEnvHome, "bin/" + pythonCmd); - final String pythonCommand = pythonCommandFile.getAbsolutePath(); + final String pythonCommand = resolvePythonCommand(); final File controllerPyFile = new File(pythonFrameworkDirectory, PYTHON_CONTROLLER_FILENAME); final ProcessBuilder processBuilder = new ProcessBuilder(); @@ -256,7 +253,7 @@ private Process launchPythonProcess(final int listeningPort, final String authTo processBuilder.environment().put("JAVA_PORT", String.valueOf(listeningPort)); processBuilder.environment().put("ENV_HOME", virtualEnvHome.getAbsolutePath()); processBuilder.environment().put("PYTHONPATH", pythonPath); - processBuilder.environment().put("PYTHON_CMD", pythonCommandFile.getAbsolutePath()); + processBuilder.environment().put("PYTHON_CMD", pythonCommand); processBuilder.environment().put("AUTH_TOKEN", authToken); // Redirect error stream to standard output stream @@ -267,6 +264,27 @@ private Process launchPythonProcess(final int listeningPort, final String authTo return processBuilder.start(); } + String resolvePythonCommand() throws IOException { + final File pythonCmdFile = new File(processConfig.getPythonCommand()); + final String pythonCmd = pythonCmdFile.getName(); + + // Find command directories according to standard Python venv conventions + final File[] virtualEnvDirectories = virtualEnvHome.listFiles((file, name) -> file.isDirectory() && (name.equals("bin") || name.equals("Scripts"))); + + final String commandExecutableDirectory; + if (virtualEnvDirectories == null || virtualEnvDirectories.length == 0) { + throw new IOException("Python binary directory could not be found in " + virtualEnvHome); + } else if( virtualEnvDirectories.length == 1) { + commandExecutableDirectory = virtualEnvDirectories[0].getName(); + } else { + // Default to bin directory for macOS and Linux + commandExecutableDirectory = "bin"; + } + + final File pythonCommandFile = new File(virtualEnvHome, commandExecutableDirectory + File.separator + pythonCmd); + return pythonCommandFile.getAbsolutePath(); + } + private void setupEnvironment() throws IOException { final File environmentCreationCompleteFile = new File(virtualEnvHome, "env-creation-complete.txt"); diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessTest.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessTest.java new file mode 100644 index 000000000000..4a1ad3263868 --- /dev/null +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/test/java/org/apache/nifi/py4j/PythonProcessTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.py4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; + +import org.apache.nifi.python.ControllerServiceTypeLookup; +import org.apache.nifi.python.PythonProcessConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.CleanupMode; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class PythonProcessTest { + + private static final String UNIX_BIN_DIR = "bin"; + + private static final String WINDOWS_SCRIPTS_DIR = "Scripts"; + + private static final String PYTHON_CMD = "python"; + + private PythonProcess pythonProcess; + + @TempDir(cleanup = CleanupMode.ON_SUCCESS) + private File virtualEnvHome; + + @Mock + private PythonProcessConfig pythonProcessConfig; + + @Mock + private ControllerServiceTypeLookup controllerServiceTypeLookup; + + @BeforeEach + public void setUp() { + this.pythonProcess = new PythonProcess(this.pythonProcessConfig, this.controllerServiceTypeLookup, virtualEnvHome, "Controller", "Controller"); + } + + @Test + void testResolvePythonCommandWindows() throws IOException { + final File scriptsDir = new File(virtualEnvHome, WINDOWS_SCRIPTS_DIR); + assertTrue(scriptsDir.mkdir()); + + when(pythonProcessConfig.getPythonCommand()).thenReturn(PYTHON_CMD); + final String result = this.pythonProcess.resolvePythonCommand(); + + final String expected = getExpectedBinaryPath(WINDOWS_SCRIPTS_DIR); + assertEquals(expected, result); + } + + @Test + void testResolvePythonCommandUnix() throws IOException { + final File binDir = new File(virtualEnvHome, UNIX_BIN_DIR); + assertTrue(binDir.mkdir()); + + when(pythonProcessConfig.getPythonCommand()).thenReturn(PYTHON_CMD); + final String result = this.pythonProcess.resolvePythonCommand(); + + final String expected = getExpectedBinaryPath(UNIX_BIN_DIR); + assertEquals(expected, result); + } + + @Test + void testResolvePythonCommandPreferBin() throws IOException { + final File binDir = new File(virtualEnvHome, UNIX_BIN_DIR); + assertTrue(binDir.mkdir()); + final File scriptsDir = new File(virtualEnvHome, WINDOWS_SCRIPTS_DIR); + assertTrue(scriptsDir.mkdir()); + + when(pythonProcessConfig.getPythonCommand()).thenReturn(PYTHON_CMD); + final String result = this.pythonProcess.resolvePythonCommand(); + + final String expected = getExpectedBinaryPath(UNIX_BIN_DIR); + assertEquals(expected, result); + } + + @Test + void testResolvePythonCommandNone() { + when(pythonProcessConfig.getPythonCommand()).thenReturn(PYTHON_CMD); + assertThrows(IOException.class, ()-> this.pythonProcess.resolvePythonCommand()); + } + + private String getExpectedBinaryPath(String binarySubDirectoryName) { + return this.virtualEnvHome.getAbsolutePath() + File.separator + binarySubDirectoryName + File.separator + PYTHON_CMD; + } +} From 940904e1eabb499c33b7d8774742a48f34281990 Mon Sep 17 00:00:00 2001 From: Matt Gilman Date: Mon, 18 Mar 2024 08:56:30 -0400 Subject: [PATCH 51/73] NIFI-12902: (#8521) - Updating packaging to account for new build output. --- .../nifi-framework/nifi-web/nifi-web-frontend/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/pom.xml index 4253b7fa7fee..2388a58ec6c6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/pom.xml @@ -80,7 +80,7 @@ ${project.build.directory}/${project.build.finalName} - ${frontend.working.dir}/dist/nifi + ${frontend.working.dir}/dist/nifi/browser false **/* From 68b880c74df0499289cce8dce7ef6a53d6ca037f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Mar 2024 23:52:02 +0000 Subject: [PATCH 52/73] NIFI-12915 Upgraded follow-redirects from 1.15.5 to 1.15.6 This closes #8524 Signed-off-by: David Handermann --- .../nifi-web-frontend/src/main/nifi/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json index 969f25b8e9cc..f44df9e0e1fd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json @@ -10012,9 +10012,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", - "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "dev": true, "funding": [ { From 4d825e6b6504b90d840b4f5a9d07467fe67159ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Mar 2024 23:08:12 +0000 Subject: [PATCH 53/73] NIFI-12915 Upgraded follow-redirects from 1.15.5 to 1.15.6 This closes #8523 Signed-off-by: David Handermann --- .../nifi-registry-web-ui/src/main/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/package-lock.json b/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/package-lock.json index 067cfb9aa50e..5cf1680f3862 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/package-lock.json +++ b/nifi-registry/nifi-registry-core/nifi-registry-web-ui/src/main/package-lock.json @@ -6926,9 +6926,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "dev": true, "funding": [ { @@ -22302,9 +22302,9 @@ } }, "follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "dev": true }, "font-awesome": { From 2303a15f8ecf5b5b48764fae3555f33103252ca6 Mon Sep 17 00:00:00 2001 From: bob Date: Sun, 17 Mar 2024 11:15:14 -0500 Subject: [PATCH 54/73] NIFI-12913 Corrected NPE for Python Log Listener ID This closes #8526 Signed-off-by: David Handermann --- .../src/main/java/org/apache/nifi/py4j/PythonProcess.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java index 3cfb2b30f4ce..2e4779bfab39 100644 --- a/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java +++ b/nifi-nar-bundles/nifi-py4j-bundle/nifi-py4j-bridge/src/main/java/org/apache/nifi/py4j/PythonProcess.java @@ -361,7 +361,9 @@ public void shutdown() { } private synchronized void killProcess() { - StandardLogLevelChangeHandler.getHandler().removeListener(logListenerId); + if (logListenerId != null) { + StandardLogLevelChangeHandler.getHandler().removeListener(logListenerId); + } if (server != null) { try { From 129130fc0966fee6dfcf8fa514841ebc07045579 Mon Sep 17 00:00:00 2001 From: lehelb Date: Sat, 16 Mar 2024 22:23:01 -0500 Subject: [PATCH 55/73] NIFI-12877 Added Sensitive Dynamic Property Support to RestLookUpService This closes #8525 Signed-off-by: David Handermann --- .../src/main/java/org/apache/nifi/lookup/RestLookupService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index e80ae5585de7..30d7939f2d43 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -30,6 +30,7 @@ import okhttp3.ResponseBody; import org.apache.nifi.annotation.behavior.DynamicProperties; import org.apache.nifi.annotation.behavior.DynamicProperty; +import org.apache.nifi.annotation.behavior.SupportsSensitiveDynamicProperties; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnDisabled; @@ -80,6 +81,7 @@ @Tags({ "rest", "lookup", "json", "xml", "http" }) @CapabilityDescription("Use a REST service to look up values.") +@SupportsSensitiveDynamicProperties @DynamicProperties({ @DynamicProperty(name = "*", value = "*", description = "All dynamic properties are added as HTTP headers with the name " + "as the header name and the value as the header value.", expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) From 12fc2f8508e2ceda306420afd4a0fc83b7bf9406 Mon Sep 17 00:00:00 2001 From: Freedom9339 Date: Thu, 14 Mar 2024 18:56:53 +0000 Subject: [PATCH 56/73] NIFI-6379 Added SSL Context to PutSNS, DeleteSQS, GetSQS, and PutSQS This closes #8505 Signed-off-by: David Handermann --- .../src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java | 1 + .../main/java/org/apache/nifi/processors/aws/sqs/DeleteSQS.java | 1 + .../src/main/java/org/apache/nifi/processors/aws/sqs/GetSQS.java | 1 + .../src/main/java/org/apache/nifi/processors/aws/sqs/PutSQS.java | 1 + 4 files changed, 4 insertions(+) diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java index c6366f713ebe..1450bbc24e89 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/sns/PutSNS.java @@ -128,6 +128,7 @@ public class PutSNS extends AbstractAwsSyncProcessor Date: Mon, 18 Mar 2024 17:33:03 -0400 Subject: [PATCH 57/73] NIFI-12912 snackbar bug (#8528) Removed unnecessary rules and class. The snackbars will be styled by Angular Material. Improved get-color-on-surface helper function. This closes #8528 --- .../nifi/src/app/_app.component-theme.scss | 20 ------------------- .../nifi/src/app/state/error/error.effects.ts | 2 +- .../src/main/nifi/src/assets/styles/_app.scss | 7 ++++++- .../src/main/nifi/src/assets/utils.scss | 5 +---- 4 files changed, 8 insertions(+), 26 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss index 9c9b304e34dd..13a92b1e40f9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/_app.component-theme.scss @@ -25,36 +25,16 @@ // Get the color palette from the color-config. $primary-palette: map.get($color-config, 'primary'); - $accent-palette: map.get($color-config, 'accent'); - $canvas-primary-palette: map.get($canvas-color-config, 'primary'); $canvas-accent-palette: map.get($canvas-color-config, 'accent'); // Get hues from palette - $primary-palette-lighter: mat.get-color-from-palette($primary-palette, lighter); $primary-palette-default: mat.get-color-from-palette($primary-palette, default); - $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); - $canvas-primary-palette-A200: mat.get-color-from-palette($canvas-primary-palette, 'A200'); $canvas-accent-palette-default: mat.get-color-from-palette($canvas-accent-palette, default); .splash { background-color: $primary-palette-default; } - // https://github.com/angular/components/issues/11426 - .nifi-snackbar .mdc-snackbar__surface { - background-color: $primary-palette-lighter !important; - } - - // https://github.com/angular/components/issues/11426 - .nifi-snackbar .mat-mdc-snack-bar-label { - color: $canvas-primary-palette-A200 !important; - } - - // https://github.com/angular/components/issues/11426 - .nifi-snackbar .mat-mdc-button:not(:disabled) .mdc-button__label { - color: $accent-palette-default; - } - .fa.fa-check.complete { color: $canvas-accent-palette-default; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/error/error.effects.ts b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/error/error.effects.ts index bc6515941f70..7344927e8526 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/error/error.effects.ts +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/app/state/error/error.effects.ts @@ -50,7 +50,7 @@ export class ErrorEffects { ofType(ErrorActions.snackBarError), map((action) => action.error), tap((error) => { - this.snackBar.open(error, 'Dismiss', { panelClass: 'nifi-snackbar', duration: 30000 }); + this.snackBar.open(error, 'Dismiss', { duration: 30000 }); }) ), { dispatch: false } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss index f5223e339594..8463bf2720c6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/styles/_app.scss @@ -175,7 +175,6 @@ $primary-palette-default: mat.get-color-from-palette($primary-palette, 'default'); $primary-palette-A400: mat.get-color-from-palette($primary-palette, 'A400'); - $accent-palette-default: mat.get-color-from-palette($accent-palette, 'default'); $accent-palette-lighter: mat.get-color-from-palette($accent-palette, 'lighter'); $warn-palette-lighter: mat.get-color-from-palette($warn-palette, lighter); @@ -190,9 +189,15 @@ $on-surface: utils.get-on-surface($canvas-color-config); $on-surface-lighter: utils.get-on-surface($canvas-color-config, lighter); + // Because snackbars have different surface colors, we need to make sure the action color has enough contrast. + $is-dark: map.get($color-config, is-dark); + $snack-surface: if($is-dark, #d9d9d9, #333333); + $snack-action-color: utils.get-color-on-surface($color-config, $snack-surface, 'accent'); + * { // Tailwind sets a default that doesn't shift with light and dark themes border-color: $on-surface-lighter; + --mat-snack-bar-button-color: #{$snack-action-color}; } .cdk-drag-disabled { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss index 7f9b8d3546bd..3e6b318801cb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/src/assets/utils.scss @@ -81,14 +81,11 @@ } @function get-color-on-surface($theme, $surface: #ffffff, $palette: 'primary') { - // Check to see if it's a dark theme - $is-dark: map.get($theme, is-dark); $color-config: mat.get-color-config($theme); $color-palette: map.get($theme, $palette); $default: mat.get-color-from-palette($color-palette, default); - $high-contrast: mat.get-color-from-palette($color-palette, if($is-dark, lighter, darker)); - + $high-contrast: mat.get-color-from-palette($color-palette, if(luminosity($default) > luminosity($surface), lighter, darker)); $on-surface: ensure-contrast($default, $surface, $high-contrast); @return $on-surface; From d9bcc8b4969a924847d9fe89784c166ce10fba0e Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Mon, 18 Mar 2024 16:17:21 -0500 Subject: [PATCH 58/73] NIFI-12920 Removed nifi-cassandra-bundle Signed-off-by: Pierre Villard This closes #8531. --- nifi-assembly/NOTICE | 5 - nifi-assembly/pom.xml | 18 - nifi-code-coverage/pom.xml | 20 - nifi-dependency-check-maven/suppressions.xml | 20 - .../pom.xml | 102 --- .../CassandraDistributedMapCache.java | 241 ------ .../nifi/controller/cassandra/QueryUtils.java | 44 -- ...g.apache.nifi.controller.ControllerService | 15 - .../src/test/java/.gitignore | 0 .../nifi-cassandra-nar/pom.xml | 51 -- .../src/main/resources/META-INF/LICENSE | 342 -------- .../src/main/resources/META-INF/NOTICE | 328 -------- .../nifi-cassandra-processors/pom.xml | 108 --- .../cassandra/AbstractCassandraProcessor.java | 533 ------------- .../processors/cassandra/PutCassandraQL.java | 406 ---------- .../cassandra/PutCassandraRecord.java | 471 ----------- .../processors/cassandra/QueryCassandra.java | 736 ------------------ .../org.apache.nifi.processor.Processor | 17 - .../AbstractCassandraProcessorTest.java | 320 -------- .../cassandra/CassandraQueryTestUtil.java | 219 ------ .../cassandra/PutCassandraQLTest.java | 434 ----------- .../cassandra/PutCassandraRecordIT.java | 132 ---- .../PutCassandraRecordInsertTest.java | 116 --- .../cassandra/PutCassandraRecordTest.java | 552 ------------- .../PutCassandraRecordUpdateTest.java | 289 ------- .../cassandra/QueryCassandraIT.java | 183 ----- .../cassandra/QueryCassandraTest.java | 594 -------------- .../nifi-cassandra-services-api-nar/pom.xml | 40 - .../src/main/resources/META-INF/LICENSE | 266 ------- .../src/main/resources/META-INF/NOTICE | 227 ------ .../nifi-cassandra-services-api/pom.xml | 40 - .../CassandraSessionProviderService.java | 35 - .../nifi-cassandra-services-nar/pom.xml | 56 -- .../src/main/resources/META-INF/LICENSE | 352 --------- .../src/main/resources/META-INF/NOTICE | 292 ------- .../nifi-cassandra-services/pom.xml | 62 -- .../service/CassandraSessionProvider.java | 315 -------- ...g.apache.nifi.controller.ControllerService | 16 - .../nifi/service/MockCassandraProcessor.java | 51 -- .../service/TestCassandraSessionProvider.java | 59 -- .../nifi-cassandra-bundle/pom.xml | 58 -- nifi-nar-bundles/pom.xml | 6 - 42 files changed, 8171 deletions(-) delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/pom.xml delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/CassandraDistributedMapCache.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/QueryUtils.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/test/java/.gitignore delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/pom.xml delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/LICENSE delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/NOTICE delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraRecord.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/QueryCassandra.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/CassandraQueryTestUtil.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraQLTest.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordIT.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordInsertTest.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordTest.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordUpdateTest.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraIT.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraTest.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/pom.xml delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/LICENSE delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/NOTICE delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/pom.xml delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/src/main/java/org/apache/nifi/cassandra/CassandraSessionProviderService.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/pom.xml delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/LICENSE delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/NOTICE delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/pom.xml delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/java/org/apache/nifi/service/CassandraSessionProvider.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/MockCassandraProcessor.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/TestCassandraSessionProvider.java delete mode 100644 nifi-nar-bundles/nifi-cassandra-bundle/pom.xml diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE index 0bee30ce6c6b..777b0f01bf5e 100644 --- a/nifi-assembly/NOTICE +++ b/nifi-assembly/NOTICE @@ -2040,11 +2040,6 @@ The following binary components are provided under the Apache Software License v the terms of a BSD style license. The original software and related information is available at http://www.jcraft.com/jsch/. - - (ASLv2) DataStax Java Driver for Apache Cassandra - Core - The following NOTICE information applies: - DataStax Java Driver for Apache Cassandra - Core - Copyright (C) 2012-2017 DataStax Inc. (ASLv2) bytebuffer-collections The following NOTICE information applies: bytebuffer-collections diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index 5e4c223e9393..8c328885ea1f 100644 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -530,24 +530,6 @@ language governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT nar - - org.apache.nifi - nifi-cassandra-nar - 2.0.0-SNAPSHOT - nar - - - org.apache.nifi - nifi-cassandra-services-api-nar - 2.0.0-SNAPSHOT - nar - - - org.apache.nifi - nifi-cassandra-services-nar - 2.0.0-SNAPSHOT - nar - org.apache.nifi nifi-registry-nar diff --git a/nifi-code-coverage/pom.xml b/nifi-code-coverage/pom.xml index 1bfa872870e6..f9cec5b9d14c 100644 --- a/nifi-code-coverage/pom.xml +++ b/nifi-code-coverage/pom.xml @@ -838,26 +838,6 @@ nifi-box-services-api 2.0.0-SNAPSHOT - - org.apache.nifi - nifi-cassandra-distributedmapcache-service - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-cassandra-processors - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-cassandra-services - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-cassandra-services-api - 2.0.0-SNAPSHOT - org.apache.nifi nifi-cdc-api diff --git a/nifi-dependency-check-maven/suppressions.xml b/nifi-dependency-check-maven/suppressions.xml index 08d0a6db6957..be9ecb301dfa 100644 --- a/nifi-dependency-check-maven/suppressions.xml +++ b/nifi-dependency-check-maven/suppressions.xml @@ -404,26 +404,6 @@ ^pkg:maven/com\.google\.guava/guava@.*$ CVE-2020-8908 - - CVE-2021-44521 applies to Apache Cassandra Server - ^pkg:maven/com\.datastax\.cassandra/cassandra\-driver\-extras@.*$ - CVE-2021-44521 - - - CVE-2020-17516 applies to Apache Cassandra Server - ^pkg:maven/com\.datastax\.cassandra/cassandra\-driver\-extras@.*$ - CVE-2020-17516 - - - CVE-2019-2684 applies to Apache Cassandra Server - ^pkg:maven/com\.datastax\.cassandra/cassandra\-driver\-extras@.*$ - CVE-2019-2684 - - - CVE-2020-13946 applies to Apache Cassandra Server - ^pkg:maven/com\.datastax\.cassandra/cassandra\-driver\-extras@.*$ - CVE-2020-13946 - Bundled versions of jQuery DataTables are not used ^pkg:javascript/jquery\.datatables@.*$ diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/pom.xml deleted file mode 100644 index 5fed4ec5a1a2..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/pom.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-cassandra-bundle - 2.0.0-SNAPSHOT - - - nifi-cassandra-distributedmapcache-service - jar - - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-record-serialization-service-api - 2.0.0-SNAPSHOT - provided - - - org.apache.nifi - nifi-utils - - - org.apache.nifi - nifi-record - - - org.apache.nifi - nifi-avro-record-utils - 2.0.0-SNAPSHOT - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.nifi - nifi-mock - - - org.apache.nifi - nifi-mock-record-utils - 2.0.0-SNAPSHOT - test - - - org.apache.nifi - nifi-distributed-cache-client-service-api - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-cassandra-services-api - 2.0.0-SNAPSHOT - - - - org.apache.nifi - nifi-cassandra-services - 2.0.0-SNAPSHOT - test - - - org.apache.nifi - nifi-ssl-context-service-api - test - - - - org.testcontainers - cassandra - ${testcontainers.version} - test - - - org.testcontainers - junit-jupiter - ${testcontainers.version} - test - - - diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/CassandraDistributedMapCache.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/CassandraDistributedMapCache.java deleted file mode 100644 index 6c1bd8e94710..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/CassandraDistributedMapCache.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.controller.cassandra; - -import com.datastax.driver.core.BoundStatement; -import com.datastax.driver.core.PreparedStatement; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnDisabled; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.cassandra.CassandraSessionProviderService; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.controller.AbstractControllerService; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.distributed.cache.client.Deserializer; -import org.apache.nifi.distributed.cache.client.DistributedMapCacheClient; -import org.apache.nifi.distributed.cache.client.Serializer; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.processor.util.StandardValidators; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.apache.nifi.controller.cassandra.QueryUtils.createDeleteStatement; -import static org.apache.nifi.controller.cassandra.QueryUtils.createExistsQuery; -import static org.apache.nifi.controller.cassandra.QueryUtils.createFetchQuery; -import static org.apache.nifi.controller.cassandra.QueryUtils.createInsertStatement; - -@Tags({"map", "cache", "distributed", "cassandra"}) -@CapabilityDescription("Provides a DistributedMapCache client that is based on Apache Cassandra.") -public class CassandraDistributedMapCache extends AbstractControllerService implements DistributedMapCacheClient { - public static final PropertyDescriptor SESSION_PROVIDER = new PropertyDescriptor.Builder() - .name("cassandra-dmc-session-provider") - .displayName("Session Provider") - .description("The client service that will configure the cassandra client connection.") - .required(true) - .identifiesControllerService(CassandraSessionProviderService.class) - .build(); - - public static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder() - .name("cassandra-dmc-table-name") - .displayName("Table Name") - .description("The name of the table where the cache will be stored.") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .build(); - - public static final PropertyDescriptor KEY_FIELD_NAME = new PropertyDescriptor.Builder() - .name("cassandra-dmc-key-field-name") - .displayName("Key Field Name") - .description("The name of the field that acts as the unique key. (The CQL type should be \"blob\")") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .build(); - - public static final PropertyDescriptor VALUE_FIELD_NAME = new PropertyDescriptor.Builder() - .name("cassandra-dmc-value-field-name") - .displayName("Value Field Name") - .description("The name of the field that will store the value. (The CQL type should be \"blob\")") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .build(); - - public static final PropertyDescriptor TTL = new PropertyDescriptor.Builder() - .name("cassandra-dmc-ttl") - .displayName("TTL") - .description("If configured, this will set a TTL (Time to Live) for each row inserted into the table so that " + - "old cache items expire after a certain period of time.") - .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .required(false) - .build(); - - public static final List DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( - SESSION_PROVIDER, TABLE_NAME, KEY_FIELD_NAME, VALUE_FIELD_NAME, TTL - )); - - private CassandraSessionProviderService sessionProviderService; - private String tableName; - private String keyField; - private String valueField; - private Long ttl; - - private Session session; - private PreparedStatement deleteStatement; - private PreparedStatement existsStatement; - private PreparedStatement fetchStatement; - private PreparedStatement insertStatement; - - @Override - protected List getSupportedPropertyDescriptors() { - return DESCRIPTORS; - } - - @OnEnabled - public void onEnabled(ConfigurationContext context) { - sessionProviderService = context.getProperty(SESSION_PROVIDER).asControllerService(CassandraSessionProviderService.class); - tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions().getValue(); - keyField = context.getProperty(KEY_FIELD_NAME).evaluateAttributeExpressions().getValue(); - valueField = context.getProperty(VALUE_FIELD_NAME).evaluateAttributeExpressions().getValue(); - if (context.getProperty(TTL).isSet()) { - ttl = context.getProperty(TTL).evaluateAttributeExpressions().asTimePeriod(TimeUnit.SECONDS); - } - - session = sessionProviderService.getCassandraSession(); - - deleteStatement = session.prepare(createDeleteStatement(keyField, tableName)); - existsStatement = session.prepare(createExistsQuery(keyField, tableName)); - fetchStatement = session.prepare(createFetchQuery(keyField, valueField, tableName)); - insertStatement = session.prepare(createInsertStatement(keyField, valueField, tableName, ttl)); - } - - @OnDisabled - public void onDisabled() { - session = null; - deleteStatement = null; - existsStatement = null; - fetchStatement = null; - insertStatement = null; - } - - @Override - public boolean putIfAbsent(K k, V v, Serializer keySerializer, Serializer valueSerializer) throws IOException { - if (containsKey(k, keySerializer)) { - return false; - } else { - put(k, v, keySerializer, valueSerializer); - return true; - } - } - - @Override - public V getAndPutIfAbsent(K k, V v, Serializer keySerializer, Serializer valueSerializer, Deserializer deserializer) throws IOException { - V got = get(k, keySerializer, deserializer); - boolean wasAbsent = putIfAbsent(k, v, keySerializer, valueSerializer); - - return !wasAbsent ? got : null; - } - - @Override - public boolean containsKey(K k, Serializer serializer) throws IOException { - byte[] key = serializeKey(k, serializer); - - BoundStatement statement = existsStatement.bind(); - ByteBuffer buffer = ByteBuffer.wrap(key); - statement.setBytes(0, buffer); - ResultSet rs =session.execute(statement); - Iterator iterator = rs.iterator(); - - if (iterator.hasNext()) { - Row row = iterator.next(); - long value = row.getLong("exist_count"); - return value > 0; - } else { - return false; - } - } - - @Override - public void put(K k, V v, Serializer keySerializer, Serializer valueSerializer) throws IOException { - BoundStatement statement = insertStatement.bind(); - statement.setBytes(0, ByteBuffer.wrap(serializeKey(k, keySerializer))); - statement.setBytes(1, ByteBuffer.wrap(serializeValue(v, valueSerializer))); - session.execute(statement); - } - - @Override - public V get(K k, Serializer serializer, Deserializer deserializer) throws IOException { - BoundStatement boundStatement = fetchStatement.bind(); - boundStatement.setBytes(0, ByteBuffer.wrap(serializeKey(k, serializer))); - ResultSet rs = session.execute(boundStatement); - Iterator iterator = rs.iterator(); - if (!iterator.hasNext()) { - return null; - } - - Row fetched = iterator.next(); - ByteBuffer buffer = fetched.getBytes(valueField); - - byte[] content = buffer.array(); - - return deserializer.deserialize(content); - } - - @Override - public void close() throws IOException { - - } - - @Override - public boolean remove(K k, Serializer serializer) throws IOException { - BoundStatement delete = deleteStatement.bind(); - delete.setBytes(0, ByteBuffer.wrap(serializeKey(k, serializer))); - session.execute(delete); - - return true; - } - - private byte[] serializeKey(K k, Serializer keySerializer) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - keySerializer.serialize(k, out); - out.close(); - - return out.toByteArray(); - } - - private byte[] serializeValue(V v, Serializer valueSerializer) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - valueSerializer.serialize(v, out); - out.close(); - - return out.toByteArray(); - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/QueryUtils.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/QueryUtils.java deleted file mode 100644 index cc42dd75d84f..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/java/org/apache/nifi/controller/cassandra/QueryUtils.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.controller.cassandra; - -public class QueryUtils { - private QueryUtils() {} - - public static String createDeleteStatement(String keyField, String table) { - return String.format("DELETE FROM %s WHERE %s = ?", table, keyField); - } - - public static String createExistsQuery(String keyField, String table) { - return String.format("SELECT COUNT(*) as exist_count FROM %s WHERE %s = ?", table, keyField); - } - - public static String createFetchQuery(String keyField, String valueField, String table) { - return String.format("SELECT %s FROM %s WHERE %s = ?", valueField, table, keyField); - } - - public static String createInsertStatement(String keyField, String valueField, String table, Long ttl) { - String retVal = String.format("INSERT INTO %s (%s, %s) VALUES(?, ?)", table, keyField, valueField); - - if (ttl != null) { - retVal += String.format(" using ttl %d", ttl); - } - - return retVal; - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService deleted file mode 100644 index c3d32c851298..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ /dev/null @@ -1,15 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.controller.cassandra.CassandraDistributedMapCache \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/test/java/.gitignore b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-distributedmapcache-service/src/test/java/.gitignore deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/pom.xml deleted file mode 100644 index 3063f64ef8bd..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/pom.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-cassandra-bundle - 2.0.0-SNAPSHOT - - - nifi-cassandra-nar - nar - - - - - - com.google.guava - guava - provided - - - - - - - org.apache.nifi - nifi-cassandra-services-api-nar - 2.0.0-SNAPSHOT - nar - - - org.apache.nifi - nifi-cassandra-processors - - - diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/LICENSE deleted file mode 100644 index 2672f6e36a1d..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,342 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -APACHE NIFI SUBCOMPONENTS: - -The Apache NiFi project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -This product bundles 'libffi' which is available under an MIT style license. - libffi - Copyright (c) 1996-2014 Anthony Green, Red Hat, Inc and others. - see https://github.com/java-native-access/jna/blob/master/native/libffi/LICENSE - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -This product bundles 'asm' which is available under a 3-Clause BSD style license. -For details see http://asm.ow2.org/asmdex-license.html - - Copyright (c) 2012 France Télécom - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. - - The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' - under an MIT style license. - - Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - -The binary distribution of this product bundles 'JNR x86asm' under an MIT -style license. - - Copyright (C) 2010 Wayne Meissner - Copyright (c) 2008-2009, Petr Kobalicek - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - - The binary distribution of this product bundles 'ParaNamer' and 'Paranamer Core' - which is available under a BSD style license. - - Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 6bd6684a5156..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,328 +0,0 @@ -nifi-cassandra-nar -Copyright 2016-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -****************** -Apache Software License v2 -****************** - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) DataStax Java Driver for Apache Cassandra - Core - The following NOTICE information applies: - DataStax Java Driver for Apache Cassandra - Core - Copyright (C) 2012-2017 DataStax Inc. - - (ASLv2) Apache Avro - The following NOTICE information applies: - Apache Avro - Copyright 2009-2017 The Apache Software Foundation - - (ASLv2) Jackson JSON processor - The following NOTICE information applies: - # Jackson JSON processor - - Jackson is a high-performance, Free/Open Source JSON processing library. - It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has - been in development since 2007. - It is currently developed by a community of developers, as well as supported - commercially by FasterXML.com. - - ## Licensing - - Jackson core and extension components may licensed under different licenses. - To find the details that apply to this artifact see the accompanying LICENSE file. - For more information, including possible other licensing options, contact - FasterXML.com (http://fasterxml.com). - - ## Credits - - A list of contributors may be found from CREDITS file, which is included - in some artifacts (usually source distributions); but is always available - from the source code management (SCM) system project uses. - - (ASLv2) Apache Commons Codec - The following NOTICE information applies: - Apache Commons Codec - Copyright 2002-2014 The Apache Software Foundation - - src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java - contains test data from http://aspell.net/test/orig/batch0.tab. - Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - - =============================================================================== - - The content of package org.apache.commons.codec.language.bm has been translated - from the original php source code available at http://stevemorse.org/phoneticinfo.htm - with permission from the original authors. - Original source copyright: - Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - - (ASLv2) Apache Commons Compress - The following NOTICE information applies: - Apache Commons Compress - Copyright 2002-2017 The Apache Software Foundation - - The files in the package org.apache.commons.compress.archivers.sevenz - were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), - which has been placed in the public domain: - - "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) - - (ASLv2) Apache Commons IO - The following NOTICE information applies: - Apache Commons IO - Copyright 2002-2016 The Apache Software Foundation - - (ASLv2) Apache Commons Lang - The following NOTICE information applies: - Apache Commons Lang - Copyright 2001-2017 The Apache Software Foundation - - This product includes software from the Spring Framework, - under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - - (ASLv2) Guava - The following NOTICE information applies: - Guava - Copyright 2015 The Guava Authors - - (ASLv2) Dropwizard Metrics - The following NOTICE information applies: - Copyright (c) 2010-2013 Coda Hale, Yammer.com - - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - - (ASLv2) The Netty Project - The following NOTICE information applies: - Copyright 2014 The Netty Project - ------------------------------------------------------------------------------- - This product contains the extensions to Java Collections Framework which has - been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - - This product contains a modified version of Robert Harder's Public Domain - Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - - This product contains a modified portion of 'Webbit', an event based - WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - - This product contains a modified portion of 'SLF4J', a simple logging - facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - - This product contains a modified portion of 'Apache Harmony', an open source - Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - - This product contains a modified portion of 'jbzip2', a Java bzip2 compression - and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - - This product contains a modified portion of 'libdivsufsort', a C API library to construct - the suffix array and the Burrows-Wheeler transformed string for any input string of - a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - - This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - - This product optionally depends on 'JZlib', a re-implementation of zlib in - pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - - This product optionally depends on 'Compress-LZF', a Java library for encoding and - decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - - This product optionally depends on 'lz4', a LZ4 Java compression - and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - - This product optionally depends on 'lzma-java', a LZMA Java compression - and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - - This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression - and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - - This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data - interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - - This product optionally depends on 'Bouncy Castle Crypto APIs' to generate - a temporary self-signed X.509 certificate when the JVM does not provide the - equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - - This product optionally depends on 'Snappy', a compression library produced - by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - - This product optionally depends on 'JBoss Marshalling', an alternative Java - serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - - This product optionally depends on 'Caliper', Google's micro- - benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - - This product optionally depends on 'Apache Log4J', a logging framework, which - can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - - This product optionally depends on 'Aalto XML', an ultra-high performance - non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - - This product contains a modified version of 'HPACK', a Java implementation of - the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - - This product contains a modified portion of 'Apache Commons Lang', a Java library - provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - This product contains a forked and modified version of Tomcat Native - - * LICENSE: - * ASL2 - * HOMEPAGE: - * http://tomcat.apache.org/native-doc/ - * https://svn.apache.org/repos/asf/tomcat/native/ - - (ASLv2) Objenesis - The following NOTICE information applies: - Objenesis - Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita - - (ASLv2) Snappy Java - The following NOTICE information applies: - This product includes software developed by Google - Snappy: http://code.google.com/p/snappy/ (New BSD License) - - This product includes software developed by Apache - PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ - (Apache 2.0 license) - - This library containd statically linked libstdc++. This inclusion is allowed by - "GCC RUntime Library Exception" - http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html - -************************ -Eclipse Public License 1.0 -************************ - -The following binary components are provided under the Eclipse Public License 1.0. See project link for details. - - (EPL 1.0) JNR Posix ( jnr.posix ) https://github.com/jnr/jnr-posix/blob/master/LICENSE.txt - -***************** -Public Domain -***************** - -The following binary components are provided to the 'Public Domain'. See project link for details. - - (Public Domain) XZ for Java (org.tukaani:xz:jar:1.5 - http://tukaani.org/xz/java.html diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml deleted file mode 100644 index 45094b06a040..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/pom.xml +++ /dev/null @@ -1,108 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-cassandra-bundle - 2.0.0-SNAPSHOT - - - nifi-cassandra-processors - jar - - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-utils - - - org.apache.nifi - nifi-properties - - - com.datastax.cassandra - cassandra-driver-core - ${cassandra.sdk.version} - provided - - - com.datastax.cassandra - cassandra-driver-extras - ${cassandra.sdk.version} - - - org.apache.nifi - nifi-ssl-context-service-api - - - org.apache.avro - avro - - - org.apache.nifi - nifi-cassandra-services-api - provided - - - org.apache.nifi - nifi-cassandra-services - 2.0.0-SNAPSHOT - test - - - org.apache.nifi - nifi-record-serialization-service-api - compile - - - org.apache.nifi - nifi-record - compile - - - - org.apache.nifi - nifi-mock - - - org.apache.nifi - nifi-mock-record-utils - - - org.apache.commons - commons-text - - - - org.testcontainers - cassandra - ${testcontainers.version} - test - - - org.testcontainers - junit-jupiter - ${testcontainers.version} - test - - - diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java deleted file mode 100644 index 844f0c5925b3..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessor.java +++ /dev/null @@ -1,533 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.CodecRegistry; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.Metadata; -import com.datastax.driver.core.ProtocolOptions; -import com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.SSLOptions; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.TypeCodec; -import com.datastax.driver.core.exceptions.AuthenticationException; -import com.datastax.driver.core.exceptions.NoHostAvailableException; -import com.datastax.driver.extras.codecs.arrays.ObjectArrayCodec; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import javax.net.ssl.SSLContext; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.cassandra.CassandraSessionProviderService; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.security.util.ClientAuth; -import org.apache.nifi.ssl.SSLContextService; - -/** - * AbstractCassandraProcessor is a base class for Cassandra processors and contains logic and variables common to most - * processors integrating with Apache Cassandra. - */ -public abstract class AbstractCassandraProcessor extends AbstractProcessor { - - public static final int DEFAULT_CASSANDRA_PORT = 9042; - - // Common descriptors - static final PropertyDescriptor CONNECTION_PROVIDER_SERVICE = new PropertyDescriptor.Builder() - .name("cassandra-connection-provider") - .displayName("Cassandra Connection Provider") - .description("Specifies the Cassandra connection providing controller service to be used to connect to Cassandra cluster.") - .required(false) - .identifiesControllerService(CassandraSessionProviderService.class) - .build(); - - static final PropertyDescriptor CONTACT_POINTS = new PropertyDescriptor.Builder() - .name("Cassandra Contact Points") - .description("Contact points are addresses of Cassandra nodes. The list of contact points should be " - + "comma-separated and in hostname:port format. Example node1:port,node2:port,...." - + " The default client port for Cassandra is 9042, but the port(s) must be explicitly specified.") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.HOSTNAME_PORT_LIST_VALIDATOR) - .build(); - - static final PropertyDescriptor KEYSPACE = new PropertyDescriptor.Builder() - .name("Keyspace") - .description("The Cassandra Keyspace to connect to. If no keyspace is specified, the query will need to " + - "include the keyspace name before any table reference, in case of 'query' native processors or " + - "if the processor exposes the 'Table' property, the keyspace name has to be provided with the " + - "table name in the form of .") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() - .name("SSL Context Service") - .description("The SSL Context Service used to provide client certificate information for TLS/SSL " - + "connections.") - .required(false) - .identifiesControllerService(SSLContextService.class) - .build(); - - static final PropertyDescriptor CLIENT_AUTH = new PropertyDescriptor.Builder() - .name("Client Auth") - .description("Client authentication policy when connecting to secure (TLS/SSL) cluster. " - + "Possible values are REQUIRED, WANT, NONE. This property is only used when an SSL Context " - + "has been defined and enabled.") - .required(false) - .allowableValues(ClientAuth.values()) - .defaultValue("REQUIRED") - .build(); - - static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder() - .name("Username") - .description("Username to access the Cassandra cluster") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder() - .name("Password") - .description("Password to access the Cassandra cluster") - .required(false) - .sensitive(true) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - static final PropertyDescriptor CONSISTENCY_LEVEL = new PropertyDescriptor.Builder() - .name("Consistency Level") - .description("The strategy for how many replicas must respond before results are returned.") - .required(false) - .allowableValues(ConsistencyLevel.values()) - .defaultValue("ONE") - .build(); - - static final PropertyDescriptor COMPRESSION_TYPE = new PropertyDescriptor.Builder() - .name("Compression Type") - .description("Enable compression at transport-level requests and responses") - .required(false) - .allowableValues(ProtocolOptions.Compression.values()) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .defaultValue("NONE") - .build(); - - static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder() - .name("Character Set") - .description("Specifies the character set of the record data.") - .required(true) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .defaultValue("UTF-8") - .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR) - .build(); - - static final Relationship REL_SUCCESS = new Relationship.Builder() - .name("success") - .description("A FlowFile is transferred to this relationship if the operation completed successfully.") - .build(); - - static final Relationship REL_FAILURE = new Relationship.Builder() - .name("failure") - .description("A FlowFile is transferred to this relationship if the operation failed.") - .build(); - - static final Relationship REL_RETRY = new Relationship.Builder().name("retry") - .description("A FlowFile is transferred to this relationship if the operation cannot be completed but attempting " - + "it again may succeed.") - .build(); - - protected static List descriptors = new ArrayList<>(); - - static { - descriptors.add(CONNECTION_PROVIDER_SERVICE); - descriptors.add(CONTACT_POINTS); - descriptors.add(KEYSPACE); - descriptors.add(PROP_SSL_CONTEXT_SERVICE); - descriptors.add(CLIENT_AUTH); - descriptors.add(USERNAME); - descriptors.add(PASSWORD); - descriptors.add(CONSISTENCY_LEVEL); - descriptors.add(COMPRESSION_TYPE); - descriptors.add(CHARSET); - } - - protected final AtomicReference cluster = new AtomicReference<>(null); - protected final AtomicReference cassandraSession = new AtomicReference<>(null); - - protected static final CodecRegistry codecRegistry = new CodecRegistry(); - - @Override - protected Collection customValidate(ValidationContext validationContext) { - Set results = new HashSet<>(); - - // Ensure that if username or password is set, then the other is too - String userName = validationContext.getProperty(USERNAME).evaluateAttributeExpressions().getValue(); - String password = validationContext.getProperty(PASSWORD).evaluateAttributeExpressions().getValue(); - - if (StringUtils.isEmpty(userName) != StringUtils.isEmpty(password)) { - results.add(new ValidationResult.Builder().subject("Username / Password configuration").valid(false).explanation( - "If username or password is specified, then the other must be specified as well").build()); - } - - // Ensure that both Connection provider service and the processor specific configurations are not provided - boolean connectionProviderIsSet = validationContext.getProperty(CONNECTION_PROVIDER_SERVICE).isSet(); - boolean contactPointsIsSet = validationContext.getProperty(CONTACT_POINTS).isSet(); - - if (connectionProviderIsSet && contactPointsIsSet) { - results.add(new ValidationResult.Builder().subject("Cassandra configuration").valid(false).explanation("both " + CONNECTION_PROVIDER_SERVICE.getDisplayName() + - " and processor level Cassandra configuration cannot be provided at the same time.").build()); - } - - if (!connectionProviderIsSet && !contactPointsIsSet) { - results.add(new ValidationResult.Builder().subject("Cassandra configuration").valid(false).explanation("either " + CONNECTION_PROVIDER_SERVICE.getDisplayName() + - " or processor level Cassandra configuration has to be provided.").build()); - } - - return results; - } - - @OnScheduled - public void onScheduled(ProcessContext context) { - final boolean connectionProviderIsSet = context.getProperty(CONNECTION_PROVIDER_SERVICE).isSet(); - - registerAdditionalCodecs(); - - if (connectionProviderIsSet) { - CassandraSessionProviderService sessionProvider = context.getProperty(CONNECTION_PROVIDER_SERVICE).asControllerService(CassandraSessionProviderService.class); - cluster.set(sessionProvider.getCluster()); - cassandraSession.set(sessionProvider.getCassandraSession()); - return; - } - - try { - connectToCassandra(context); - } catch (NoHostAvailableException nhae) { - getLogger().error("No host in the Cassandra cluster can be contacted successfully to execute this statement", nhae); - getLogger().error(nhae.getCustomMessage(10, true, false)); - throw new ProcessException(nhae); - } catch (AuthenticationException ae) { - getLogger().error("Invalid username/password combination", ae); - throw new ProcessException(ae); - } - } - - void connectToCassandra(ProcessContext context) { - if (cluster.get() == null) { - ComponentLog log = getLogger(); - final String contactPointList = context.getProperty(CONTACT_POINTS).evaluateAttributeExpressions().getValue(); - final String consistencyLevel = context.getProperty(CONSISTENCY_LEVEL).getValue(); - final String compressionType = context.getProperty(COMPRESSION_TYPE).getValue(); - List contactPoints = getContactPoints(contactPointList); - - // Set up the client for secure (SSL/TLS communications) if configured to do so - final SSLContextService sslService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); - final SSLContext sslContext; - - if (sslService != null) { - sslContext = sslService.createContext(); - } else { - sslContext = null; - } - - final String username, password; - PropertyValue usernameProperty = context.getProperty(USERNAME).evaluateAttributeExpressions(); - PropertyValue passwordProperty = context.getProperty(PASSWORD).evaluateAttributeExpressions(); - - if (usernameProperty != null && passwordProperty != null) { - username = usernameProperty.getValue(); - password = passwordProperty.getValue(); - } else { - username = null; - password = null; - } - - // Create the cluster and connect to it - Cluster newCluster = createCluster(contactPoints, sslContext, username, password, compressionType); - PropertyValue keyspaceProperty = context.getProperty(KEYSPACE).evaluateAttributeExpressions(); - - final Session newSession; - // For Java 11, the getValue() call was added so the test could pass - if (keyspaceProperty != null && keyspaceProperty.getValue() != null) { - newSession = newCluster.connect(keyspaceProperty.getValue()); - } else { - newSession = newCluster.connect(); - } - - newCluster.getConfiguration().getQueryOptions().setConsistencyLevel(ConsistencyLevel.valueOf(consistencyLevel)); - Metadata metadata = newCluster.getMetadata(); - - log.info("Connected to Cassandra cluster: {}", new Object[]{metadata.getClusterName()}); - - cluster.set(newCluster); - cassandraSession.set(newSession); - } - } - - protected void registerAdditionalCodecs() { - // Conversion between a String[] and a list of varchar - CodecRegistry.DEFAULT_INSTANCE.register(new ObjectArrayCodec<>( - DataType.list(DataType.varchar()), - String[].class, - TypeCodec.varchar())); - } - - /** - * Uses a Cluster.Builder to create a Cassandra cluster reference using the given parameters - * - * @param contactPoints The contact points (hostname:port list of Cassandra nodes) - * @param sslContext The SSL context (used for secure connections) - * @param username The username for connection authentication - * @param password The password for connection authentication - * @param compressionType Enable compression at transport-level requests and responses. - * @return A reference to the Cluster object associated with the given Cassandra configuration - */ - protected Cluster createCluster(List contactPoints, SSLContext sslContext, - String username, String password, String compressionType) { - Cluster.Builder builder = Cluster.builder().addContactPointsWithPorts(contactPoints); - if (sslContext != null) { - final SSLOptions sslOptions = RemoteEndpointAwareJdkSSLOptions.builder() - .withSSLContext(sslContext) - .build(); - - builder = builder.withSSL(sslOptions); - if (ProtocolOptions.Compression.SNAPPY.name().equals(compressionType)) { - builder = builder.withCompression(ProtocolOptions.Compression.SNAPPY); - } else if (ProtocolOptions.Compression.LZ4.name().equals(compressionType)) { - builder = builder.withCompression(ProtocolOptions.Compression.LZ4); - } - } - - if (username != null && password != null) { - builder = builder.withCredentials(username, password); - } - - return builder.build(); - } - - public void stop(ProcessContext context) { - // We don't want to close the connection when using 'Cassandra Connection Provider' - // because each time @OnUnscheduled/@OnShutdown annotated method is triggered on a - // processor, the connection would be closed which is not ideal for a centralized - // connection provider controller service - if (!context.getProperty(CONNECTION_PROVIDER_SERVICE).isSet()) { - if (cassandraSession.get() != null) { - cassandraSession.get().close(); - cassandraSession.set(null); - } - if (cluster.get() != null) { - cluster.get().close(); - cluster.set(null); - } - } - } - - - protected static Object getCassandraObject(Row row, int i, DataType dataType) { - if (dataType.equals(DataType.blob())) { - return row.getBytes(i); - - } else if (dataType.equals(DataType.varint()) || dataType.equals(DataType.decimal())) { - // Avro can't handle BigDecimal and BigInteger as numbers - it will throw an - // AvroRuntimeException such as: "Unknown datum type: java.math.BigDecimal: 38" - return row.getObject(i).toString(); - - } else if (dataType.equals(DataType.cboolean())) { - return row.getBool(i); - - } else if (dataType.equals(DataType.cint())) { - return row.getInt(i); - - } else if (dataType.equals(DataType.bigint()) - || dataType.equals(DataType.counter())) { - return row.getLong(i); - - } else if (dataType.equals(DataType.ascii()) - || dataType.equals(DataType.text()) - || dataType.equals(DataType.varchar())) { - return row.getString(i); - - } else if (dataType.equals(DataType.cfloat())) { - return row.getFloat(i); - - } else if (dataType.equals(DataType.cdouble())) { - return row.getDouble(i); - - } else if (dataType.equals(DataType.timestamp())) { - return row.getTimestamp(i); - } else if (dataType.equals(DataType.date())) { - return row.getDate(i); - - } else if (dataType.equals(DataType.time())) { - return row.getTime(i); - - } else if (dataType.isCollection()) { - - List typeArguments = dataType.getTypeArguments(); - if (typeArguments == null || typeArguments.size() == 0) { - throw new IllegalArgumentException("Column[" + i + "] " + dataType.getName() - + " is a collection but no type arguments were specified!"); - } - // Get the first type argument, to be used for lists and sets (and the first in a map) - DataType firstArg = typeArguments.get(0); - TypeCodec firstCodec = codecRegistry.codecFor(firstArg); - if (dataType.equals(DataType.set(firstArg))) { - return row.getSet(i, firstCodec.getJavaType()); - } else if (dataType.equals(DataType.list(firstArg))) { - return row.getList(i, firstCodec.getJavaType()); - } else { - // Must be an n-arg collection like map - DataType secondArg = typeArguments.get(1); - TypeCodec secondCodec = codecRegistry.codecFor(secondArg); - if (dataType.equals(DataType.map(firstArg, secondArg))) { - return row.getMap(i, firstCodec.getJavaType(), secondCodec.getJavaType()); - } - } - - } else { - // The different types that we support are numbers (int, long, double, float), - // as well as boolean values and Strings. Since Avro doesn't provide - // timestamp types, we want to convert those to Strings. So we will cast anything other - // than numbers or booleans to strings by using the toString() method. - return row.getObject(i).toString(); - } - return null; - } - - /** - * This method will create a schema a union field consisting of null and the specified type. - * - * @param dataType The data type of the field - */ - protected static Schema getUnionFieldType(String dataType) { - return SchemaBuilder.builder().unionOf().nullBuilder().endNull().and().type(getSchemaForType(dataType)).endUnion(); - } - - /** - * This method will create an Avro schema for the specified type. - * - * @param dataType The data type of the field - */ - protected static Schema getSchemaForType(final String dataType) { - final SchemaBuilder.TypeBuilder typeBuilder = SchemaBuilder.builder(); - final Schema returnSchema = switch (dataType) { - case "string" -> typeBuilder.stringType(); - case "boolean" -> typeBuilder.booleanType(); - case "int" -> typeBuilder.intType(); - case "long" -> typeBuilder.longType(); - case "float" -> typeBuilder.floatType(); - case "double" -> typeBuilder.doubleType(); - case "bytes" -> typeBuilder.bytesType(); - default -> throw new IllegalArgumentException("Unknown Avro primitive type: " + dataType); - }; - return returnSchema; - } - - protected static String getPrimitiveAvroTypeFromCassandraType(final DataType dataType) { - // Map types from Cassandra to Avro where possible - if (dataType.equals(DataType.ascii()) - || dataType.equals(DataType.text()) - || dataType.equals(DataType.varchar()) - // Nonstandard types represented by this processor as a string - || dataType.equals(DataType.timestamp()) - || dataType.equals(DataType.timeuuid()) - || dataType.equals(DataType.uuid()) - || dataType.equals(DataType.inet()) - || dataType.equals(DataType.varint())) { - return "string"; - - } else if (dataType.equals(DataType.cboolean())) { - return "boolean"; - - } else if (dataType.equals(DataType.cint())) { - return "int"; - - } else if (dataType.equals(DataType.bigint()) - || dataType.equals(DataType.counter())) { - return "long"; - - } else if (dataType.equals(DataType.cfloat())) { - return "float"; - - } else if (dataType.equals(DataType.cdouble())) { - return "double"; - - } else if (dataType.equals(DataType.blob())) { - return "bytes"; - - } else { - throw new IllegalArgumentException("createSchema: Unknown Cassandra data type " + dataType.getName() - + " cannot be converted to Avro type"); - } - } - - protected static DataType getPrimitiveDataTypeFromString(String dataTypeName) { - Set primitiveTypes = DataType.allPrimitiveTypes(); - for (DataType primitiveType : primitiveTypes) { - if (primitiveType.toString().equals(dataTypeName)) { - return primitiveType; - } - } - return null; - } - - /** - * Gets a list of InetSocketAddress objects that correspond to host:port entries for Cassandra contact points - * - * @param contactPointList A comma-separated list of Cassandra contact points (host:port,host2:port2, etc.) - * @return List of InetSocketAddresses for the Cassandra contact points - */ - protected List getContactPoints(String contactPointList) { - - if (contactPointList == null) { - return null; - } - final String[] contactPointStringList = contactPointList.split(","); - List contactPoints = new ArrayList<>(); - - for (String contactPointEntry : contactPointStringList) { - - String[] addresses = contactPointEntry.split(":"); - final String hostName = addresses[0].trim(); - final int port = (addresses.length > 1) ? Integer.parseInt(addresses[1].trim()) : DEFAULT_CASSANDRA_PORT; - - contactPoints.add(new InetSocketAddress(hostName, port)); - } - return contactPoints; - } -} - diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java deleted file mode 100644 index cb00be83ce5c..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.BoundStatement; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.PreparedStatement; -import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.TypeCodec; -import com.datastax.driver.core.exceptions.InvalidTypeException; -import com.datastax.driver.core.exceptions.NoHostAvailableException; -import com.datastax.driver.core.exceptions.QueryExecutionException; -import com.datastax.driver.core.exceptions.QueryValidationException; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import org.apache.nifi.annotation.behavior.SystemResourceConsideration; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.ReadsAttribute; -import org.apache.nifi.annotation.behavior.ReadsAttributes; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.annotation.behavior.SystemResource; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.annotation.lifecycle.OnStopped; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.stream.io.StreamUtils; -import org.apache.nifi.util.StringUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@SupportsBatching -@Tags({"cassandra", "cql", "put", "insert", "update", "set"}) -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@CapabilityDescription("Execute provided Cassandra Query Language (CQL) statement on a Cassandra 1.x, 2.x, or 3.0.x cluster. " - + "The content of an incoming FlowFile is expected to be the CQL command to execute. The CQL command may use " - + "the ? to escape parameters. In this case, the parameters to use must exist as FlowFile attributes with the " - + "naming convention cql.args.N.type and cql.args.N.value, where N is a positive integer. The cql.args.N.type " - + "is expected to be a lowercase string indicating the Cassandra type.") -@ReadsAttributes({ - @ReadsAttribute(attribute = "cql.args.N.type", - description = "Incoming FlowFiles are expected to be parameterized CQL statements. The type of each " - + "parameter is specified as a lowercase string corresponding to the Cassandra data type (text, " - + "int, boolean, e.g.). In the case of collections, the primitive type(s) of the elements in the " - + "collection should be comma-delimited, follow the collection type, and be enclosed in angle brackets " - + "(< and >), for example set or map."), - @ReadsAttribute(attribute = "cql.args.N.value", - description = "Incoming FlowFiles are expected to be parameterized CQL statements. The value of the " - + "parameters are specified as cql.args.1.value, cql.args.2.value, cql.args.3.value, and so on. The " - + " type of the cql.args.1.value parameter is specified by the cql.args.1.type attribute.") -}) -@SystemResourceConsideration(resource = SystemResource.MEMORY) -public class PutCassandraQL extends AbstractCassandraProcessor { - - public static final PropertyDescriptor STATEMENT_TIMEOUT = new PropertyDescriptor.Builder() - .name("Max Wait Time") - .displayName("Max Wait Time") - .description("The maximum amount of time allowed for a running CQL select query. Must be of format " - + " where is a non-negative integer and TimeUnit is a supported " - + "Time Unit, such as: nanos, millis, secs, mins, hrs, days. A value of zero means there is no limit. ") - .defaultValue("0 seconds") - .required(true) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) - .build(); - - public static final PropertyDescriptor STATEMENT_CACHE_SIZE = new PropertyDescriptor.Builder() - .name("putcql-stmt-cache-size") - .displayName("Statement Cache Size") - .description("The maximum number of CQL Prepared Statements to cache. This can improve performance if many incoming flow files have the same CQL statement " - + "with different values for the parameters. If this property is set to zero, the cache is effectively disabled.") - .defaultValue("0") - .required(true) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) - .build(); - - private final static List propertyDescriptors; - - private final static Set relationships; - - private static final Pattern CQL_TYPE_ATTRIBUTE_PATTERN = Pattern.compile("cql\\.args\\.(\\d+)\\.type"); - - // Matches on top-level type (primitive types like text,int) and also for collections (like list and map) - private static final Pattern CQL_TYPE_PATTERN = Pattern.compile("([^<]+)(<([^,>]+)(,([^,>]+))*>)?"); - - /** - * LRU cache for the compiled patterns. The size of the cache is determined by the value of the Statement Cache Size property - */ - @VisibleForTesting - private ConcurrentMap statementCache; - - /* - * Will ensure that the list of property descriptors is build only once. - * Will also create a Set of relationships - */ - static { - List _propertyDescriptors = new ArrayList<>(); - _propertyDescriptors.addAll(descriptors); - _propertyDescriptors.add(STATEMENT_TIMEOUT); - _propertyDescriptors.add(STATEMENT_CACHE_SIZE); - propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors); - - Set _relationships = new HashSet<>(); - _relationships.add(REL_SUCCESS); - _relationships.add(REL_FAILURE); - _relationships.add(REL_RETRY); - relationships = Collections.unmodifiableSet(_relationships); - } - - @Override - protected List getSupportedPropertyDescriptors() { - return propertyDescriptors; - } - - @Override - public Set getRelationships() { - return relationships; - } - - - @OnScheduled - public void onScheduled(final ProcessContext context) { - super.onScheduled(context); - - // Initialize the prepared statement cache - int statementCacheSize = context.getProperty(STATEMENT_CACHE_SIZE).evaluateAttributeExpressions().asInteger(); - statementCache = CacheBuilder.newBuilder() - .maximumSize(statementCacheSize) - .build() - .asMap(); - } - - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - ComponentLog logger = getLogger(); - FlowFile flowFile = session.get(); - if (flowFile == null) { - return; - } - - final long startNanos = System.nanoTime(); - final long statementTimeout = context.getProperty(STATEMENT_TIMEOUT).evaluateAttributeExpressions(flowFile).asTimePeriod(TimeUnit.MILLISECONDS); - final Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue()); - - // The documentation for the driver recommends the session remain open the entire time the processor is running - // and states that it is thread-safe. This is why connectionSession is not in a try-with-resources. - final Session connectionSession = cassandraSession.get(); - - String cql = getCQL(session, flowFile, charset); - try { - PreparedStatement statement = statementCache.get(cql); - if(statement == null) { - statement = connectionSession.prepare(cql); - statementCache.put(cql, statement); - } - BoundStatement boundStatement = statement.bind(); - - Map attributes = flowFile.getAttributes(); - for (final Map.Entry entry : attributes.entrySet()) { - final String key = entry.getKey(); - final Matcher matcher = CQL_TYPE_ATTRIBUTE_PATTERN.matcher(key); - if (matcher.matches()) { - final int parameterIndex = Integer.parseInt(matcher.group(1)); - String paramType = entry.getValue(); - if (StringUtils.isEmpty(paramType)) { - throw new ProcessException("Value of the " + key + " attribute is null or empty, it must contain a valid value"); - } - - paramType = paramType.trim(); - final String valueAttrName = "cql.args." + parameterIndex + ".value"; - final String parameterValue = attributes.get(valueAttrName); - - try { - setStatementObject(boundStatement, parameterIndex - 1, valueAttrName, parameterValue, paramType); - } catch (final InvalidTypeException | IllegalArgumentException e) { - throw new ProcessException("The value of the " + valueAttrName + " is '" + parameterValue - + "', which cannot be converted into the necessary data type: " + paramType, e); - } - } - } - - try { - ResultSetFuture future = connectionSession.executeAsync(boundStatement); - if (statementTimeout > 0) { - future.getUninterruptibly(statementTimeout, TimeUnit.MILLISECONDS); - } else { - future.getUninterruptibly(); - } - // Emit a Provenance SEND event - final long transmissionMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); - - // This isn't a real URI but since Cassandra is distributed we just use the cluster name - String transitUri = "cassandra://" + connectionSession.getCluster().getMetadata().getClusterName(); - session.getProvenanceReporter().send(flowFile, transitUri, transmissionMillis, true); - session.transfer(flowFile, REL_SUCCESS); - - } catch (final TimeoutException e) { - throw new ProcessException(e); - } - - - } catch (final NoHostAvailableException nhae) { - getLogger().error("No host in the Cassandra cluster can be contacted successfully to execute this statement", nhae); - // Log up to 10 error messages. Otherwise if a 1000-node cluster was specified but there was no connectivity, - // a thousand error messages would be logged. However we would like information from Cassandra itself, so - // cap the error limit at 10, format the messages, and don't include the stack trace (it is displayed by the - // logger message above). - getLogger().error(nhae.getCustomMessage(10, true, false)); - flowFile = session.penalize(flowFile); - session.transfer(flowFile, REL_RETRY); - - } catch (final QueryExecutionException qee) { - logger.error("Cannot execute the statement with the requested consistency level successfully", qee); - flowFile = session.penalize(flowFile); - session.transfer(flowFile, REL_RETRY); - - } catch (final QueryValidationException qve) { - logger.error("The CQL statement {} is invalid due to syntax error, authorization issue, or another " - + "validation problem; routing {} to failure", cql, flowFile, qve); - flowFile = session.penalize(flowFile); - session.transfer(flowFile, REL_FAILURE); - - } catch (final ProcessException e) { - logger.error("Unable to execute CQL select statement {} for {} due to {}; routing to failure", - new Object[]{cql, flowFile, e}); - flowFile = session.penalize(flowFile); - session.transfer(flowFile, REL_FAILURE); - } - } - - /** - * Determines the CQL statement that should be executed for the given FlowFile - * - * @param session the session that can be used to access the given FlowFile - * @param flowFile the FlowFile whose CQL statement should be executed - * @return the CQL that is associated with the given FlowFile - */ - - private String getCQL(final ProcessSession session, final FlowFile flowFile, final Charset charset) { - // Read the CQL from the FlowFile's content - final byte[] buffer = new byte[(int) flowFile.getSize()]; - session.read(flowFile, new InputStreamCallback() { - @Override - public void process(final InputStream in) throws IOException { - StreamUtils.fillBuffer(in, buffer); - } - }); - - // Create the PreparedStatement string to use for this FlowFile. - return new String(buffer, charset); - } - - /** - * Determines how to map the given value to the appropriate Cassandra data type and returns the object as - * represented by the given type. This can be used in a Prepared/BoundStatement. - * - * @param statement the BoundStatement for setting objects on - * @param paramIndex the index of the parameter at which to set the object - * @param attrName the name of the attribute that the parameter is coming from - for logging purposes - * @param paramValue the value of the CQL parameter to set - * @param paramType the Cassandra data type of the CQL parameter to set - * @throws IllegalArgumentException if the PreparedStatement throws a CQLException when calling the appropriate setter - */ - protected void setStatementObject(final BoundStatement statement, final int paramIndex, final String attrName, - final String paramValue, final String paramType) throws IllegalArgumentException { - if (paramValue == null) { - statement.setToNull(paramIndex); - return; - } else if (paramType == null) { - throw new IllegalArgumentException("Parameter type for " + attrName + " cannot be null"); - - } else { - // Parse the top-level type and any parameterized types (for collections) - final Matcher matcher = CQL_TYPE_PATTERN.matcher(paramType); - - // If the matcher doesn't match, this should fall through to the exception at the bottom - if (matcher.find() && matcher.groupCount() > 1) { - String mainTypeString = matcher.group(1).toLowerCase(); - DataType mainType = getPrimitiveDataTypeFromString(mainTypeString); - if (mainType != null) { - TypeCodec typeCodec = codecRegistry.codecFor(mainType); - - // Need the right statement.setXYZ() method - if (mainType.equals(DataType.ascii()) - || mainType.equals(DataType.text()) - || mainType.equals(DataType.varchar()) - || mainType.equals(DataType.inet()) - || mainType.equals(DataType.varint())) { - // These are strings, so just use the paramValue - statement.setString(paramIndex, paramValue); - - } else if (mainType.equals(DataType.cboolean())) { - statement.setBool(paramIndex, (boolean) typeCodec.parse(paramValue)); - - } else if (mainType.equals(DataType.cint())) { - statement.setInt(paramIndex, (int) typeCodec.parse(paramValue)); - - } else if (mainType.equals(DataType.bigint()) - || mainType.equals(DataType.counter())) { - statement.setLong(paramIndex, (long) typeCodec.parse(paramValue)); - - } else if (mainType.equals(DataType.cfloat())) { - statement.setFloat(paramIndex, (float) typeCodec.parse(paramValue)); - - } else if (mainType.equals(DataType.cdouble())) { - statement.setDouble(paramIndex, (double) typeCodec.parse(paramValue)); - - } else if (mainType.equals(DataType.blob())) { - statement.setBytes(paramIndex, (ByteBuffer) typeCodec.parse(paramValue)); - - } else if (mainType.equals(DataType.timestamp())) { - statement.setTimestamp(paramIndex, (Date) typeCodec.parse(paramValue)); - } else if (mainType.equals(DataType.timeuuid()) - || mainType.equals(DataType.uuid())) { - statement.setUUID(paramIndex, (UUID) typeCodec.parse(paramValue)); - } - return; - } else { - // Get the first parameterized type - if (matcher.groupCount() > 2) { - String firstParamTypeName = matcher.group(3); - DataType firstParamType = getPrimitiveDataTypeFromString(firstParamTypeName); - if (firstParamType == null) { - throw new IllegalArgumentException("Nested collections are not supported"); - } - - // Check for map type - if (DataType.Name.MAP.toString().equalsIgnoreCase(mainTypeString)) { - if (matcher.groupCount() > 4) { - String secondParamTypeName = matcher.group(5); - DataType secondParamType = getPrimitiveDataTypeFromString(secondParamTypeName); - DataType mapType = DataType.map(firstParamType, secondParamType); - statement.setMap(paramIndex, (Map) codecRegistry.codecFor(mapType).parse(paramValue)); - return; - } - } else { - // Must be set or list - if (DataType.Name.SET.toString().equalsIgnoreCase(mainTypeString)) { - DataType setType = DataType.set(firstParamType); - statement.setSet(paramIndex, (Set) codecRegistry.codecFor(setType).parse(paramValue)); - return; - } else if (DataType.Name.LIST.toString().equalsIgnoreCase(mainTypeString)) { - DataType listType = DataType.list(firstParamType); - statement.setList(paramIndex, (List) codecRegistry.codecFor(listType).parse(paramValue)); - return; - } - } - } else { - throw new IllegalArgumentException( - "Collection type " + mainTypeString + " needs parameterized type(s), such as set"); - } - - } - } - - } - throw new IllegalArgumentException("Cannot create object of type " + paramType + " using input " + paramValue); - } - - @OnStopped - public void stop(ProcessContext context) { - super.stop(context); - statementCache.clear(); - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraRecord.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraRecord.java deleted file mode 100644 index 7f6cb9b17caa..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraRecord.java +++ /dev/null @@ -1,471 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.BatchStatement; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.Statement; -import com.datastax.driver.core.querybuilder.Assignment; -import com.datastax.driver.core.querybuilder.Insert; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import com.datastax.driver.core.querybuilder.Update; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.ReadsAttribute; -import org.apache.nifi.annotation.behavior.ReadsAttributes; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnShutdown; -import org.apache.nifi.annotation.lifecycle.OnUnscheduled; -import org.apache.nifi.components.AllowableValue; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.serialization.RecordReader; -import org.apache.nifi.serialization.RecordReaderFactory; -import org.apache.nifi.serialization.record.DataType; -import org.apache.nifi.serialization.record.Record; -import org.apache.nifi.serialization.record.RecordFieldType; -import org.apache.nifi.serialization.record.RecordSchema; -import org.apache.nifi.serialization.record.type.ArrayDataType; -import org.apache.nifi.serialization.record.util.DataTypeUtils; -import org.apache.nifi.util.StopWatch; - -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import static java.lang.String.format; - -@Tags({"cassandra", "cql", "put", "insert", "update", "set", "record"}) -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@CapabilityDescription("This is a record aware processor that reads the content of the incoming FlowFile as individual records using the " + - "configured 'Record Reader' and writes them to Apache Cassandra using native protocol version 3 or higher.") -@ReadsAttributes({ - @ReadsAttribute(attribute = "cql.statement.type", description = "If 'Use cql.statement.type Attribute' is selected for the Statement " + - "Type property, the value of the cql.statement.type Attribute will be used to determine which type of statement (UPDATE, INSERT) " + - "will be generated and executed"), - @ReadsAttribute(attribute = "cql.update.method", description = "If 'Use cql.update.method Attribute' is selected for the Update " + - "Method property, the value of the cql.update.method Attribute will be used to determine which operation (Set, Increment, Decrement) " + - "will be used to generate and execute the Update statement. Ignored if the Statement Type property is not set to UPDATE"), - @ReadsAttribute(attribute = "cql.batch.statement.type", description = "If 'Use cql.batch.statement.type Attribute' is selected for the Batch " + - "Statement Type property, the value of the cql.batch.statement.type Attribute will be used to determine which type of batch statement " + - "(LOGGED, UNLOGGED, COUNTER) will be generated and executed") -}) -public class PutCassandraRecord extends AbstractCassandraProcessor { - static final AllowableValue UPDATE_TYPE = new AllowableValue("UPDATE", "UPDATE", - "Use an UPDATE statement."); - static final AllowableValue INSERT_TYPE = new AllowableValue("INSERT", "INSERT", - "Use an INSERT statement."); - static final AllowableValue STATEMENT_TYPE_USE_ATTR_TYPE = new AllowableValue("USE_ATTR", "Use cql.statement.type Attribute", - "The value of the cql.statement.type Attribute will be used to determine which type of statement (UPDATE, INSERT) " + - "will be generated and executed"); - static final String STATEMENT_TYPE_ATTRIBUTE = "cql.statement.type"; - - static final AllowableValue INCR_TYPE = new AllowableValue("INCREMENT", "Increment", - "Use an increment operation (+=) for the Update statement."); - static final AllowableValue SET_TYPE = new AllowableValue("SET", "Set", - "Use a set operation (=) for the Update statement."); - static final AllowableValue DECR_TYPE = new AllowableValue("DECREMENT", "Decrement", - "Use a decrement operation (-=) for the Update statement."); - static final AllowableValue UPDATE_METHOD_USE_ATTR_TYPE = new AllowableValue("USE_ATTR", "Use cql.update.method Attribute", - "The value of the cql.update.method Attribute will be used to determine which operation (Set, Increment, Decrement) " + - "will be used to generate and execute the Update statement."); - static final String UPDATE_METHOD_ATTRIBUTE = "cql.update.method"; - - static final AllowableValue LOGGED_TYPE = new AllowableValue("LOGGED", "LOGGED", - "Use a LOGGED batch statement"); - static final AllowableValue UNLOGGED_TYPE = new AllowableValue("UNLOGGED", "UNLOGGED", - "Use an UNLOGGED batch statement"); - static final AllowableValue COUNTER_TYPE = new AllowableValue("COUNTER", "COUNTER", - "Use a COUNTER batch statement"); - static final AllowableValue BATCH_STATEMENT_TYPE_USE_ATTR_TYPE = new AllowableValue("USE_ATTR", "Use cql.batch.statement.type Attribute", - "The value of the cql.batch.statement.type Attribute will be used to determine which type of batch statement (LOGGED, UNLOGGED or COUNTER) " + - "will be used to generate and execute the Update statement."); - static final String BATCH_STATEMENT_TYPE_ATTRIBUTE = "cql.batch.statement.type"; - - static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder() - .name("put-cassandra-record-reader") - .displayName("Record Reader") - .description("Specifies the type of Record Reader controller service to use for parsing the incoming data " + - "and determining the schema") - .identifiesControllerService(RecordReaderFactory.class) - .required(true) - .build(); - - static final PropertyDescriptor STATEMENT_TYPE = new PropertyDescriptor.Builder() - .name("put-cassandra-record-statement-type") - .displayName("Statement Type") - .description("Specifies the type of CQL Statement to generate.") - .required(true) - .defaultValue(INSERT_TYPE.getValue()) - .allowableValues(UPDATE_TYPE, INSERT_TYPE, STATEMENT_TYPE_USE_ATTR_TYPE) - .build(); - - static final PropertyDescriptor UPDATE_METHOD = new PropertyDescriptor.Builder() - .name("put-cassandra-record-update-method") - .displayName("Update Method") - .description("Specifies the method to use to SET the values. This property is used if the Statement Type is " + - "UPDATE and ignored otherwise.") - .required(false) - .defaultValue(SET_TYPE.getValue()) - .allowableValues(INCR_TYPE, DECR_TYPE, SET_TYPE, UPDATE_METHOD_USE_ATTR_TYPE) - .build(); - - static final PropertyDescriptor UPDATE_KEYS = new PropertyDescriptor.Builder() - .name("put-cassandra-record-update-keys") - .displayName("Update Keys") - .description("A comma-separated list of column names that uniquely identifies a row in the database for UPDATE statements. " - + "If the Statement Type is UPDATE and this property is not set, the conversion to CQL will fail. " - + "This property is ignored if the Statement Type is not UPDATE.") - .addValidator(StandardValidators.createListValidator(true, false, StandardValidators.NON_EMPTY_VALIDATOR)) - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - static final PropertyDescriptor TABLE = new PropertyDescriptor.Builder() - .name("put-cassandra-record-table") - .displayName("Table name") - .description("The name of the Cassandra table to which the records have to be written.") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() - .name("put-cassandra-record-batch-size") - .displayName("Batch size") - .description("Specifies the number of 'Insert statements' to be grouped together to execute as a batch (BatchStatement)") - .defaultValue("100") - .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .required(true) - .build(); - - static final PropertyDescriptor BATCH_STATEMENT_TYPE = new PropertyDescriptor.Builder() - .name("put-cassandra-record-batch-statement-type") - .displayName("Batch Statement Type") - .description("Specifies the type of 'Batch Statement' to be used.") - .allowableValues(LOGGED_TYPE, UNLOGGED_TYPE, COUNTER_TYPE, BATCH_STATEMENT_TYPE_USE_ATTR_TYPE) - .defaultValue(LOGGED_TYPE.getValue()) - .required(false) - .build(); - - static final PropertyDescriptor CONSISTENCY_LEVEL = new PropertyDescriptor.Builder() - .fromPropertyDescriptor(AbstractCassandraProcessor.CONSISTENCY_LEVEL) - .allowableValues(ConsistencyLevel.SERIAL.name(), ConsistencyLevel.LOCAL_SERIAL.name()) - .defaultValue(ConsistencyLevel.SERIAL.name()) - .build(); - - private final static List propertyDescriptors = Collections.unmodifiableList(Arrays.asList( - CONNECTION_PROVIDER_SERVICE, CONTACT_POINTS, KEYSPACE, TABLE, STATEMENT_TYPE, UPDATE_KEYS, UPDATE_METHOD, CLIENT_AUTH, USERNAME, PASSWORD, - RECORD_READER_FACTORY, BATCH_SIZE, CONSISTENCY_LEVEL, BATCH_STATEMENT_TYPE, PROP_SSL_CONTEXT_SERVICE)); - - private final static Set relationships = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE))); - - @Override - protected List getSupportedPropertyDescriptors() { - return propertyDescriptors; - } - - @Override - public Set getRelationships() { - return relationships; - } - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - FlowFile inputFlowFile = session.get(); - - if (inputFlowFile == null) { - return; - } - - final String cassandraTable = context.getProperty(TABLE).evaluateAttributeExpressions(inputFlowFile).getValue(); - final RecordReaderFactory recordParserFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class); - final int batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger(); - final String serialConsistencyLevel = context.getProperty(CONSISTENCY_LEVEL).getValue(); - final String updateKeys = context.getProperty(UPDATE_KEYS).evaluateAttributeExpressions(inputFlowFile).getValue(); - - // Get the statement type from the attribute if necessary - final String statementTypeProperty = context.getProperty(STATEMENT_TYPE).getValue(); - String statementType = statementTypeProperty; - if (STATEMENT_TYPE_USE_ATTR_TYPE.getValue().equals(statementTypeProperty)) { - statementType = inputFlowFile.getAttribute(STATEMENT_TYPE_ATTRIBUTE); - } - - // Get the update method from the attribute if necessary - final String updateMethodProperty = context.getProperty(UPDATE_METHOD).getValue(); - String updateMethod = updateMethodProperty; - if (UPDATE_METHOD_USE_ATTR_TYPE.getValue().equals(updateMethodProperty)) { - updateMethod = inputFlowFile.getAttribute(UPDATE_METHOD_ATTRIBUTE); - } - - - // Get the batch statement type from the attribute if necessary - final String batchStatementTypeProperty = context.getProperty(BATCH_STATEMENT_TYPE).getValue(); - String batchStatementType = batchStatementTypeProperty; - if (BATCH_STATEMENT_TYPE_USE_ATTR_TYPE.getValue().equals(batchStatementTypeProperty)) { - batchStatementType = inputFlowFile.getAttribute(BATCH_STATEMENT_TYPE_ATTRIBUTE).toUpperCase(); - } - if (StringUtils.isEmpty(batchStatementType)) { - throw new IllegalArgumentException(format("Batch Statement Type is not specified, FlowFile %s", inputFlowFile)); - } - - final BatchStatement batchStatement; - final Session connectionSession = cassandraSession.get(); - final AtomicInteger recordsAdded = new AtomicInteger(0); - final StopWatch stopWatch = new StopWatch(true); - - boolean error = false; - - try (final InputStream inputStream = session.read(inputFlowFile); - final RecordReader reader = recordParserFactory.createRecordReader(inputFlowFile, inputStream, getLogger())){ - - // throw an exception if statement type is not set - if (StringUtils.isEmpty(statementType)) { - throw new IllegalArgumentException(format("Statement Type is not specified, FlowFile %s", inputFlowFile)); - } - - // throw an exception if the statement type is set to update and updateKeys is empty - if (UPDATE_TYPE.getValue().equalsIgnoreCase(statementType) && StringUtils.isEmpty(updateKeys)) { - throw new IllegalArgumentException(format("Update Keys are not specified, FlowFile %s", inputFlowFile)); - } - - // throw an exception if the Update Method is Increment or Decrement and the batch statement type is not UNLOGGED or COUNTER - if (INCR_TYPE.getValue().equalsIgnoreCase(updateMethod) || DECR_TYPE.getValue().equalsIgnoreCase(updateMethod)) { - if (!(UNLOGGED_TYPE.getValue().equalsIgnoreCase(batchStatementType) || COUNTER_TYPE.getValue().equalsIgnoreCase(batchStatementType))) { - throw new IllegalArgumentException(format("Increment/Decrement Update Method can only be used with COUNTER " + - "or UNLOGGED Batch Statement Type, FlowFile %s", inputFlowFile)); - } - } - - final RecordSchema schema = reader.getSchema(); - Record record; - - batchStatement = new BatchStatement(BatchStatement.Type.valueOf(batchStatementType)); - batchStatement.setSerialConsistencyLevel(ConsistencyLevel.valueOf(serialConsistencyLevel)); - - while((record = reader.nextRecord()) != null) { - Map recordContentMap = (Map) DataTypeUtils - .convertRecordFieldtoObject(record, RecordFieldType.RECORD.getRecordDataType(record.getSchema())); - - Statement query; - if (INSERT_TYPE.getValue().equalsIgnoreCase(statementType)) { - query = generateInsert(cassandraTable, schema, recordContentMap); - } else if (UPDATE_TYPE.getValue().equalsIgnoreCase(statementType)) { - query = generateUpdate(cassandraTable, schema, updateKeys, updateMethod, recordContentMap); - } else { - throw new IllegalArgumentException(format("Statement Type %s is not valid, FlowFile %s", statementType, inputFlowFile)); - } - - if (getLogger().isDebugEnabled()) { - getLogger().debug("Query: {}", query.toString()); - } - - batchStatement.add(query); - - if (recordsAdded.incrementAndGet() == batchSize) { - connectionSession.execute(batchStatement); - batchStatement.clear(); - recordsAdded.set(0); - } - } - - if (batchStatement.size() != 0) { - connectionSession.execute(batchStatement); - batchStatement.clear(); - } - - } catch (Exception e) { - error = true; - getLogger().error("Unable to write the records into Cassandra table due to {}", new Object[] {e}); - session.transfer(inputFlowFile, REL_FAILURE); - } finally { - if (!error) { - stopWatch.stop(); - long duration = stopWatch.getDuration(TimeUnit.MILLISECONDS); - String transitUri = "cassandra://" + connectionSession.getCluster().getMetadata().getClusterName() + "." + cassandraTable; - - session.getProvenanceReporter().send(inputFlowFile, transitUri, "Inserted " + recordsAdded.get() + " records", duration); - session.transfer(inputFlowFile, REL_SUCCESS); - } - } - - } - - protected Statement generateUpdate(String cassandraTable, RecordSchema schema, String updateKeys, String updateMethod, Map recordContentMap) { - Update updateQuery; - - // Split up the update key names separated by a comma, should not be empty - final Set updateKeyNames; - updateKeyNames = Arrays.stream(updateKeys.split(",")) - .map(String::trim) - .filter(StringUtils::isNotEmpty) - .collect(Collectors.toSet()); - if (updateKeyNames.isEmpty()) { - throw new IllegalArgumentException("No Update Keys were specified"); - } - - // Verify if all update keys are present in the record - for (String updateKey : updateKeyNames) { - if (!schema.getFieldNames().contains(updateKey)) { - throw new IllegalArgumentException("Update key '" + updateKey + "' is not present in the record schema"); - } - } - - // Prepare keyspace/table names - if (cassandraTable.contains(".")) { - String[] keyspaceAndTable = cassandraTable.split("\\."); - updateQuery = QueryBuilder.update(keyspaceAndTable[0], keyspaceAndTable[1]); - } else { - updateQuery = QueryBuilder.update(cassandraTable); - } - - // Loop through the field names, setting those that are not in the update key set, and using those - // in the update key set as conditions. - for (String fieldName : schema.getFieldNames()) { - Object fieldValue = recordContentMap.get(fieldName); - - if (updateKeyNames.contains(fieldName)) { - updateQuery.where(QueryBuilder.eq(fieldName, fieldValue)); - } else { - Assignment assignment; - if (SET_TYPE.getValue().equalsIgnoreCase(updateMethod)) { - assignment = QueryBuilder.set(fieldName, fieldValue); - } else if (INCR_TYPE.getValue().equalsIgnoreCase(updateMethod)) { - assignment = QueryBuilder.incr(fieldName, convertFieldObjectToLong(fieldName, fieldValue)); - } else if (DECR_TYPE.getValue().equalsIgnoreCase(updateMethod)) { - assignment = QueryBuilder.decr(fieldName, convertFieldObjectToLong(fieldName, fieldValue)); - } else { - throw new IllegalArgumentException("Update Method '" + updateMethod + "' is not valid."); - } - updateQuery.with(assignment); - } - } - return updateQuery; - } - - private Long convertFieldObjectToLong(String name, Object value) { - if (!(value instanceof Number)) { - throw new IllegalArgumentException("Field '" + name + "' is not of type Number"); - } - return ((Number) value).longValue(); - } - - @Override - protected Collection customValidate(ValidationContext validationContext) { - Set results = (Set) super.customValidate(validationContext); - - String statementType = validationContext.getProperty(STATEMENT_TYPE).getValue(); - - if (UPDATE_TYPE.getValue().equalsIgnoreCase(statementType)) { - // Check that update keys are set - String updateKeys = validationContext.getProperty(UPDATE_KEYS).getValue(); - if (StringUtils.isEmpty(updateKeys)) { - results.add(new ValidationResult.Builder().subject("Update statement configuration").valid(false).explanation( - "if the Statement Type is set to Update, then the Update Keys must be specified as well").build()); - } - - // Check that if the update method is set to increment or decrement that the batch statement type is set to - // unlogged or counter (or USE_ATTR_TYPE, which we cannot check at this point). - String updateMethod = validationContext.getProperty(UPDATE_METHOD).getValue(); - String batchStatementType = validationContext.getProperty(BATCH_STATEMENT_TYPE).getValue(); - if (INCR_TYPE.getValue().equalsIgnoreCase(updateMethod) - || DECR_TYPE.getValue().equalsIgnoreCase(updateMethod)) { - if (!(COUNTER_TYPE.getValue().equalsIgnoreCase(batchStatementType) - || UNLOGGED_TYPE.getValue().equalsIgnoreCase(batchStatementType) - || BATCH_STATEMENT_TYPE_USE_ATTR_TYPE.getValue().equalsIgnoreCase(batchStatementType))) { - results.add(new ValidationResult.Builder().subject("Update method configuration").valid(false).explanation( - "if the Update Method is set to Increment or Decrement, then the Batch Statement Type must be set " + - "to either COUNTER or UNLOGGED").build()); - } - } - } - - return results; - } - - protected Statement generateInsert(String cassandraTable, RecordSchema schema, Map recordContentMap) { - Insert insertQuery; - if (cassandraTable.contains(".")) { - String[] keyspaceAndTable = cassandraTable.split("\\."); - insertQuery = QueryBuilder.insertInto(keyspaceAndTable[0], keyspaceAndTable[1]); - } else { - insertQuery = QueryBuilder.insertInto(cassandraTable); - } - for (String fieldName : schema.getFieldNames()) { - Object value = recordContentMap.get(fieldName); - - if (value != null && value.getClass().isArray()) { - Object[] array = (Object[]) value; - - if (array.length > 0) { - if (array[0] instanceof Byte) { - Object[] temp = (Object[]) value; - byte[] newArray = new byte[temp.length]; - for (int x = 0; x < temp.length; x++) { - newArray[x] = (Byte) temp[x]; - } - value = ByteBuffer.wrap(newArray); - } - } - } - - if (schema.getDataType(fieldName).isPresent()) { - DataType fieldDataType = schema.getDataType(fieldName).get(); - if (fieldDataType.getFieldType() == RecordFieldType.ARRAY) { - if (((ArrayDataType)fieldDataType).getElementType().getFieldType() == RecordFieldType.STRING) { - value = Arrays.stream((Object[])value).toArray(String[]::new); - } - } - } - - insertQuery.value(fieldName, value); - } - return insertQuery; - } - - @OnUnscheduled - public void stop(ProcessContext context) { - super.stop(context); - } - - @OnShutdown - public void shutdown(ProcessContext context) { - super.stop(context); - } - -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/QueryCassandra.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/QueryCassandra.java deleted file mode 100644 index 67bd8e1d0b22..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/QueryCassandra.java +++ /dev/null @@ -1,736 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.ColumnDefinitions; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.exceptions.NoHostAvailableException; -import com.datastax.driver.core.exceptions.QueryExecutionException; -import com.datastax.driver.core.exceptions.QueryValidationException; -import com.google.common.annotations.VisibleForTesting; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.file.DataFileWriter; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.DatumWriter; -import org.apache.commons.text.StringEscapeUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.lifecycle.OnShutdown; -import org.apache.nifi.annotation.lifecycle.OnUnscheduled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.flowfile.attributes.CoreAttributes; -import org.apache.nifi.annotation.behavior.WritesAttribute; -import org.apache.nifi.annotation.behavior.WritesAttributes; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.flowfile.attributes.FragmentAttributes; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.util.StopWatch; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.Charset; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; - -@Tags({"cassandra", "cql", "select"}) -@InputRequirement(InputRequirement.Requirement.INPUT_ALLOWED) -@CapabilityDescription("Execute provided Cassandra Query Language (CQL) select query on a Cassandra 1.x, 2.x, or 3.0.x cluster. Query result " - + "may be converted to Avro or JSON format. Streaming is used so arbitrarily large result sets are supported. This processor can be " - + "scheduled to run on a timer, or cron expression, using the standard scheduling methods, or it can be triggered by an incoming FlowFile. " - + "If it is triggered by an incoming FlowFile, then attributes of that FlowFile will be available when evaluating the " - + "select query. FlowFile attribute 'executecql.row.count' indicates how many rows were selected.") -@WritesAttributes({ - @WritesAttribute(attribute = "executecql.row.count", description = "The number of rows returned by the CQL query"), - @WritesAttribute(attribute = "fragment.identifier", description = "If 'Max Rows Per Flow File' is set then all FlowFiles from the same query result set " - + "will have the same value for the fragment.identifier attribute. This can then be used to correlate the results."), - @WritesAttribute(attribute = "fragment.count", description = "If 'Max Rows Per Flow File' is set then this is the total number of " - + "FlowFiles produced by a single ResultSet. This can be used in conjunction with the " - + "fragment.identifier attribute in order to know how many FlowFiles belonged to the same incoming ResultSet. If Output Batch Size is set, then this " - + "attribute will not be populated."), - @WritesAttribute(attribute = "fragment.index", description = "If 'Max Rows Per Flow File' is set then the position of this FlowFile in the list of " - + "outgoing FlowFiles that were all derived from the same result set FlowFile. This can be " - + "used in conjunction with the fragment.identifier attribute to know which FlowFiles originated from the same query result set and in what order " - + "FlowFiles were produced") -}) -public class QueryCassandra extends AbstractCassandraProcessor { - - public static final String AVRO_FORMAT = "Avro"; - public static final String JSON_FORMAT = "JSON"; - - public static final String RESULT_ROW_COUNT = "executecql.row.count"; - - public static final String FRAGMENT_ID = FragmentAttributes.FRAGMENT_ID.key(); - public static final String FRAGMENT_INDEX = FragmentAttributes.FRAGMENT_INDEX.key(); - public static final String FRAGMENT_COUNT = FragmentAttributes.FRAGMENT_COUNT.key(); - - public static final PropertyDescriptor CQL_SELECT_QUERY = new PropertyDescriptor.Builder() - .name("CQL select query") - .description("CQL select query") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor QUERY_TIMEOUT = new PropertyDescriptor.Builder() - .name("Max Wait Time") - .description("The maximum amount of time allowed for a running CQL select query. Must be of format " - + " where is a non-negative integer and TimeUnit is a supported " - + "Time Unit, such as: nanos, millis, secs, mins, hrs, days. A value of zero means there is no limit. ") - .defaultValue("0 seconds") - .required(true) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) - .build(); - - public static final PropertyDescriptor FETCH_SIZE = new PropertyDescriptor.Builder() - .name("Fetch size") - .description("The number of result rows to be fetched from the result set at a time. Zero is the default " - + "and means there is no limit.") - .defaultValue("0") - .required(true) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.INTEGER_VALIDATOR) - .build(); - - public static final PropertyDescriptor MAX_ROWS_PER_FLOW_FILE = new PropertyDescriptor.Builder() - .name("Max Rows Per Flow File") - .description("The maximum number of result rows that will be included in a single FlowFile. This will allow you to break up very large " - + "result sets into multiple FlowFiles. If the value specified is zero, then all rows are returned in a single FlowFile.") - .defaultValue("0") - .required(true) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.INTEGER_VALIDATOR) - .build(); - - public static final PropertyDescriptor OUTPUT_BATCH_SIZE = new PropertyDescriptor.Builder() - .name("qdbt-output-batch-size") - .displayName("Output Batch Size") - .description("The number of output FlowFiles to queue before committing the process session. When set to zero, the session will be committed when all result set rows " - + "have been processed and the output FlowFiles are ready for transfer to the downstream relationship. For large result sets, this can cause a large burst of FlowFiles " - + "to be transferred at the end of processor execution. If this property is set, then when the specified number of FlowFiles are ready for transfer, then the session will " - + "be committed, thus releasing the FlowFiles to the downstream relationship. NOTE: The maxvalue.* and fragment.count attributes will not be set on FlowFiles when this " - + "property is set.") - .defaultValue("0") - .required(true) - .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .build(); - - public static final PropertyDescriptor OUTPUT_FORMAT = new PropertyDescriptor.Builder() - .name("Output Format") - .description("The format to which the result rows will be converted. If JSON is selected, the output will " - + "contain an object with field 'results' containing an array of result rows. Each row in the array is a " - + "map of the named column to its value. For example: { \"results\": [{\"userid\":1, \"name\":\"Joe Smith\"}]}") - .required(true) - .allowableValues(AVRO_FORMAT, JSON_FORMAT) - .defaultValue(AVRO_FORMAT) - .build(); - - public static final PropertyDescriptor TIMESTAMP_FORMAT_PATTERN = new PropertyDescriptor.Builder() - .name("timestamp-format-pattern") - .displayName("Timestamp Format Pattern for JSON output") - .description("Pattern to use when converting timestamp fields to JSON. Note: the formatted timestamp will be in UTC timezone.") - .required(true) - .defaultValue("yyyy-MM-dd HH:mm:ssZ") - .addValidator((subject, input, context) -> { - final ValidationResult.Builder vrb = new ValidationResult.Builder().subject(subject).input(input); - try { - DateTimeFormatter.ofPattern(input); - vrb.valid(true).explanation("Valid date format pattern"); - } catch (Exception ex) { - vrb.valid(false).explanation("the pattern is invalid: " + ex.getMessage()); - } - return vrb.build(); - }) - .build(); - - private final static List propertyDescriptors; - - private final static Set relationships; - - /* - * Will ensure that the list of property descriptors is build only once. - * Will also create a Set of relationships - */ - static { - List _propertyDescriptors = new ArrayList<>(); - _propertyDescriptors.addAll(descriptors); - _propertyDescriptors.add(CQL_SELECT_QUERY); - _propertyDescriptors.add(QUERY_TIMEOUT); - _propertyDescriptors.add(FETCH_SIZE); - _propertyDescriptors.add(MAX_ROWS_PER_FLOW_FILE); - _propertyDescriptors.add(OUTPUT_BATCH_SIZE); - _propertyDescriptors.add(OUTPUT_FORMAT); - _propertyDescriptors.add(TIMESTAMP_FORMAT_PATTERN); - propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors); - - Set _relationships = new HashSet<>(); - _relationships.add(REL_SUCCESS); - _relationships.add(REL_FAILURE); - _relationships.add(REL_RETRY); - relationships = Collections.unmodifiableSet(_relationships); - } - - @Override - public Set getRelationships() { - return relationships; - } - - @Override - public final List getSupportedPropertyDescriptors() { - return propertyDescriptors; - } - - @OnScheduled - public void onScheduled(final ProcessContext context) { - super.onScheduled(context); - - final int fetchSize = context.getProperty(FETCH_SIZE).evaluateAttributeExpressions().asInteger(); - if (fetchSize > 0) { - synchronized (cluster.get()) { - cluster.get().getConfiguration().getQueryOptions().setFetchSize(fetchSize); - } - } - - } - - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - FlowFile fileToProcess = null; - - if (context.hasIncomingConnection()) { - fileToProcess = session.get(); - - // If we have no FlowFile, and all incoming connections are self-loops then we can continue on. - // However, if we have no FlowFile and we have connections coming from other Processors, then - // we know that we should run only if we have a FlowFile. - if (fileToProcess == null && context.hasNonLoopConnection()) { - return; - } - } - - final ComponentLog logger = getLogger(); - final String selectQuery = context.getProperty(CQL_SELECT_QUERY).evaluateAttributeExpressions(fileToProcess).getValue(); - final long queryTimeout = context.getProperty(QUERY_TIMEOUT).evaluateAttributeExpressions(fileToProcess).asTimePeriod(TimeUnit.MILLISECONDS); - final String outputFormat = context.getProperty(OUTPUT_FORMAT).getValue(); - final long maxRowsPerFlowFile = context.getProperty(MAX_ROWS_PER_FLOW_FILE).evaluateAttributeExpressions().asInteger(); - final long outputBatchSize = context.getProperty(OUTPUT_BATCH_SIZE).evaluateAttributeExpressions().asInteger(); - final Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(fileToProcess).getValue()); - final StopWatch stopWatch = new StopWatch(true); - - final List resultSetFlowFiles = new LinkedList<>(); - - try { - // The documentation for the driver recommends the session remain open the entire time the processor is running - // and states that it is thread-safe. This is why connectionSession is not in a try-with-resources. - final Session connectionSession = cassandraSession.get(); - final ResultSet resultSet; - - if (queryTimeout > 0) { - resultSet = connectionSession.execute(selectQuery, queryTimeout, TimeUnit.MILLISECONDS); - }else{ - resultSet = connectionSession.execute(selectQuery); - } - final AtomicLong nrOfRows = new AtomicLong(0L); - - if(fileToProcess == null) { - fileToProcess = session.create(); - } - - int fragmentIndex = 0; - final String fragmentId = UUID.randomUUID().toString(); - - while(true) { - - fileToProcess = session.write(fileToProcess, out -> { - try { - logger.debug("Executing CQL query {}", selectQuery); - if (queryTimeout > 0) { - if (AVRO_FORMAT.equals(outputFormat)) { - nrOfRows.set(convertToAvroStream(resultSet, maxRowsPerFlowFile, - out, queryTimeout, TimeUnit.MILLISECONDS)); - } else if (JSON_FORMAT.equals(outputFormat)) { - nrOfRows.set(convertToJsonStream(resultSet, maxRowsPerFlowFile, - out, charset, queryTimeout, TimeUnit.MILLISECONDS)); - } - } else { - if (AVRO_FORMAT.equals(outputFormat)) { - nrOfRows.set(convertToAvroStream(resultSet, maxRowsPerFlowFile, - out, 0, null)); - } else if (JSON_FORMAT.equals(outputFormat)) { - nrOfRows.set(convertToJsonStream(resultSet, maxRowsPerFlowFile, - out, charset, 0, null)); - } - } - } catch (final TimeoutException | InterruptedException | ExecutionException e) { - throw new ProcessException(e); - } - }); - - // set attribute how many rows were selected - fileToProcess = session.putAttribute(fileToProcess, RESULT_ROW_COUNT, String.valueOf(nrOfRows.get())); - - // set mime.type based on output format - fileToProcess = session.putAttribute(fileToProcess, CoreAttributes.MIME_TYPE.key(), - JSON_FORMAT.equals(outputFormat) ? "application/json" : "application/avro-binary"); - - if (logger.isDebugEnabled()) { - logger.info("{} contains {} records; transferring to 'success'", - fileToProcess, nrOfRows.get()); - } - if (maxRowsPerFlowFile > 0) { - fileToProcess = session.putAttribute(fileToProcess, FRAGMENT_ID, fragmentId); - fileToProcess = session.putAttribute(fileToProcess, FRAGMENT_INDEX, String.valueOf(fragmentIndex)); - } - session.getProvenanceReporter().modifyContent(fileToProcess, "Retrieved " + nrOfRows.get() + " rows", - stopWatch.getElapsed(TimeUnit.MILLISECONDS)); - resultSetFlowFiles.add(fileToProcess); - - if (outputBatchSize > 0) { - - if (resultSetFlowFiles.size() == outputBatchSize) { - session.transfer(resultSetFlowFiles, REL_SUCCESS); - session.commitAsync(); - resultSetFlowFiles.clear(); - } - } - fragmentIndex++; - resultSet.fetchMoreResults().get(); - if (resultSet.isExhausted()) { - // If we are splitting results but not outputting batches, set count on all FlowFiles - if (outputBatchSize == 0 && maxRowsPerFlowFile > 0) { - for (int i = 0; i < resultSetFlowFiles.size(); i++) { - resultSetFlowFiles.set(i, - session.putAttribute(resultSetFlowFiles.get(i), FRAGMENT_COUNT, Integer.toString(fragmentIndex))); - } - } - session.transfer(resultSetFlowFiles, REL_SUCCESS); - session.commitAsync(); - resultSetFlowFiles.clear(); - break; - } - fileToProcess = session.create(); - } - - } catch (final NoHostAvailableException nhae) { - getLogger().error("No host in the Cassandra cluster can be contacted successfully to execute this query", nhae); - // Log up to 10 error messages. Otherwise if a 1000-node cluster was specified but there was no connectivity, - // a thousand error messages would be logged. However we would like information from Cassandra itself, so - // cap the error limit at 10, format the messages, and don't include the stack trace (it is displayed by the - // logger message above). - getLogger().error(nhae.getCustomMessage(10, true, false)); - if (fileToProcess == null) { - fileToProcess = session.create(); - } - fileToProcess = session.penalize(fileToProcess); - session.transfer(fileToProcess, REL_RETRY); - } catch (final QueryExecutionException qee) { - logger.error("Cannot execute the query with the requested consistency level successfully", qee); - if (fileToProcess == null) { - fileToProcess = session.create(); - } - fileToProcess = session.penalize(fileToProcess); - session.transfer(fileToProcess, REL_RETRY); - - } catch (final QueryValidationException qve) { - if (context.hasIncomingConnection()) { - logger.error("The CQL query {} is invalid due to syntax error, authorization issue, or another " - + "validation problem; routing {} to failure", - selectQuery, fileToProcess, qve); - - if (fileToProcess == null) { - fileToProcess = session.create(); - } - fileToProcess = session.penalize(fileToProcess); - session.transfer(fileToProcess, REL_FAILURE); - } else { - // This can happen if any exceptions occur while setting up the connection, statement, etc. - logger.error("The CQL query {} is invalid due to syntax error, authorization issue, or another " - + "validation problem", selectQuery, qve); - if (fileToProcess != null) { - session.remove(fileToProcess); - } - context.yield(); - } - } catch (InterruptedException|ExecutionException ex) { - if (context.hasIncomingConnection()) { - logger.error("The CQL query {} has yielded an unknown error, routing {} to failure", - selectQuery, fileToProcess, ex); - - if (fileToProcess == null) { - fileToProcess = session.create(); - } - fileToProcess = session.penalize(fileToProcess); - session.transfer(fileToProcess, REL_FAILURE); - } else { - // This can happen if any exceptions occur while setting up the connection, statement, etc. - logger.error("The CQL query {} has run into an unknown error.", selectQuery, ex); - if (fileToProcess != null) { - session.remove(fileToProcess); - } - context.yield(); - } - } catch (final ProcessException e) { - if (context.hasIncomingConnection()) { - logger.error("Unable to execute CQL select query {} for {} due to {}; routing to failure", - selectQuery, fileToProcess, e); - if (fileToProcess == null) { - fileToProcess = session.create(); - } - fileToProcess = session.penalize(fileToProcess); - session.transfer(fileToProcess, REL_FAILURE); - - } else { - logger.error("Unable to execute CQL select query {} due to {}", - selectQuery, e); - if (fileToProcess != null) { - session.remove(fileToProcess); - } - context.yield(); - } - } - session.commitAsync(); - } - - @OnUnscheduled - public void stop(ProcessContext context) { - super.stop(context); - } - - @OnShutdown - public void shutdown(ProcessContext context) { - super.stop(context); - } - - /** - * Converts a result set into an Avro record and writes it to the given stream. - * - * @param rs The result set to convert - * @param outStream The stream to which the Avro record will be written - * @param timeout The max number of timeUnits to wait for a result set fetch to complete - * @param timeUnit The unit of time (SECONDS, e.g.) associated with the timeout amount - * @return The number of rows from the result set written to the stream - * @throws IOException If the Avro record cannot be written - * @throws InterruptedException If a result set fetch is interrupted - * @throws TimeoutException If a result set fetch has taken longer than the specified timeout - * @throws ExecutionException If any error occurs during the result set fetch - */ - public static long convertToAvroStream(final ResultSet rs, long maxRowsPerFlowFile, - final OutputStream outStream, - long timeout, TimeUnit timeUnit) - throws IOException, InterruptedException, TimeoutException, ExecutionException { - - final Schema schema = createSchema(rs); - final GenericRecord rec = new GenericData.Record(schema); - final DatumWriter datumWriter = new GenericDatumWriter<>(schema); - - try (final DataFileWriter dataFileWriter = new DataFileWriter<>(datumWriter)) { - dataFileWriter.create(schema, outStream); - - ColumnDefinitions columnDefinitions = rs.getColumnDefinitions(); - long nrOfRows = 0; - long rowsAvailableWithoutFetching = rs.getAvailableWithoutFetching(); - - if (columnDefinitions != null) { - - // Grab the ones we have - if (rowsAvailableWithoutFetching == 0 - || rowsAvailableWithoutFetching < maxRowsPerFlowFile) { - // Get more - if (timeout <= 0 || timeUnit == null) { - rs.fetchMoreResults().get(); - } else { - rs.fetchMoreResults().get(timeout, timeUnit); - } - } - - Row row; - while ((maxRowsPerFlowFile == 0) || nrOfRows < maxRowsPerFlowFile) { - try { - row = rs.iterator().next(); - } catch (NoSuchElementException nsee) { - break; - } - - // iterator().next() is like iterator().one() => return null on end - // https://docs.datastax.com/en/drivers/java/2.0/com/datastax/driver/core/ResultSet.html#one-- - if (row == null) { - break; - } - - for (int i = 0; i < columnDefinitions.size(); i++) { - final DataType dataType = columnDefinitions.getType(i); - - if (row.isNull(i)) { - rec.put(i, null); - } else { - rec.put(i, getCassandraObject(row, i, dataType)); - } - } - - dataFileWriter.append(rec); - nrOfRows += 1; - } - } - return nrOfRows; - } - } - - private static String getFormattedDate(final Optional context, Date value) { - final String dateFormatPattern = context - .map(_context -> _context.getProperty(TIMESTAMP_FORMAT_PATTERN).getValue()) - .orElse(TIMESTAMP_FORMAT_PATTERN.getDefaultValue()); - final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(dateFormatPattern); - final OffsetDateTime offsetDateTime = value.toInstant().atOffset(ZoneOffset.UTC); - return dateTimeFormatter.format(offsetDateTime); - } - - public static long convertToJsonStream(final ResultSet rs, long maxRowsPerFlowFile, - final OutputStream outStream, - Charset charset, long timeout, TimeUnit timeUnit) - throws IOException, InterruptedException, TimeoutException, ExecutionException { - return convertToJsonStream(Optional.empty(), rs, maxRowsPerFlowFile, outStream, charset, timeout, timeUnit); - } - - /** - * Converts a result set into an Json object and writes it to the given stream using the specified character set. - * - * @param rs The result set to convert - * @param outStream The stream to which the JSON object will be written - * @param timeout The max number of timeUnits to wait for a result set fetch to complete - * @param timeUnit The unit of time (SECONDS, e.g.) associated with the timeout amount - * @return The number of rows from the result set written to the stream - * @throws IOException If the JSON object cannot be written - * @throws InterruptedException If a result set fetch is interrupted - * @throws TimeoutException If a result set fetch has taken longer than the specified timeout - * @throws ExecutionException If any error occurs during the result set fetch - */ - @VisibleForTesting - public static long convertToJsonStream(final Optional context, - final ResultSet rs, long maxRowsPerFlowFile, - final OutputStream outStream, - Charset charset, long timeout, TimeUnit timeUnit) - throws IOException, InterruptedException, TimeoutException, ExecutionException { - - try { - // Write the initial object brace - outStream.write("{\"results\":[".getBytes(charset)); - ColumnDefinitions columnDefinitions = rs.getColumnDefinitions(); - long nrOfRows = 0; - long rowsAvailableWithoutFetching = rs.getAvailableWithoutFetching(); - - if (columnDefinitions != null) { - - // Grab the ones we have - if (rowsAvailableWithoutFetching == 0) { - // Get more - if (timeout <= 0 || timeUnit == null) { - rs.fetchMoreResults().get(); - } else { - rs.fetchMoreResults().get(timeout, timeUnit); - } - rowsAvailableWithoutFetching = rs.getAvailableWithoutFetching(); - } - - if(maxRowsPerFlowFile == 0){ - maxRowsPerFlowFile = rowsAvailableWithoutFetching; - } - - Row row; - while(nrOfRows < maxRowsPerFlowFile){ - try { - row = rs.iterator().next(); - }catch (NoSuchElementException nsee){ - nrOfRows -= 1; - break; - } - - // iterator().next() is like iterator().one() => return null on end - // https://docs.datastax.com/en/drivers/java/2.0/com/datastax/driver/core/ResultSet.html#one-- - if(row == null){ - break; - } - - if (nrOfRows != 0) { - outStream.write(",".getBytes(charset)); - } - - outStream.write("{".getBytes(charset)); - for (int i = 0; i < columnDefinitions.size(); i++) { - final DataType dataType = columnDefinitions.getType(i); - final String colName = columnDefinitions.getName(i); - if (i != 0) { - outStream.write(",".getBytes(charset)); - } - if (row.isNull(i)) { - outStream.write(("\"" + colName + "\"" + ":null").getBytes(charset)); - } else { - Object value = getCassandraObject(row, i, dataType); - String valueString; - if (value instanceof List || value instanceof Set) { - boolean first = true; - StringBuilder sb = new StringBuilder("["); - for (Object element : ((Collection) value)) { - if (!first) { - sb.append(","); - } - sb.append(getJsonElement(context, element)); - first = false; - } - sb.append("]"); - valueString = sb.toString(); - } else if (value instanceof Map) { - boolean first = true; - StringBuilder sb = new StringBuilder("{"); - for (Object element : ((Map) value).entrySet()) { - Map.Entry entry = (Map.Entry) element; - Object mapKey = entry.getKey(); - Object mapValue = entry.getValue(); - - if (!first) { - sb.append(","); - } - sb.append(getJsonElement(context, mapKey)); - sb.append(":"); - sb.append(getJsonElement(context, mapValue)); - first = false; - } - sb.append("}"); - valueString = sb.toString(); - } else { - valueString = getJsonElement(context, value); - } - outStream.write(("\"" + colName + "\":" - + valueString + "").getBytes(charset)); - } - } - nrOfRows += 1; - outStream.write("}".getBytes(charset)); - } - } - return nrOfRows; - } finally { - outStream.write("]}".getBytes()); - } - } - - protected static String getJsonElement(Object value) { - return getJsonElement(Optional.empty(), value); - } - - protected static String getJsonElement(final Optional context, Object value) { - if (value instanceof Number) { - return value.toString(); - } else if (value instanceof Date) { - return "\"" + getFormattedDate(context, (Date) value) + "\""; - } else if (value instanceof String) { - return "\"" + StringEscapeUtils.escapeJson((String) value) + "\""; - } else { - return "\"" + value.toString() + "\""; - } - } - - /** - * Creates an Avro schema from the given result set. The metadata (column definitions, data types, etc.) is used - * to determine a schema for Avro. - * - * @param rs The result set from which an Avro schema will be created - * @return An Avro schema corresponding to the given result set's metadata - */ - public static Schema createSchema(final ResultSet rs) { - final ColumnDefinitions columnDefinitions = rs.getColumnDefinitions(); - final int nrOfColumns = (columnDefinitions == null ? 0 : columnDefinitions.size()); - String tableName = "NiFi_Cassandra_Query_Record"; - if (nrOfColumns > 0) { - String tableNameFromMeta = columnDefinitions.getTable(0); - if (!StringUtils.isBlank(tableNameFromMeta)) { - tableName = tableNameFromMeta; - } - } - - final SchemaBuilder.FieldAssembler builder = SchemaBuilder.record(tableName).namespace("any.data").fields(); - if (columnDefinitions != null) { - for (int i = 0; i < nrOfColumns; i++) { - - DataType dataType = columnDefinitions.getType(i); - if (dataType == null) { - throw new IllegalArgumentException("No data type for column[" + i + "] with name " + columnDefinitions.getName(i)); - } - - // Map types from Cassandra to Avro where possible - if (dataType.isCollection()) { - List typeArguments = dataType.getTypeArguments(); - if (typeArguments == null || typeArguments.size() == 0) { - throw new IllegalArgumentException("Column[" + i + "] " + dataType.getName() - + " is a collection but no type arguments were specified!"); - } - // Get the first type argument, to be used for lists and sets - DataType firstArg = typeArguments.get(0); - if (dataType.equals(DataType.set(firstArg)) - || dataType.equals(DataType.list(firstArg))) { - builder.name(columnDefinitions.getName(i)).type().unionOf().nullBuilder().endNull().and().array() - .items(getUnionFieldType(getPrimitiveAvroTypeFromCassandraType(firstArg))).endUnion().noDefault(); - } else { - // Must be an n-arg collection like map - DataType secondArg = typeArguments.get(1); - if (dataType.equals(DataType.map(firstArg, secondArg))) { - builder.name(columnDefinitions.getName(i)).type().unionOf().nullBuilder().endNull().and().map().values( - getUnionFieldType(getPrimitiveAvroTypeFromCassandraType(secondArg))).endUnion().noDefault(); - } - } - } else { - builder.name(columnDefinitions.getName(i)) - .type(getUnionFieldType(getPrimitiveAvroTypeFromCassandraType(dataType))).noDefault(); - } - } - } - return builder.endRecord(); - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor deleted file mode 100644 index 88aa0d889d5f..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.processors.cassandra.QueryCassandra -org.apache.nifi.processors.cassandra.PutCassandraQL -org.apache.nifi.processors.cassandra.PutCassandraRecord \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java deleted file mode 100644 index ce21961316c1..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/AbstractCassandraProcessorTest.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.Configuration; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.Metadata; -import com.datastax.driver.core.Row; -import com.google.common.collect.Sets; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.service.CassandraSessionProvider; -import org.apache.nifi.ssl.SSLContextService; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.net.ssl.SSLContext; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Unit tests for the AbstractCassandraProcessor class - */ -public class AbstractCassandraProcessorTest { - - MockAbstractCassandraProcessor processor; - private TestRunner testRunner; - - @BeforeEach - public void setUp() throws Exception { - processor = new MockAbstractCassandraProcessor(); - testRunner = TestRunners.newTestRunner(processor); - } - - @Test - public void testCustomValidate() throws Exception { - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, ""); - testRunner.assertNotValid(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost"); - testRunner.assertNotValid(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.assertValid(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042, node2: 4399"); - testRunner.assertValid(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, " localhost : 9042, node2: 4399"); - testRunner.assertValid(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042, node2"); - testRunner.assertNotValid(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:65536"); - testRunner.assertNotValid(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "user"); - testRunner.assertNotValid(); // Needs a password set if user is set - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.assertValid(); - } - - @Test - public void testCustomValidateEL() throws Exception { - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "${host}"); - testRunner.setProperty(AbstractCassandraProcessor.KEYSPACE, "${keyspace}"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "${user}"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "${password}"); - testRunner.setProperty(AbstractCassandraProcessor.CHARSET, "${charset}"); - testRunner.assertValid(); - } - - @SuppressWarnings("unchecked") - @Test - public void testGetCassandraObject() throws Exception { - Row row = CassandraQueryTestUtil.createRow("user1", "Joe", "Smith", - Sets.newHashSet("jsmith@notareal.com", "joes@fakedomain.com"), Arrays.asList("New York, NY", "Santa Clara, CA"), - new HashMap() {{ - put(Calendar.getInstance().getTime(), "Set my alarm for a month from now"); - }}, true, 1.0f, 2.0); - - assertEquals("user1", AbstractCassandraProcessor.getCassandraObject(row, 0, DataType.text())); - assertEquals("Joe", AbstractCassandraProcessor.getCassandraObject(row, 1, DataType.text())); - assertEquals("Smith", AbstractCassandraProcessor.getCassandraObject(row, 2, DataType.text())); - Set emails = (Set) AbstractCassandraProcessor.getCassandraObject(row, 3, DataType.set(DataType.text())); - assertNotNull(emails); - assertEquals(2, emails.size()); - List topPlaces = (List) AbstractCassandraProcessor.getCassandraObject(row, 4, DataType.list(DataType.text())); - assertNotNull(topPlaces); - Map todoMap = (Map) AbstractCassandraProcessor.getCassandraObject( - row, 5, DataType.map(DataType.timestamp(), DataType.text())); - assertNotNull(todoMap); - assertEquals(1, todoMap.values().size()); - Boolean registered = (Boolean) AbstractCassandraProcessor.getCassandraObject(row, 6, DataType.cboolean()); - assertNotNull(registered); - assertTrue(registered); - } - - @Test - public void testGetSchemaForType() throws Exception { - assertEquals(AbstractCassandraProcessor.getSchemaForType("string").getType().getName(), "string"); - assertEquals(AbstractCassandraProcessor.getSchemaForType("boolean").getType().getName(), "boolean"); - assertEquals(AbstractCassandraProcessor.getSchemaForType("int").getType().getName(), "int"); - assertEquals(AbstractCassandraProcessor.getSchemaForType("long").getType().getName(), "long"); - assertEquals(AbstractCassandraProcessor.getSchemaForType("float").getType().getName(), "float"); - assertEquals(AbstractCassandraProcessor.getSchemaForType("double").getType().getName(), "double"); - assertEquals(AbstractCassandraProcessor.getSchemaForType("bytes").getType().getName(), "bytes"); - } - - @Test - public void testGetSchemaForTypeBadType() throws Exception { - assertThrows(IllegalArgumentException.class, () -> AbstractCassandraProcessor.getSchemaForType("nothing")); - } - - @Test - public void testGetPrimitiveAvroTypeFromCassandraType() throws Exception { - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.ascii())); - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.text())); - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.varchar())); - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.timestamp())); - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.timeuuid())); - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.uuid())); - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.inet())); - assertEquals("string", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.varint())); - - assertEquals("boolean", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.cboolean())); - assertEquals("int", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.cint())); - - assertEquals("long", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.bigint())); - assertEquals("long", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.counter())); - - assertEquals("float", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.cfloat())); - assertEquals("double", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.cdouble())); - - assertEquals("bytes", AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(DataType.blob())); - } - - @Test - public void testGetPrimitiveAvroTypeFromCassandraTypeBadType() throws Exception { - DataType mockDataType = mock(DataType.class); - assertThrows(IllegalArgumentException.class, () -> AbstractCassandraProcessor.getPrimitiveAvroTypeFromCassandraType(mockDataType)); - } - - @Test - public void testGetPrimitiveDataTypeFromString() { - assertEquals(DataType.ascii(), AbstractCassandraProcessor.getPrimitiveDataTypeFromString("ascii")); - } - - @Test - public void testGetContactPoints() throws Exception { - List contactPoints = processor.getContactPoints(""); - assertNotNull(contactPoints); - assertEquals(1, contactPoints.size()); - assertEquals("localhost", contactPoints.get(0).getHostName()); - assertEquals(AbstractCassandraProcessor.DEFAULT_CASSANDRA_PORT, contactPoints.get(0).getPort()); - - contactPoints = processor.getContactPoints("192.168.99.100:9042"); - assertNotNull(contactPoints); - assertEquals(1, contactPoints.size()); - assertEquals("192.168.99.100", contactPoints.get(0).getAddress().getHostAddress()); - assertEquals(9042, contactPoints.get(0).getPort()); - - contactPoints = processor.getContactPoints("192.168.99.100:9042, mydomain.com : 4000"); - assertNotNull(contactPoints); - assertEquals(2, contactPoints.size()); - assertEquals("192.168.99.100", contactPoints.get(0).getAddress().getHostAddress()); - assertEquals(9042, contactPoints.get(0).getPort()); - assertEquals("mydomain.com", contactPoints.get(1).getHostName()); - assertEquals(4000, contactPoints.get(1).getPort()); - } - - @Test - public void testConnectToCassandra() throws Exception { - // Follow the non-null path - Cluster cluster = mock(Cluster.class); - processor.setCluster(cluster); - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - processor.connectToCassandra(testRunner.getProcessContext()); - processor.stop(testRunner.getProcessContext()); - assertNull(processor.getCluster()); - - // Now do a connect where a cluster is "built" - processor.connectToCassandra(testRunner.getProcessContext()); - assertEquals("cluster1", processor.getCluster().getMetadata().getClusterName()); - } - - @Test - public void testConnectToCassandraWithSSL() throws Exception { - SSLContextService sslService = mock(SSLContextService.class); - when(sslService.getIdentifier()).thenReturn("ssl-context"); - testRunner.addControllerService("ssl-context", sslService); - testRunner.enableControllerService(sslService); - testRunner.setProperty(AbstractCassandraProcessor.PROP_SSL_CONTEXT_SERVICE, "ssl-context"); - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.assertValid(sslService); - processor.connectToCassandra(testRunner.getProcessContext()); - assertNotNull(processor.getCluster()); - processor.setCluster(null); - // Try with a ClientAuth value - testRunner.setProperty(AbstractCassandraProcessor.CLIENT_AUTH, "WANT"); - processor.connectToCassandra(testRunner.getProcessContext()); - assertNotNull(processor.getCluster()); - } - - @Test - public void testConnectToCassandraUsernamePassword() throws Exception { - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "user"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - // Now do a connect where a cluster is "built" - processor.connectToCassandra(testRunner.getProcessContext()); - assertNotNull(processor.getCluster()); - } - - @Test - public void testCustomValidateCassandraConnectionConfiguration() throws InitializationException { - MockCassandraSessionProvider sessionProviderService = new MockCassandraSessionProvider(); - - testRunner.addControllerService("cassandra-connection-provider", sessionProviderService); - testRunner.setProperty(sessionProviderService, CassandraSessionProvider.CONTACT_POINTS, "localhost:9042"); - testRunner.setProperty(sessionProviderService, CassandraSessionProvider.KEYSPACE, "somekyespace"); - - testRunner.setProperty(AbstractCassandraProcessor.CONNECTION_PROVIDER_SERVICE, "cassandra-connection-provider"); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.setProperty(AbstractCassandraProcessor.KEYSPACE, "some-keyspace"); - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "user"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.enableControllerService(sessionProviderService); - - testRunner.assertNotValid(); - - testRunner.removeProperty(AbstractCassandraProcessor.CONTACT_POINTS); - testRunner.removeProperty(AbstractCassandraProcessor.KEYSPACE); - testRunner.removeProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL); - testRunner.removeProperty(AbstractCassandraProcessor.USERNAME); - testRunner.removeProperty(AbstractCassandraProcessor.PASSWORD); - - testRunner.assertValid(); - } - - /** - * Provides a stubbed processor instance for testing - */ - public static class MockAbstractCassandraProcessor extends AbstractCassandraProcessor { - - @Override - protected List getSupportedPropertyDescriptors() { - return Arrays.asList(CONNECTION_PROVIDER_SERVICE, CONTACT_POINTS, KEYSPACE, USERNAME, PASSWORD, CONSISTENCY_LEVEL, CHARSET); - } - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - - } - - @Override - protected Cluster createCluster(List contactPoints, SSLContext sslContext, - String username, String password, String compressionType) { - Cluster mockCluster = mock(Cluster.class); - Metadata mockMetadata = mock(Metadata.class); - when(mockMetadata.getClusterName()).thenReturn("cluster1"); - when(mockCluster.getMetadata()).thenReturn(mockMetadata); - Configuration config = Configuration.builder().build(); - when(mockCluster.getConfiguration()).thenReturn(config); - return mockCluster; - } - - public Cluster getCluster() { - return cluster.get(); - } - - public void setCluster(Cluster newCluster) { - this.cluster.set(newCluster); - } - } - - /** - * Mock CassandraSessionProvider implementation for testing purpose - */ - private class MockCassandraSessionProvider extends CassandraSessionProvider { - - @OnEnabled - public void onEnabled(final ConfigurationContext context) { - - } - - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/CassandraQueryTestUtil.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/CassandraQueryTestUtil.java deleted file mode 100644 index 6bdb400159b2..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/CassandraQueryTestUtil.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.ColumnDefinitions; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.google.common.collect.Sets; -import com.google.common.reflect.TypeToken; -import com.google.common.util.concurrent.ListenableFuture; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Set; -import java.util.Map; -import java.util.List; -import java.util.HashMap; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.TimeZone; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Utility methods for Cassandra processors' unit tests - */ -public class CassandraQueryTestUtil { - - static final Date TEST_DATE; - static { - Calendar c = GregorianCalendar.getInstance(TimeZone.getTimeZone("PST")); - c.set(2020, Calendar.JANUARY, 1, 10, 10, 10); - c.set(Calendar.MILLISECOND, 10); - TEST_DATE = c.getTime(); - } - - public static ResultSet createMockResultSet(boolean falseThenTrue) throws Exception { - ResultSet resultSet = mock(ResultSet.class); - ColumnDefinitions columnDefinitions = mock(ColumnDefinitions.class); - when(columnDefinitions.size()).thenReturn(9); - when(columnDefinitions.getName(anyInt())).thenAnswer(new Answer() { - - List colNames = Arrays.asList( - "user_id", "first_name", "last_name", "emails", "top_places", "todo", "registered", "scale", "metric"); - - @Override - public String answer(InvocationOnMock invocationOnMock) { - return colNames.get((Integer) invocationOnMock.getArguments()[0]); - - } - }); - - when(columnDefinitions.getTable(0)).thenReturn("users"); - - when(columnDefinitions.getType(anyInt())).thenAnswer(new Answer() { - - List dataTypes = Arrays.asList( - DataType.text(), DataType.text(), DataType.text(), DataType.set(DataType.text()), - DataType.list(DataType.text()), DataType.map(DataType.timestamp(), DataType.text()), DataType.cboolean(), - DataType.cfloat(), DataType.cdouble() - ); - - @Override - public DataType answer(InvocationOnMock invocationOnMock) throws Throwable { - return dataTypes.get((Integer) invocationOnMock.getArguments()[0]); - - } - }); - - final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ssZ"); - final Date aMonthPrior = Date.from(OffsetDateTime.parse("2016-01-03 05:00:00+0000", dateTimeFormatter).toInstant()); - final Date testDate = Date.from(OffsetDateTime.parse("2016-02-03 05:00:00+0000", dateTimeFormatter).toInstant()); - - List rows = Arrays.asList( - createRow("user1", "Joe", "Smith", Sets.newHashSet("jsmith@notareal.com"), - Arrays.asList("New York, NY", "Santa Clara, CA"), - new HashMap() {{ - put(aMonthPrior, "Set my alarm \"for\" a month from now"); - }}, false, 1.0f, 2.0), - createRow("user2", "Mary", "Jones", Sets.newHashSet("mjones@notareal.com"), - Collections.singletonList("Orlando, FL"), - new HashMap() {{ - put(testDate, "Get milk and bread"); - }}, true, 3.0f, 4.0) - ); - - ListenableFuture future = mock(ListenableFuture.class); - when(future.get()).thenReturn(rows); - when(resultSet.fetchMoreResults()).thenReturn(future); - - when(resultSet.iterator()).thenReturn(rows.iterator()); - when(resultSet.all()).thenReturn(rows); - when(resultSet.getAvailableWithoutFetching()).thenReturn(rows.size()); - when(resultSet.isFullyFetched()).thenReturn(false).thenReturn(true); - if(falseThenTrue) { - when(resultSet.isExhausted()).thenReturn(false, true); - }else{ - when(resultSet.isExhausted()).thenReturn(true); - } - when(resultSet.getColumnDefinitions()).thenReturn(columnDefinitions); - - return resultSet; - } - - public static ResultSet createMockResultSet() throws Exception { - return createMockResultSet(true); - } - - public static ResultSet createMockResultSetOneColumn() throws Exception { - ResultSet resultSet = mock(ResultSet.class); - ColumnDefinitions columnDefinitions = mock(ColumnDefinitions.class); - when(columnDefinitions.size()).thenReturn(1); - when(columnDefinitions.getName(anyInt())).thenAnswer(new Answer() { - List colNames = Arrays.asList("user_id"); - @Override - public String answer(InvocationOnMock invocationOnMock) throws Throwable { - return colNames.get((Integer) invocationOnMock.getArguments()[0]); - - } - }); - - when(columnDefinitions.getTable(0)).thenReturn("users"); - - when(columnDefinitions.getType(anyInt())).thenAnswer(new Answer() { - List dataTypes = Arrays.asList(DataType.text()); - @Override - public DataType answer(InvocationOnMock invocationOnMock) throws Throwable { - return dataTypes.get((Integer) invocationOnMock.getArguments()[0]); - - } - }); - - List rows = Arrays.asList( - createRow("user1"), - createRow("user2") - ); - - ListenableFuture future = mock(ListenableFuture.class); - when(future.get()).thenReturn(rows); - when(resultSet.fetchMoreResults()).thenReturn(future); - - when(resultSet.iterator()).thenReturn(rows.iterator()); - when(resultSet.all()).thenReturn(rows); - when(resultSet.getAvailableWithoutFetching()).thenReturn(rows.size()); - when(resultSet.isFullyFetched()).thenReturn(false).thenReturn(true); - when(resultSet.isExhausted()).thenReturn(false).thenReturn(true); - when(resultSet.getColumnDefinitions()).thenReturn(columnDefinitions); - return resultSet; - } - - public static ResultSet createMockDateResultSet() throws Exception { - ResultSet resultSet = mock(ResultSet.class); - ColumnDefinitions columnDefinitions = mock(ColumnDefinitions.class); - - when(columnDefinitions.size()).thenReturn(1); - when(columnDefinitions.getName(anyInt())).thenReturn("date"); - when(columnDefinitions.getTable(0)).thenReturn("users"); - when(columnDefinitions.getType(anyInt())).thenReturn(DataType.timestamp()); - - Row row = mock(Row.class); - when(row.getTimestamp(0)).thenReturn(TEST_DATE); - List rows = Collections.singletonList(row); - - when(resultSet.iterator()).thenReturn(rows.iterator()); - when(resultSet.all()).thenReturn(rows); - when(resultSet.getAvailableWithoutFetching()).thenReturn(rows.size()); - when(resultSet.isFullyFetched()).thenReturn(false).thenReturn(true); - when(resultSet.getColumnDefinitions()).thenReturn(columnDefinitions); - return resultSet; - } - - public static Row createRow(String user_id, String first_name, String last_name, Set emails, - List top_places, Map todo, boolean registered, - float scale, double metric) { - Row row = mock(Row.class); - when(row.getString(0)).thenReturn(user_id); - when(row.getString(1)).thenReturn(first_name); - when(row.getString(2)).thenReturn(last_name); - when(row.getSet(eq(3), any(TypeToken.class))).thenReturn(emails); - when(row.getList(eq(4), any(TypeToken.class))).thenReturn(top_places); - when(row.getMap(eq(5), any(TypeToken.class), any(TypeToken.class))).thenReturn(todo); - when(row.getBool(6)).thenReturn(registered); - when(row.getFloat(7)).thenReturn(scale); - when(row.getDouble(8)).thenReturn(metric); - - return row; - } - - public static Row createRow(String user_id) { - Row row = mock(Row.class); - when(row.getString(0)).thenReturn(user_id); - return row; - } -} - diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraQLTest.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraQLTest.java deleted file mode 100644 index 8b3534855783..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraQLTest.java +++ /dev/null @@ -1,434 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.BoundStatement; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.Configuration; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.Metadata; -import com.datastax.driver.core.PreparedStatement; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SniEndPoint; -import com.datastax.driver.core.Statement; -import com.datastax.driver.core.exceptions.InvalidQueryException; -import com.datastax.driver.core.exceptions.NoHostAvailableException; -import com.datastax.driver.core.exceptions.UnavailableException; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.net.ssl.SSLContext; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Unit tests for the PutCassandraQL processor - */ -public class PutCassandraQLTest { - - private TestRunner testRunner; - private MockPutCassandraQL processor; - - @BeforeEach - public void setUp() { - processor = new MockPutCassandraQL(); - testRunner = TestRunners.newTestRunner(processor); - } - - @Test - public void testProcessorConfigValidity() { - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.assertValid(); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.assertNotValid(); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "username"); - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.assertValid(); - } - - @Test - public void testProcessorELConfigValidity() { - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "${hosts}"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "${pass}"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "${user}"); - testRunner.setProperty(AbstractCassandraProcessor.CHARSET, "${charset}"); - testRunner.setProperty(PutCassandraQL.STATEMENT_TIMEOUT, "${timeout}"); - - testRunner.assertValid(); - } - - @Test - public void testProcessorHappyPath() { - setUpStandardTestConfig(); - - testRunner.enqueue("INSERT INTO users (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - new HashMap() { - { - put("cql.args.1.type", "int"); - put("cql.args.1.value", "1"); - put("cql.args.2.type", "text"); - put("cql.args.2.value", "Joe"); - put("cql.args.3.type", "text"); - // No value for arg 3 to test setNull - put("cql.args.4.type", "map"); - put("cql.args.4.value", "{'a':'Hello', 'b':'World'}"); - put("cql.args.5.type", "list"); - put("cql.args.5.value", "[true,false,true]"); - put("cql.args.6.type", "set"); - put("cql.args.6.value", "{1.0, 2.0}"); - put("cql.args.7.type", "bigint"); - put("cql.args.7.value", "20000000"); - put("cql.args.8.type", "float"); - put("cql.args.8.value", "1.0"); - put("cql.args.9.type", "blob"); - put("cql.args.9.value", "0xDEADBEEF"); - put("cql.args.10.type", "timestamp"); - put("cql.args.10.value", "2016-07-01T15:21:05Z"); - - } - }); - - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_SUCCESS, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorHappyPathELConfig() { - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "${hosts}"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "${pass}"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "${user}"); - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.setProperty(AbstractCassandraProcessor.CHARSET, "${charset}"); - testRunner.setProperty(PutCassandraQL.STATEMENT_TIMEOUT, "${timeout}"); - testRunner.assertValid(); - - testRunner.setEnvironmentVariableValue("hosts", "localhost:9042"); - testRunner.setEnvironmentVariableValue("user", "username"); - testRunner.setEnvironmentVariableValue("pass", "password"); - testRunner.setEnvironmentVariableValue("charset", "UTF-8"); - testRunner.setEnvironmentVariableValue("timeout", "30 sec"); - - testRunner.enqueue("INSERT INTO users (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - new HashMap() { - { - put("cql.args.1.type", "int"); - put("cql.args.1.value", "1"); - put("cql.args.2.type", "text"); - put("cql.args.2.value", "Joe"); - put("cql.args.3.type", "text"); - // No value for arg 3 to test setNull - put("cql.args.4.type", "map"); - put("cql.args.4.value", "{'a':'Hello', 'b':'World'}"); - put("cql.args.5.type", "list"); - put("cql.args.5.value", "[true,false,true]"); - put("cql.args.6.type", "set"); - put("cql.args.6.value", "{1.0, 2.0}"); - put("cql.args.7.type", "bigint"); - put("cql.args.7.value", "20000000"); - put("cql.args.8.type", "float"); - put("cql.args.8.value", "1.0"); - put("cql.args.9.type", "blob"); - put("cql.args.9.value", "0xDEADBEEF"); - put("cql.args.10.type", "timestamp"); - put("cql.args.10.value", "2016-07-01T15:21:05Z"); - - } - }); - - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_SUCCESS, 1); - testRunner.clearTransferState(); - } - - @Test - public void testMultipleQuery() { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraQL.STATEMENT_CACHE_SIZE, "1"); - - HashMap testData = new HashMap<>(); - testData.put("cql.args.1.type", "int"); - testData.put("cql.args.1.value", "1"); - testData.put("cql.args.2.type", "text"); - testData.put("cql.args.2.value", "Joe"); - testData.put("cql.args.3.type", "text"); - // No value for arg 3 to test setNull - testData.put("cql.args.4.type", "map"); - testData.put("cql.args.4.value", "{'a':'Hello', 'b':'World'}"); - testData.put("cql.args.5.type", "list"); - testData.put("cql.args.5.value", "[true,false,true]"); - testData.put("cql.args.6.type", "set"); - testData.put("cql.args.6.value", "{1.0, 2.0}"); - testData.put("cql.args.7.type", "bigint"); - testData.put("cql.args.7.value", "20000000"); - testData.put("cql.args.8.type", "float"); - testData.put("cql.args.8.value", "1.0"); - testData.put("cql.args.9.type", "blob"); - testData.put("cql.args.9.value", "0xDEADBEEF"); - testData.put("cql.args.10.type", "timestamp"); - testData.put("cql.args.10.value", "2016-07-01T15:21:05Z"); - - testRunner.enqueue("INSERT INTO users (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - testData); - - testRunner.enqueue("INSERT INTO newusers (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - testData); - - // Change it up a bit, the same statement is executed with different data - testData.put("cql.args.1.value", "2"); - testRunner.enqueue("INSERT INTO users (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - testData); - - testRunner.enqueue("INSERT INTO users (user_id) VALUES ('user_id data');"); - - testRunner.run(4, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_SUCCESS, 4); - } - - @Test - public void testProcessorBadTimestamp() { - setUpStandardTestConfig(); - processor.setExceptionToThrow( - new InvalidQueryException(new SniEndPoint(new InetSocketAddress("localhost", 9042), ""), "invalid timestamp")); - testRunner.enqueue("INSERT INTO users (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - new HashMap() { - { - put("cql.args.1.type", "int"); - put("cql.args.1.value", "1"); - put("cql.args.2.type", "text"); - put("cql.args.2.value", "Joe"); - put("cql.args.3.type", "text"); - // No value for arg 3 to test setNull - put("cql.args.4.type", "map"); - put("cql.args.4.value", "{'a':'Hello', 'b':'World'}"); - put("cql.args.5.type", "list"); - put("cql.args.5.value", "[true,false,true]"); - put("cql.args.6.type", "set"); - put("cql.args.6.value", "{1.0, 2.0}"); - put("cql.args.7.type", "bigint"); - put("cql.args.7.value", "20000000"); - put("cql.args.8.type", "float"); - put("cql.args.8.value", "1.0"); - put("cql.args.9.type", "blob"); - put("cql.args.9.value", "0xDEADBEEF"); - put("cql.args.10.type", "timestamp"); - put("cql.args.10.value", "not a timestamp"); - - } - }); - - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_FAILURE, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorUuid() { - setUpStandardTestConfig(); - - testRunner.enqueue("INSERT INTO users (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - new HashMap() { - { - put("cql.args.1.type", "int"); - put("cql.args.1.value", "1"); - put("cql.args.2.type", "text"); - put("cql.args.2.value", "Joe"); - put("cql.args.3.type", "text"); - // No value for arg 3 to test setNull - put("cql.args.4.type", "map"); - put("cql.args.4.value", "{'a':'Hello', 'b':'World'}"); - put("cql.args.5.type", "list"); - put("cql.args.5.value", "[true,false,true]"); - put("cql.args.6.type", "set"); - put("cql.args.6.value", "{1.0, 2.0}"); - put("cql.args.7.type", "bigint"); - put("cql.args.7.value", "20000000"); - put("cql.args.8.type", "float"); - put("cql.args.8.value", "1.0"); - put("cql.args.9.type", "blob"); - put("cql.args.9.value", "0xDEADBEEF"); - put("cql.args.10.type", "uuid"); - put("cql.args.10.value", "5442b1f6-4c16-11ea-87f5-45a32dbc5199"); - - } - }); - - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_SUCCESS, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorBadUuid() { - setUpStandardTestConfig(); - - testRunner.enqueue("INSERT INTO users (user_id, first_name, last_name, properties, bits, scaleset, largenum, scale, byteobject, ts) VALUES ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", - new HashMap() { - { - put("cql.args.1.type", "int"); - put("cql.args.1.value", "1"); - put("cql.args.2.type", "text"); - put("cql.args.2.value", "Joe"); - put("cql.args.3.type", "text"); - // No value for arg 3 to test setNull - put("cql.args.4.type", "map"); - put("cql.args.4.value", "{'a':'Hello', 'b':'World'}"); - put("cql.args.5.type", "list"); - put("cql.args.5.value", "[true,false,true]"); - put("cql.args.6.type", "set"); - put("cql.args.6.value", "{1.0, 2.0}"); - put("cql.args.7.type", "bigint"); - put("cql.args.7.value", "20000000"); - put("cql.args.8.type", "float"); - put("cql.args.8.value", "1.0"); - put("cql.args.9.type", "blob"); - put("cql.args.9.value", "0xDEADBEEF"); - put("cql.args.10.type", "uuid"); - put("cql.args.10.value", "bad-uuid"); - - } - }); - - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_FAILURE, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorInvalidQueryException() { - setUpStandardTestConfig(); - - // Test exceptions - processor.setExceptionToThrow( - new InvalidQueryException(new SniEndPoint(new InetSocketAddress("localhost", 9042), ""), "invalid query")); - testRunner.enqueue("UPDATE users SET cities = [ 'New York', 'Los Angeles' ] WHERE user_id = 'coast2coast';"); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_FAILURE, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorUnavailableException() { - setUpStandardTestConfig(); - - processor.setExceptionToThrow( - new UnavailableException(new SniEndPoint(new InetSocketAddress("localhost", 9042), ""), ConsistencyLevel.ALL, 5, 2)); - testRunner.enqueue("UPDATE users SET cities = [ 'New York', 'Los Angeles' ] WHERE user_id = 'coast2coast';"); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_RETRY, 1); - } - - @Test - public void testProcessorNoHostAvailableException() { - setUpStandardTestConfig(); - - processor.setExceptionToThrow(new NoHostAvailableException(new HashMap<>())); - testRunner.enqueue("UPDATE users SET cities = [ 'New York', 'Los Angeles' ] WHERE user_id = 'coast2coast';"); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_RETRY, 1); - } - - @Test - public void testProcessorProcessException() { - setUpStandardTestConfig(); - - processor.setExceptionToThrow(new ProcessException()); - testRunner.enqueue("UPDATE users SET cities = [ 'New York', 'Los Angeles' ] WHERE user_id = 'coast2coast';"); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraQL.REL_FAILURE, 1); - } - - private void setUpStandardTestConfig() { - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "username"); - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.assertValid(); - } - - /** - * Provides a stubbed processor instance for testing - */ - private static class MockPutCassandraQL extends PutCassandraQL { - - private Exception exceptionToThrow = null; - private Session mockSession = mock(Session.class); - - @Override - protected Cluster createCluster(List contactPoints, SSLContext sslContext, - String username, String password, String compressionType) { - Cluster mockCluster = mock(Cluster.class); - try { - Metadata mockMetadata = mock(Metadata.class); - when(mockMetadata.getClusterName()).thenReturn("cluster1"); - when(mockCluster.getMetadata()).thenReturn(mockMetadata); - when(mockCluster.connect()).thenReturn(mockSession); - when(mockCluster.connect(anyString())).thenReturn(mockSession); - Configuration config = Configuration.builder().build(); - when(mockCluster.getConfiguration()).thenReturn(config); - ResultSetFuture future = mock(ResultSetFuture.class); - ResultSet rs = CassandraQueryTestUtil.createMockResultSet(); - PreparedStatement ps = mock(PreparedStatement.class); - when(mockSession.prepare(anyString())).thenReturn(ps); - BoundStatement bs = mock(BoundStatement.class); - when(ps.bind()).thenReturn(bs); - when(future.getUninterruptibly()).thenReturn(rs); - try { - doReturn(rs).when(future).getUninterruptibly(anyLong(), any(TimeUnit.class)); - } catch (TimeoutException te) { - throw new IllegalArgumentException("Mocked cluster doesn't time out"); - } - if (exceptionToThrow != null) { - doThrow(exceptionToThrow).when(mockSession).executeAsync(anyString()); - doThrow(exceptionToThrow).when(mockSession).executeAsync(any(Statement.class)); - - } else { - when(mockSession.executeAsync(anyString())).thenReturn(future); - when(mockSession.executeAsync(any(Statement.class))).thenReturn(future); - } - when(mockSession.getCluster()).thenReturn(mockCluster); - } catch (Exception e) { - fail(e.getMessage()); - } - return mockCluster; - } - - void setExceptionToThrow(Exception e) { - exceptionToThrow = e; - doThrow(exceptionToThrow).when(mockSession).executeAsync(anyString()); - doThrow(exceptionToThrow).when(mockSession).executeAsync(any(Statement.class)); - } - - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordIT.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordIT.java deleted file mode 100644 index f0943c2afbcd..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordIT.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import com.datastax.driver.core.querybuilder.Select; -import com.datastax.driver.core.querybuilder.Truncate; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.serialization.record.MockRecordParser; -import org.apache.nifi.serialization.record.RecordFieldType; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.CassandraContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.DockerImageName; - -import java.net.InetSocketAddress; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@Testcontainers -public class PutCassandraRecordIT { - @Container - private static final CassandraContainer CASSANDRA_CONTAINER = new CassandraContainer(DockerImageName.parse("cassandra:4.1")); - - private static TestRunner testRunner; - private static MockRecordParser recordReader; - - private static Cluster cluster; - private static Session session; - - private static final String KEYSPACE = "sample_keyspace"; - private static final String TABLE = "sample_table"; - - @BeforeAll - public static void setup() throws InitializationException { - recordReader = new MockRecordParser(); - testRunner = TestRunners.newTestRunner(PutCassandraRecord.class); - - InetSocketAddress contactPoint = CASSANDRA_CONTAINER.getContactPoint(); - testRunner.setProperty(PutCassandraRecord.RECORD_READER_FACTORY, "reader"); - testRunner.setProperty(PutCassandraRecord.CONTACT_POINTS, contactPoint.getHostString() + ":" + contactPoint.getPort()); - testRunner.setProperty(PutCassandraRecord.KEYSPACE, KEYSPACE); - testRunner.setProperty(PutCassandraRecord.TABLE, TABLE); - testRunner.setProperty(PutCassandraRecord.CONSISTENCY_LEVEL, "SERIAL"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, "LOGGED"); - testRunner.addControllerService("reader", recordReader); - testRunner.enableControllerService(recordReader); - - cluster = Cluster.builder().addContactPoint(contactPoint.getHostName()) - .withPort(contactPoint.getPort()).build(); - session = cluster.connect(); - - String createKeyspace = "CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class':'SimpleStrategy','replication_factor':1};"; - String createTable = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + TABLE + "(id int PRIMARY KEY, name text, age int);"; - - session.execute(createKeyspace); - session.execute(createTable); - } - - @Test - public void testSimplePut() { - recordReader.addSchemaField("id", RecordFieldType.INT); - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - - recordReader.addRecord(1, "Ram", 42); - recordReader.addRecord(2, "Jeane", 47); - recordReader.addRecord(3, "Ilamaran", 27); - recordReader.addRecord(4, "Jian", 14); - recordReader.addRecord(5, "Sakura", 24); - - testRunner.enqueue(""); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(PutCassandraRecord.REL_SUCCESS, 1); - assertEquals(5, getRecordsCount()); - } - - private int getRecordsCount() { - Select selectQuery = QueryBuilder.select().all().from(KEYSPACE, TABLE); - ResultSet result = session.execute(selectQuery); - - List resultsList = result.all() - .stream() - .map(r -> r.getInt(0)) - .collect(Collectors.toList()); - - dropRecords(); - return resultsList.size(); - } - - private void dropRecords() { - Truncate query = QueryBuilder.truncate(KEYSPACE, TABLE); - session.execute(query); - } - - @AfterAll - public static void shutdown() { - String dropKeyspace = "DROP KEYSPACE " + KEYSPACE; - String dropTable = "DROP TABLE IF EXISTS " + KEYSPACE + "." + TABLE; - - session.execute(dropTable); - session.execute(dropKeyspace); - - session.close(); - cluster.close(); - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordInsertTest.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordInsertTest.java deleted file mode 100644 index 2cb09db79057..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordInsertTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.querybuilder.Insert; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.nifi.serialization.record.RecordFieldType; -import org.apache.nifi.serialization.record.RecordSchema; -import org.apache.nifi.util.Tuple; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.when; - -public class PutCassandraRecordInsertTest { - private PutCassandraRecord testSubject; - - @Mock - private RecordSchema schema; - private AutoCloseable mockCloseable; - - @BeforeEach - public void setUp() { - mockCloseable = MockitoAnnotations.openMocks(this); - testSubject = new PutCassandraRecord(); - } - - @AfterEach - public void closeMock() throws Exception { - if (mockCloseable != null) { - mockCloseable.close(); - } - } - - @Test - public void testGenerateInsert() { - testGenerateInsert( - "keyspace.table", - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("integerField", 15), - new Tuple<>("longField", 67L), - new Tuple<>("stringField", "abcd") - ), - Arrays.asList( - new Tuple<>("keyField", RecordFieldType.INT.getDataType()), - new Tuple<>("integerField", RecordFieldType.INT.getDataType()), - new Tuple<>("longField", RecordFieldType.LONG.getDataType()), - new Tuple<>("stringField", RecordFieldType.STRING.getDataType()) - ), - "INSERT INTO keyspace.table (keyField,integerField,longField,stringField) VALUES (1,15,67,'abcd');" - ); - } - - @Test - public void testGenerateInsertStringArray() { - testGenerateInsert( - "keyspace.table", - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("integerField", 15), - new Tuple<>("arrayField", new Object[]{"test1", "test2"}) - ), - Arrays.asList( - new Tuple<>("keyField", RecordFieldType.INT.getDataType()), - new Tuple<>("integerField", RecordFieldType.INT.getDataType()), - new Tuple<>("arrayField", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType())) - ), - "INSERT INTO keyspace.table (keyField,integerField,arrayField) VALUES (1,15,['test1','test2']);" - ); - } - - private void testGenerateInsert(String table, List> records, List> recordSchema, String expected) { - Map recordContentMap = records.stream() - .collect(Collectors.toMap(Tuple::getKey, Tuple::getValue)); - - Map recordSchemaMap = recordSchema.stream() - .collect(Collectors.toMap(Tuple::getKey, Tuple::getValue)); - - List fieldNames = records.stream().map(Tuple::getKey).collect(Collectors.toList()); - - when(schema.getFieldNames()).thenReturn(fieldNames); - when(schema.getDataType(anyString())).thenAnswer(i -> Optional.of(recordSchemaMap.get(i.getArgument(0)))); - - Insert actual = (Insert)testSubject.generateInsert(table, schema, recordContentMap); - actual.setForceNoValues(true); - - // Codecs are normally registered in the onScheduled method - testSubject.registerAdditionalCodecs(); - - assertEquals(expected, actual.getQueryString()); - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordTest.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordTest.java deleted file mode 100644 index db1727bd0b6a..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordTest.java +++ /dev/null @@ -1,552 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.BoundStatement; -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.Configuration; -import com.datastax.driver.core.Metadata; -import com.datastax.driver.core.PreparedStatement; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.Statement; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.serialization.record.MockRecordParser; -import org.apache.nifi.serialization.record.RecordField; -import org.apache.nifi.serialization.record.RecordFieldType; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.net.ssl.SSLContext; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class PutCassandraRecordTest { - - private TestRunner testRunner; - private MockRecordParser recordReader; - - @BeforeEach - public void setUp() throws Exception { - MockPutCassandraRecord processor = new MockPutCassandraRecord(); - recordReader = new MockRecordParser(); - testRunner = TestRunners.newTestRunner(processor); - testRunner.setProperty(PutCassandraRecord.RECORD_READER_FACTORY, "reader"); - } - - @Test - public void testProcessorConfigValidity() throws InitializationException { - testRunner.setProperty(PutCassandraRecord.CONTACT_POINTS, "localhost:9042"); - testRunner.assertNotValid(); - - testRunner.setProperty(PutCassandraRecord.PASSWORD, "password"); - testRunner.assertNotValid(); - - testRunner.setProperty(PutCassandraRecord.USERNAME, "username"); - testRunner.assertNotValid(); - - testRunner.setProperty(PutCassandraRecord.CONSISTENCY_LEVEL, "SERIAL"); - testRunner.assertNotValid(); - - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, "LOGGED"); - testRunner.assertNotValid(); - - testRunner.setProperty(PutCassandraRecord.KEYSPACE, "sampleks"); - testRunner.assertNotValid(); - - testRunner.setProperty(PutCassandraRecord.TABLE, "sampletbl"); - testRunner.assertNotValid(); - - testRunner.addControllerService("reader", recordReader); - testRunner.enableControllerService(recordReader); - testRunner.assertValid(); - } - - private void setUpStandardTestConfig() throws InitializationException { - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "username"); - testRunner.setProperty(PutCassandraRecord.CONSISTENCY_LEVEL, "SERIAL"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, "LOGGED"); - testRunner.setProperty(PutCassandraRecord.TABLE, "sampleks.sampletbl"); - testRunner.addControllerService("reader", recordReader); - testRunner.enableControllerService(recordReader); - } - - @Test - public void testSimplePut() throws InitializationException { - setUpStandardTestConfig(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("sport", RecordFieldType.STRING); - - recordReader.addRecord("John Doe", 48, "Soccer"); - recordReader.addRecord("Jane Doe", 47, "Tennis"); - recordReader.addRecord("Sally Doe", 47, "Curling"); - recordReader.addRecord("Jimmy Doe", 14, null); - recordReader.addRecord("Pizza Doe", 14, null); - - testRunner.enqueue(""); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(PutCassandraRecord.REL_SUCCESS, 1); - } - - @Test - public void testStringArrayPut() throws InitializationException { - setUpStandardTestConfig(); - - recordReader.addSchemaField(new RecordField("names", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()))); - recordReader.addSchemaField("age", RecordFieldType.INT); - - recordReader.addRecord(new Object[]{"John", "Doe"}, 1); - recordReader.addRecord(new Object[]{"John", "Doe"}, 2); - recordReader.addRecord(new Object[]{"John", "Doe"}, 3); - - testRunner.enqueue(""); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(PutCassandraRecord.REL_SUCCESS, 1); - } - - @Test - public void testSimpleUpdate() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.SET_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "name,age"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.INT); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 2L); - recordReader.addRecord("Sally Doe", 47, 0); - - testRunner.enqueue(""); - testRunner.run(); - - testRunner.assertAllFlowFilesTransferred(PutCassandraRecord.REL_SUCCESS, 1); - } - - @Test - public void testUpdateInvalidFieldType() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.INCR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "name,age"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.STRING); - - recordReader.addRecord("John Doe", 48,"1"); - recordReader.addRecord("Jane Doe", 47, "1"); - recordReader.addRecord("Sally Doe", 47, "1"); - - testRunner.enqueue(""); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testUpdateEmptyUpdateKeys() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.INCR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, ""); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - testRunner.assertNotValid(); - } - - @Test - public void testUpdateNullUpdateKeys() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.SET_TYPE); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - testRunner.assertNotValid(); - } - - @Test - public void testUpdateSetLoggedBatch() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.SET_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "name,age"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.LOGGED_TYPE); - - testRunner.assertValid(); - } - - @Test - public void testUpdateCounterWrongBatchStatementType() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.INCR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "name,age"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.LOGGED_TYPE); - - testRunner.assertNotValid(); - } - - @Test - public void testUpdateWithUpdateMethodAndKeyAttributes() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.UPDATE_METHOD_USE_ATTR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "${cql.update.keys}"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.LONG); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - Map attributes = new HashMap<>(); - attributes.put("cql.update.method", "Increment"); - attributes.put("cql.update.keys", "name,age"); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testInsertWithStatementAttribute() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.STATEMENT_TYPE_USE_ATTR_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.LONG); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - Map attributes = new HashMap<>(); - attributes.put("cql.statement.type", "Insert"); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testInsertWithStatementAttributeInvalid() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.STATEMENT_TYPE_USE_ATTR_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.LONG); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - Map attributes = new HashMap<>(); - attributes.put("cql.statement.type", "invalid-type"); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testInsertWithBatchStatementAttribute() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.INSERT_TYPE); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.BATCH_STATEMENT_TYPE_USE_ATTR_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.LONG); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - Map attributes = new HashMap<>(); - attributes.put("cql.batch.statement.type", "counter"); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testInsertWithBatchStatementAttributeInvalid() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.INSERT_TYPE); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.BATCH_STATEMENT_TYPE_USE_ATTR_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.LONG); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - Map attributes = new HashMap<>(); - attributes.put("cql.batch.statement.type", "invalid-type"); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testUpdateWithAttributesInvalidUpdateMethod() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.UPDATE_METHOD_USE_ATTR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "${cql.update.keys}"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.INT); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - Map attributes = new HashMap<>(); - attributes.put("cql.update.method", "invalid-method"); - attributes.put("cql.update.keys", "name,age"); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testUpdateWithAttributesIncompatibleBatchStatementType() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.INCR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "name,age"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.BATCH_STATEMENT_TYPE_USE_ATTR_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.INT); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - Map attributes = new HashMap<>(); - attributes.put("cql.batch.statement.type", "LOGGED"); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testUpdateWithAttributesEmptyUpdateKeysAttribute() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.UPDATE_METHOD_USE_ATTR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "${cql.update.keys}"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.LONG); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - HashMap attributes = new HashMap<>(); - attributes.put("cql.update.method", "Increment"); - attributes.put("cql.update.keys", ""); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testUpdateWithAttributesEmptyUpdateMethodAttribute() throws InitializationException { - setUpStandardTestConfig(); - testRunner.setProperty(PutCassandraRecord.STATEMENT_TYPE, PutCassandraRecord.UPDATE_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_METHOD, PutCassandraRecord.UPDATE_METHOD_USE_ATTR_TYPE); - testRunner.setProperty(PutCassandraRecord.UPDATE_KEYS, "name,age"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, PutCassandraRecord.COUNTER_TYPE); - - testRunner.assertValid(); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("goals", RecordFieldType.LONG); - - recordReader.addRecord("John Doe", 48, 1L); - recordReader.addRecord("Jane Doe", 47, 1L); - recordReader.addRecord("Sally Doe", 47, 1L); - - HashMap attributes = new HashMap<>(); - attributes.put("cql.update.method", ""); - testRunner.enqueue("", attributes); - testRunner.run(); - - testRunner.assertTransferCount(PutCassandraRecord.REL_FAILURE, 1); - testRunner.assertTransferCount(PutCassandraRecord.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutCassandraRecord.REL_RETRY, 0); - } - - @Test - public void testEL() throws InitializationException { - testRunner.setProperty(PutCassandraRecord.CONTACT_POINTS, "${contact.points}"); - testRunner.setProperty(PutCassandraRecord.PASSWORD, "${pass}"); - testRunner.setProperty(PutCassandraRecord.USERNAME, "${user}"); - testRunner.setProperty(PutCassandraRecord.CONSISTENCY_LEVEL, "SERIAL"); - testRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, "LOGGED"); - testRunner.setProperty(PutCassandraRecord.TABLE, "sampleks.sampletbl"); - testRunner.addControllerService("reader", recordReader); - testRunner.enableControllerService(recordReader); - - testRunner.assertValid(); - - testRunner.setEnvironmentVariableValue("contact.points", "localhost:9042"); - testRunner.setEnvironmentVariableValue("user", "username"); - testRunner.setEnvironmentVariableValue("pass", "password"); - - recordReader.addSchemaField("name", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - recordReader.addSchemaField("sport", RecordFieldType.STRING); - - recordReader.addRecord("John Doe", 48, "Soccer"); - recordReader.addRecord("Jane Doe", 47, "Tennis"); - recordReader.addRecord("Sally Doe", 47, "Curling"); - recordReader.addRecord("Jimmy Doe", 14, null); - recordReader.addRecord("Pizza Doe", 14, null); - - testRunner.enqueue(""); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(PutCassandraRecord.REL_SUCCESS, 1); - } - - private static class MockPutCassandraRecord extends PutCassandraRecord { - private Exception exceptionToThrow = null; - private Session mockSession = mock(Session.class); - - @Override - protected Cluster createCluster(List contactPoints, SSLContext sslContext, - String username, String password, String compressionType) { - Cluster mockCluster = mock(Cluster.class); - try { - Metadata mockMetadata = mock(Metadata.class); - when(mockMetadata.getClusterName()).thenReturn("cluster1"); - when(mockCluster.getMetadata()).thenReturn(mockMetadata); - when(mockCluster.connect()).thenReturn(mockSession); - when(mockCluster.connect(anyString())).thenReturn(mockSession); - Configuration config = Configuration.builder().build(); - when(mockCluster.getConfiguration()).thenReturn(config); - ResultSetFuture future = mock(ResultSetFuture.class); - ResultSet rs = CassandraQueryTestUtil.createMockResultSet(); - PreparedStatement ps = mock(PreparedStatement.class); - when(mockSession.prepare(anyString())).thenReturn(ps); - BoundStatement bs = mock(BoundStatement.class); - when(ps.bind()).thenReturn(bs); - when(future.getUninterruptibly()).thenReturn(rs); - try { - doReturn(rs).when(future).getUninterruptibly(anyLong(), any(TimeUnit.class)); - } catch (TimeoutException te) { - throw new IllegalArgumentException("Mocked cluster doesn't time out"); - } - if (exceptionToThrow != null) { - doThrow(exceptionToThrow).when(mockSession).executeAsync(anyString()); - doThrow(exceptionToThrow).when(mockSession).executeAsync(any(Statement.class)); - - } else { - when(mockSession.executeAsync(anyString())).thenReturn(future); - when(mockSession.executeAsync(any(Statement.class))).thenReturn(future); - } - when(mockSession.getCluster()).thenReturn(mockCluster); - } catch (Exception e) { - fail(e.getMessage()); - } - return mockCluster; - } - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordUpdateTest.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordUpdateTest.java deleted file mode 100644 index 6afc14ac08c9..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/PutCassandraRecordUpdateTest.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.nifi.serialization.record.RecordSchema; -import org.apache.nifi.util.Tuple; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class PutCassandraRecordUpdateTest { - private PutCassandraRecord testSubject; - - private final RecordSchema schema = mock(RecordSchema.class); - - @BeforeEach - public void setUp() { - testSubject = new PutCassandraRecord(); - } - - - @Test - public void testGenerateUpdateWithEmptyKeyList() { - Stream.of("", ",", ",,,").forEach(updateKeys -> testGenerateUpdate( - "keyspace.table", - updateKeys, - PutCassandraRecord.SET_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("stringField", "newStringValue") - ), - new IllegalArgumentException("No Update Keys were specified") - )); - } - - @Test - public void testGenerateUpdateWithMissingKey() { - testGenerateUpdate( - "keyspace.table", - "keyField,missingKeyField", - PutCassandraRecord.SET_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("stringField", "newStringValue") - ), - new IllegalArgumentException("Update key 'missingKeyField' is not present in the record schema") - ); - } - - @Test - public void testGenerateUpdateWithInvalidUpdateMethod() { - testGenerateUpdate( - "keyspace.table", - "keyField", - "invalidUpdateMethod", - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("longField", 15L) - ), - new IllegalArgumentException("Update Method 'invalidUpdateMethod' is not valid.") - ); - } - - @Test - public void testGenerateUpdateIncrementString() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.INCR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("stringField", "15") - ), - new IllegalArgumentException("Field 'stringField' is not of type Number") - ); - } - - @Test - public void testGenerateUpdateSimpleTableName() { - testGenerateUpdate( - "table", - "keyField1", - PutCassandraRecord.SET_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField1", 1), - new Tuple<>("stringField", "newStringValue") - ), - "UPDATE table SET stringField='newStringValue' WHERE keyField1=1;" - ); - } - - @Test - public void testGenerateUpdateKeyspacedTableName() { - testGenerateUpdate( - "keyspace.table", - "keyField1", - PutCassandraRecord.SET_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField1", 1), - new Tuple<>("stringField", "newStringValue") - ), - "UPDATE keyspace.table SET stringField='newStringValue' WHERE keyField1=1;" - ); - } - - @Test - public void testGenerateUpdateMultipleKeys() { - testGenerateUpdate( - "keyspace.table", - "keyField1,keyField2,keyField3", - PutCassandraRecord.SET_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField1", 1), - new Tuple<>("keyField2", "key2"), - new Tuple<>("keyField3", 123L), - new Tuple<>("stringField", "newStringValue") - ), - "UPDATE keyspace.table SET stringField='newStringValue' WHERE keyField1=1 AND keyField2='key2' AND keyField3=123;" - ); - } - - @Test - public void testGenerateUpdateIncrementLong() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.INCR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("longField", 15L) - ), - "UPDATE keyspace.table SET longField=longField+15 WHERE keyField=1;" - ); - } - - @Test - public void testGenerateUpdateDecrementLong() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.DECR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("longField", 15L) - ), - "UPDATE keyspace.table SET longField=longField-15 WHERE keyField=1;" - ); - } - - @Test - public void testGenerateUpdateIncrementInteger() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.INCR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("integerField", 15) - ), - "UPDATE keyspace.table SET integerField=integerField+15 WHERE keyField=1;" - ); - } - - @Test - public void testGenerateUpdateIncrementFloat() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.INCR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("floatField", 15.05F) - ), - "UPDATE keyspace.table SET floatField=floatField+15 WHERE keyField=1;" - ); - } - - @Test - public void testGenerateUpdateIncrementDouble() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.INCR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("doubleField", 15.05D) - ), - "UPDATE keyspace.table SET doubleField=doubleField+15 WHERE keyField=1;" - ); - } - - @Test - public void testGenerateUpdateSetMultipleValues() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.SET_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("stringField", "newStringValue"), - new Tuple<>("integerField", 15), - new Tuple<>("longField", 67L) - ), - "UPDATE keyspace.table SET stringField='newStringValue',integerField=15,longField=67 WHERE keyField=1;" - ); - } - - @Test - public void testGenerateUpdateIncrementMultipleValues() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.INCR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("integerField", 15), - new Tuple<>("longField", 67L) - ), - "UPDATE keyspace.table SET integerField=integerField+15,longField=longField+67 WHERE keyField=1;" - ); - } - - @Test - public void testGenerateUpdateDecrementMultipleValues() { - testGenerateUpdate( - "keyspace.table", - "keyField", - PutCassandraRecord.DECR_TYPE.getValue(), - Arrays.asList( - new Tuple<>("keyField", 1), - new Tuple<>("integerField", 15), - new Tuple<>("longField", 67L) - ), - "UPDATE keyspace.table SET integerField=integerField-15,longField=longField-67 WHERE keyField=1;" - ); - } - - private void testGenerateUpdate(String table, String updateKeys, String updateMethod, List> records, String expected) { - Map recordContentMap = records.stream() - .collect(Collectors.toMap(Tuple::getKey, Tuple::getValue)); - - List fieldNames = records.stream().map(Tuple::getKey).collect(Collectors.toList()); - - when(schema.getFieldNames()).thenReturn(fieldNames); - Statement actual = testSubject.generateUpdate(table, schema, updateKeys, updateMethod, recordContentMap); - - assertEquals(expected, actual.toString()); - } - - private void testGenerateUpdate(String table, String updateKeys, String updateMethod, List> records, E expected) { - Map recordContentMap = records.stream() - .collect(Collectors.toMap(Tuple::getKey, Tuple::getValue)); - - List fieldNames = records.stream().map(Tuple::getKey).collect(Collectors.toList()); - - when(schema.getFieldNames()).thenReturn(fieldNames); - try { - testSubject.generateUpdate("keyspace.table", schema, updateKeys, updateMethod, recordContentMap); - fail(); - } catch (Exception e) { - assertEquals(expected.getClass(), e.getClass()); - assertEquals(expected.getMessage(), e.getMessage()); - } - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraIT.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraIT.java deleted file mode 100644 index ac1f089d7993..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraIT.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.querybuilder.QueryBuilder; -import com.datastax.driver.core.querybuilder.Select; -import com.datastax.driver.core.querybuilder.Truncate; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.serialization.record.MockRecordParser; -import org.apache.nifi.serialization.record.RecordFieldType; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.CassandraContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.DockerImageName; - -import java.net.InetSocketAddress; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@Testcontainers -public class QueryCassandraIT { - @Container - private static final CassandraContainer CASSANDRA_CONTAINER = new CassandraContainer(DockerImageName.parse("cassandra:4.1")); - - private static TestRunner putCassandraTestRunner; - private static TestRunner queryCassandraTestRunner; - private static MockRecordParser recordReader; - - private static Cluster cluster; - private static Session session; - - private static final int LOAD_FLOW_FILE_SIZE = 100; - private static final int LOAD_FLOW_FILE_BATCH_SIZE = 10; - - private static final String KEYSPACE = "sample_keyspace"; - private static final String TABLE = "sample_table"; - - @BeforeAll - public static void setup() throws InitializationException { - recordReader = new MockRecordParser(); - putCassandraTestRunner = TestRunners.newTestRunner(PutCassandraRecord.class); - queryCassandraTestRunner = TestRunners.newTestRunner(QueryCassandra.class); - - InetSocketAddress contactPoint = CASSANDRA_CONTAINER.getContactPoint(); - putCassandraTestRunner.setProperty(PutCassandraRecord.RECORD_READER_FACTORY, "reader"); - putCassandraTestRunner.setProperty(PutCassandraRecord.CONTACT_POINTS, contactPoint.getHostString() + ":" + contactPoint.getPort()); - putCassandraTestRunner.setProperty(PutCassandraRecord.KEYSPACE, KEYSPACE); - putCassandraTestRunner.setProperty(PutCassandraRecord.TABLE, TABLE); - putCassandraTestRunner.setProperty(PutCassandraRecord.CONSISTENCY_LEVEL, "SERIAL"); - putCassandraTestRunner.setProperty(PutCassandraRecord.BATCH_STATEMENT_TYPE, "LOGGED"); - putCassandraTestRunner.addControllerService("reader", recordReader); - putCassandraTestRunner.enableControllerService(recordReader); - - queryCassandraTestRunner.setProperty(QueryCassandra.CONTACT_POINTS, contactPoint.getHostName() + ":" + contactPoint.getPort()); - queryCassandraTestRunner.setProperty(QueryCassandra.FETCH_SIZE, "10"); - queryCassandraTestRunner.setProperty(QueryCassandra.OUTPUT_BATCH_SIZE, "10"); - queryCassandraTestRunner.setProperty(QueryCassandra.KEYSPACE, KEYSPACE); - queryCassandraTestRunner.setProperty(QueryCassandra.CQL_SELECT_QUERY, "select * from " + TABLE + ";"); - - cluster = Cluster.builder().addContactPoint(contactPoint.getHostName()) - .withPort(contactPoint.getPort()).build(); - session = cluster.connect(); - - String createKeyspace = "CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class':'SimpleStrategy','replication_factor':1};"; - String createTable = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + TABLE + "(id int PRIMARY KEY, uuid text, age int);"; - - session.execute(createKeyspace); - session.execute(createTable); - loadData(); - } - - private static void loadData() { - recordReader.addSchemaField("id", RecordFieldType.INT); - recordReader.addSchemaField("uuid", RecordFieldType.STRING); - recordReader.addSchemaField("age", RecordFieldType.INT); - int recordCount = 0; - - for (int i = 0; i resultsList = result.all() - .stream() - .map(r -> r.getInt(0)) - .collect(Collectors.toList()); - - return resultsList.size(); - } - - private void dropRecords() { - Truncate query = QueryBuilder.truncate(KEYSPACE, TABLE); - session.execute(query); - } - - @AfterAll - public static void shutdown() { - String dropKeyspace = "DROP KEYSPACE " + KEYSPACE; - String dropTable = "DROP TABLE IF EXISTS " + KEYSPACE + "." + TABLE; - - session.execute(dropTable); - session.execute(dropKeyspace); - - session.close(); - cluster.close(); - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraTest.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraTest.java deleted file mode 100644 index fa3505b83143..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/test/java/org/apache/nifi/processors/cassandra/QueryCassandraTest.java +++ /dev/null @@ -1,594 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cassandra; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.Configuration; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.Metadata; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.ResultSetFuture; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SniEndPoint; -import com.datastax.driver.core.exceptions.InvalidQueryException; -import com.datastax.driver.core.exceptions.NoHostAvailableException; -import com.datastax.driver.core.exceptions.ReadTimeoutException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.avro.Schema; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.MockProcessContext; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import javax.net.ssl.SSLContext; -import java.io.ByteArrayOutputStream; -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class QueryCassandraTest { - - private TestRunner testRunner; - private MockQueryCassandra processor; - - @BeforeEach - public void setUp() { - processor = new MockQueryCassandra(); - testRunner = TestRunners.newTestRunner(processor); - } - - @Test - public void testProcessorConfigValid() { - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.assertNotValid(); - testRunner.setProperty(QueryCassandra.CQL_SELECT_QUERY, "select * from test"); - testRunner.assertValid(); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.assertNotValid(); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "username"); - testRunner.assertValid(); - - testRunner.setProperty(QueryCassandra.TIMESTAMP_FORMAT_PATTERN, "invalid format"); - testRunner.assertNotValid(); - testRunner.setProperty(QueryCassandra.TIMESTAMP_FORMAT_PATTERN, "yyyy-MM-dd HH:mm:ss.SSSZ"); - testRunner.assertValid(); - } - - @Test - public void testProcessorELConfigValid() { - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "${hosts}"); - testRunner.setProperty(QueryCassandra.CQL_SELECT_QUERY, "${query}"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "${pass}"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "${user}"); - testRunner.assertValid(); - } - - @Test - public void testProcessorNoInputFlowFileAndExceptions() { - setUpStandardProcessorConfig(); - - // Test no input flowfile - testRunner.setIncomingConnection(false); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_SUCCESS, 1); - testRunner.clearTransferState(); - - // Test exceptions - processor.setExceptionToThrow(new NoHostAvailableException(new HashMap<>())); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_RETRY, 1); - testRunner.clearTransferState(); - - processor.setExceptionToThrow( - new ReadTimeoutException(new SniEndPoint(new InetSocketAddress("localhost", 9042), ""), ConsistencyLevel.ANY, 0, 1, false)); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_RETRY, 1); - testRunner.clearTransferState(); - - processor.setExceptionToThrow( - new InvalidQueryException(new SniEndPoint(new InetSocketAddress("localhost", 9042), ""), "invalid query")); - testRunner.run(1, true, true); - // No files transferred to failure if there was no incoming connection - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_FAILURE, 0); - testRunner.clearTransferState(); - - processor.setExceptionToThrow(new ProcessException()); - testRunner.run(1, true, true); - // No files transferred to failure if there was no incoming connection - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_FAILURE, 0); - testRunner.clearTransferState(); - processor.setExceptionToThrow(null); - - } - - @Test - public void testProcessorJsonOutput() { - setUpStandardProcessorConfig(); - testRunner.setIncomingConnection(false); - - // Test JSON output - testRunner.setProperty(QueryCassandra.OUTPUT_FORMAT, QueryCassandra.JSON_FORMAT); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_SUCCESS, 1); - List files = testRunner.getFlowFilesForRelationship(QueryCassandra.REL_SUCCESS); - assertNotNull(files); - assertEquals(1, files.size(), "One file should be transferred to success"); - assertEquals("{\"results\":[{\"user_id\":\"user1\",\"first_name\":\"Joe\",\"last_name\":\"Smith\"," - + "\"emails\":[\"jsmith@notareal.com\"],\"top_places\":[\"New York, NY\",\"Santa Clara, CA\"]," - + "\"todo\":{\"2016-01-03 05:00:00+0000\":\"Set my alarm \\\"for\\\" a month from now\"}," - + "\"registered\":\"false\",\"scale\":1.0,\"metric\":2.0}," - + "{\"user_id\":\"user2\",\"first_name\":\"Mary\",\"last_name\":\"Jones\"," - + "\"emails\":[\"mjones@notareal.com\"],\"top_places\":[\"Orlando, FL\"]," - + "\"todo\":{\"2016-02-03 05:00:00+0000\":\"Get milk and bread\"}," - + "\"registered\":\"true\",\"scale\":3.0,\"metric\":4.0}]}", - new String(files.get(0).toByteArray())); - } - - @Test - public void testProcessorJsonOutputFragmentAttributes() { - processor = new MockQueryCassandraTwoRounds(); - testRunner = TestRunners.newTestRunner(processor); - setUpStandardProcessorConfig(); - testRunner.setIncomingConnection(false); - testRunner.setProperty(QueryCassandra.MAX_ROWS_PER_FLOW_FILE, "1"); - - // Test JSON output - testRunner.setProperty(QueryCassandra.OUTPUT_FORMAT, QueryCassandra.JSON_FORMAT); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_SUCCESS, 2); - List files = testRunner.getFlowFilesForRelationship(QueryCassandra.REL_SUCCESS); - assertNotNull(files); - assertEquals(2, files.size(), "Two files should be transferred to success"); - String indexIdentifier = null; - for (int i = 0; i < files.size(); i++) { - MockFlowFile flowFile = files.get(i); - flowFile.assertAttributeEquals(QueryCassandra.FRAGMENT_INDEX, String.valueOf(i)); - if (indexIdentifier == null) { - indexIdentifier = flowFile.getAttribute(QueryCassandra.FRAGMENT_ID); - } else { - flowFile.assertAttributeEquals(QueryCassandra.FRAGMENT_ID, indexIdentifier); - } - flowFile.assertAttributeEquals(QueryCassandra.FRAGMENT_COUNT, String.valueOf(files.size())); - } - } - - @Test - public void testProcessorELConfigJsonOutput() { - setUpStandardProcessorConfig(); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "${hosts}"); - testRunner.setProperty(QueryCassandra.CQL_SELECT_QUERY, "${query}"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "${pass}"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "${user}"); - testRunner.setProperty(AbstractCassandraProcessor.CHARSET, "${charset}"); - testRunner.setProperty(QueryCassandra.QUERY_TIMEOUT, "${timeout}"); - testRunner.setProperty(QueryCassandra.FETCH_SIZE, "${fetch}"); - testRunner.setProperty(QueryCassandra.MAX_ROWS_PER_FLOW_FILE, "${max-rows-per-flow}"); - testRunner.setIncomingConnection(false); - testRunner.assertValid(); - - testRunner.setEnvironmentVariableValue("hosts", "localhost:9042"); - testRunner.setEnvironmentVariableValue("user", "username"); - testRunner.setEnvironmentVariableValue("pass", "password"); - testRunner.setEnvironmentVariableValue("charset", "UTF-8"); - testRunner.setEnvironmentVariableValue("timeout", "30 sec"); - testRunner.setEnvironmentVariableValue("fetch", "0"); - testRunner.setEnvironmentVariableValue("max-rows-per-flow", "0"); - - // Test JSON output - testRunner.setProperty(QueryCassandra.OUTPUT_FORMAT, QueryCassandra.JSON_FORMAT); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_SUCCESS, 1); - List files = testRunner.getFlowFilesForRelationship(QueryCassandra.REL_SUCCESS); - assertNotNull(files); - assertEquals(1, files.size(), "One file should be transferred to success"); - assertEquals("{\"results\":[{\"user_id\":\"user1\",\"first_name\":\"Joe\",\"last_name\":\"Smith\"," - + "\"emails\":[\"jsmith@notareal.com\"],\"top_places\":[\"New York, NY\",\"Santa Clara, CA\"]," - + "\"todo\":{\"2016-01-03 05:00:00+0000\":\"Set my alarm \\\"for\\\" a month from now\"}," - + "\"registered\":\"false\",\"scale\":1.0,\"metric\":2.0}," - + "{\"user_id\":\"user2\",\"first_name\":\"Mary\",\"last_name\":\"Jones\"," - + "\"emails\":[\"mjones@notareal.com\"],\"top_places\":[\"Orlando, FL\"]," - + "\"todo\":{\"2016-02-03 05:00:00+0000\":\"Get milk and bread\"}," - + "\"registered\":\"true\",\"scale\":3.0,\"metric\":4.0}]}", - new String(files.get(0).toByteArray())); - } - - @Test - public void testProcessorJsonOutputWithQueryTimeout() { - setUpStandardProcessorConfig(); - testRunner.setProperty(QueryCassandra.QUERY_TIMEOUT, "5 sec"); - testRunner.setIncomingConnection(false); - - // Test JSON output - testRunner.setProperty(QueryCassandra.OUTPUT_FORMAT, QueryCassandra.JSON_FORMAT); - testRunner.run(1, true, true); - testRunner.assertAllFlowFilesTransferred(QueryCassandra.REL_SUCCESS, 1); - List files = testRunner.getFlowFilesForRelationship(QueryCassandra.REL_SUCCESS); - assertNotNull(files); - assertEquals(1, files.size(), "One file should be transferred to success"); - } - - @Test - public void testProcessorEmptyFlowFile() { - setUpStandardProcessorConfig(); - - // Run with empty flowfile - testRunner.setIncomingConnection(true); - processor.setExceptionToThrow(null); - testRunner.enqueue("".getBytes()); - testRunner.run(1, true, true); - testRunner.assertTransferCount(QueryCassandra.REL_SUCCESS, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorEmptyFlowFileMaxRowsPerFlowFileEqOne() { - - processor = new MockQueryCassandraTwoRounds(); - testRunner = TestRunners.newTestRunner(processor); - - setUpStandardProcessorConfig(); - - testRunner.setIncomingConnection(true); - testRunner.setProperty(QueryCassandra.MAX_ROWS_PER_FLOW_FILE, "1"); - processor.setExceptionToThrow(null); - testRunner.enqueue("".getBytes()); - testRunner.run(1, true, true); - testRunner.assertTransferCount(QueryCassandra.REL_SUCCESS, 2); - testRunner.clearTransferState(); - } - - - @Test - public void testProcessorEmptyFlowFileAndNoHostAvailableException() { - setUpStandardProcessorConfig(); - - // Test exceptions - processor.setExceptionToThrow(new NoHostAvailableException(new HashMap<>())); - testRunner.enqueue("".getBytes()); - testRunner.run(1, true, true); - testRunner.assertTransferCount(QueryCassandra.REL_RETRY, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorEmptyFlowFileAndInetSocketAddressConsistencyLevelANY() { - setUpStandardProcessorConfig(); - - processor.setExceptionToThrow( - new ReadTimeoutException(new SniEndPoint(new InetSocketAddress("localhost", 9042), ""), ConsistencyLevel.ANY, 0, 1, false)); - testRunner.enqueue("".getBytes()); - testRunner.run(1, true, true); - testRunner.assertTransferCount(QueryCassandra.REL_RETRY, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorEmptyFlowFileAndInetSocketAddressDefault() { - setUpStandardProcessorConfig(); - - processor.setExceptionToThrow( - new InvalidQueryException(new SniEndPoint(new InetSocketAddress("localhost", 9042), ""), "invalid query")); - testRunner.enqueue("".getBytes()); - testRunner.run(1, true, true); - testRunner.assertTransferCount(QueryCassandra.REL_FAILURE, 1); - testRunner.clearTransferState(); - } - - @Test - public void testProcessorEmptyFlowFileAndExceptionsProcessException() { - setUpStandardProcessorConfig(); - - processor.setExceptionToThrow(new ProcessException()); - testRunner.enqueue("".getBytes()); - testRunner.run(1, true, true); - testRunner.assertTransferCount(QueryCassandra.REL_FAILURE, 1); - } - - @Test - public void testCreateSchemaOneColumn() throws Exception { - ResultSet rs = CassandraQueryTestUtil.createMockResultSetOneColumn(); - Schema schema = QueryCassandra.createSchema(rs); - assertNotNull(schema); - assertEquals(schema.getName(), "users"); - } - - @Test - public void testCreateSchema() throws Exception { - ResultSet rs = CassandraQueryTestUtil.createMockResultSet(true); - Schema schema = QueryCassandra.createSchema(rs); - assertNotNull(schema); - assertEquals(Schema.Type.RECORD, schema.getType()); - - // Check record fields, starting with user_id - Schema.Field field = schema.getField("user_id"); - assertNotNull(field); - Schema fieldSchema = field.schema(); - Schema.Type type = fieldSchema.getType(); - assertEquals(Schema.Type.UNION, type); - // Assert individual union types, first is null - assertEquals(Schema.Type.NULL, fieldSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.STRING, fieldSchema.getTypes().get(1).getType()); - - field = schema.getField("first_name"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - assertEquals(Schema.Type.UNION, type); - // Assert individual union types, first is null - assertEquals(Schema.Type.NULL, fieldSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.STRING, fieldSchema.getTypes().get(1).getType()); - - field = schema.getField("last_name"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - assertEquals(Schema.Type.UNION, type); - // Assert individual union types, first is null - assertEquals(Schema.Type.NULL, fieldSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.STRING, fieldSchema.getTypes().get(1).getType()); - - field = schema.getField("emails"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - // Should be a union of null and array - assertEquals(Schema.Type.UNION, type); - assertEquals(Schema.Type.NULL, fieldSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.ARRAY, fieldSchema.getTypes().get(1).getType()); - Schema arraySchema = fieldSchema.getTypes().get(1); - // Assert individual array element types are unions of null and String - Schema elementSchema = arraySchema.getElementType(); - assertEquals(Schema.Type.UNION, elementSchema.getType()); - assertEquals(Schema.Type.NULL, elementSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.STRING, elementSchema.getTypes().get(1).getType()); - - field = schema.getField("top_places"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - // Should be a union of null and array - assertEquals(Schema.Type.UNION, type); - assertEquals(Schema.Type.ARRAY, fieldSchema.getTypes().get(1).getType()); - arraySchema = fieldSchema.getTypes().get(1); - // Assert individual array element types are unions of null and String - elementSchema = arraySchema.getElementType(); - assertEquals(Schema.Type.UNION, elementSchema.getType()); - assertEquals(Schema.Type.NULL, elementSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.STRING, elementSchema.getTypes().get(1).getType()); - - field = schema.getField("todo"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - // Should be a union of null and map - assertEquals(Schema.Type.UNION, type); - assertEquals(Schema.Type.MAP, fieldSchema.getTypes().get(1).getType()); - Schema mapSchema = fieldSchema.getTypes().get(1); - // Assert individual map value types are unions of null and String - Schema valueSchema = mapSchema.getValueType(); - assertEquals(Schema.Type.NULL, valueSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.STRING, valueSchema.getTypes().get(1).getType()); - - field = schema.getField("registered"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - assertEquals(Schema.Type.UNION, type); - // Assert individual union types, first is null - assertEquals(Schema.Type.NULL, fieldSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.BOOLEAN, fieldSchema.getTypes().get(1).getType()); - - field = schema.getField("scale"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - assertEquals(Schema.Type.UNION, type); - // Assert individual union types, first is null - assertEquals(Schema.Type.NULL, fieldSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.FLOAT, fieldSchema.getTypes().get(1).getType()); - - field = schema.getField("metric"); - assertNotNull(field); - fieldSchema = field.schema(); - type = fieldSchema.getType(); - assertEquals(Schema.Type.UNION, type); - // Assert individual union types, first is null - assertEquals(Schema.Type.NULL, fieldSchema.getTypes().get(0).getType()); - assertEquals(Schema.Type.DOUBLE, fieldSchema.getTypes().get(1).getType()); - } - - @Test - public void testConvertToAvroStream() throws Exception { - processor = new MockQueryCassandraTwoRounds(); - testRunner = TestRunners.newTestRunner(processor); - setUpStandardProcessorConfig(); - ResultSet rs = CassandraQueryTestUtil.createMockResultSet(false); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - long numberOfRows = QueryCassandra.convertToAvroStream(rs, 0, baos, 0, null); - assertEquals(2, numberOfRows); - } - - @Test - public void testConvertToJSONStream() throws Exception { - setUpStandardProcessorConfig(); - ResultSet rs = CassandraQueryTestUtil.createMockResultSet(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - long numberOfRows = QueryCassandra.convertToJsonStream(rs, 0, baos, StandardCharsets.UTF_8, - 0, null); - assertEquals(2, numberOfRows); - } - - @Test - public void testDefaultDateFormatInConvertToJSONStream() throws Exception { - ResultSet rs = CassandraQueryTestUtil.createMockDateResultSet(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(QueryCassandra.TIMESTAMP_FORMAT_PATTERN.getDefaultValue()); - - long numberOfRows = QueryCassandra.convertToJsonStream(Optional.of(testRunner.getProcessContext()), rs, 0, baos, - StandardCharsets.UTF_8, 0, null); - assertEquals(1, numberOfRows); - - Map>> map = new ObjectMapper().readValue(baos.toByteArray(), HashMap.class); - String date = map.get("results").get(0).get("date"); - assertEquals(formatter.format(CassandraQueryTestUtil.TEST_DATE.toInstant().atOffset(ZoneOffset.UTC)), date); - } - - @Test - public void testCustomDateFormatInConvertToJSONStream() throws Exception { - MockProcessContext context = (MockProcessContext) testRunner.getProcessContext(); - ResultSet rs = CassandraQueryTestUtil.createMockDateResultSet(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - final String customDateFormat = "yyyy-MM-dd HH:mm:ss.SSSZ"; - context.setProperty(QueryCassandra.TIMESTAMP_FORMAT_PATTERN, customDateFormat); - final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(customDateFormat); - - long numberOfRows = QueryCassandra.convertToJsonStream(Optional.of(context), rs, 0, baos, StandardCharsets.UTF_8, 0, null); - assertEquals(1, numberOfRows); - - Map>> map = new ObjectMapper().readValue(baos.toByteArray(), HashMap.class); - String date = map.get("results").get(0).get("date"); - assertEquals(formatter.format(CassandraQueryTestUtil.TEST_DATE.toInstant().atOffset(ZoneOffset.UTC)), date); - } - - private void setUpStandardProcessorConfig() { - testRunner.setProperty(AbstractCassandraProcessor.CONSISTENCY_LEVEL, "ONE"); - testRunner.setProperty(AbstractCassandraProcessor.CONTACT_POINTS, "localhost:9042"); - testRunner.setProperty(QueryCassandra.CQL_SELECT_QUERY, "select * from test"); - testRunner.setProperty(AbstractCassandraProcessor.PASSWORD, "password"); - testRunner.setProperty(AbstractCassandraProcessor.USERNAME, "username"); - testRunner.setProperty(QueryCassandra.MAX_ROWS_PER_FLOW_FILE, "0"); - } - - /** - * Provides a stubbed processor instance for testing - */ - private static class MockQueryCassandra extends QueryCassandra { - - private Exception exceptionToThrow = null; - - @Override - protected Cluster createCluster(List contactPoints, SSLContext sslContext, - String username, String password, String compressionType) { - Cluster mockCluster = mock(Cluster.class); - try { - Metadata mockMetadata = mock(Metadata.class); - when(mockMetadata.getClusterName()).thenReturn("cluster1"); - when(mockCluster.getMetadata()).thenReturn(mockMetadata); - Session mockSession = mock(Session.class); - when(mockCluster.connect()).thenReturn(mockSession); - when(mockCluster.connect(anyString())).thenReturn(mockSession); - Configuration config = Configuration.builder().build(); - when(mockCluster.getConfiguration()).thenReturn(config); - ResultSetFuture future = mock(ResultSetFuture.class); - ResultSet rs = CassandraQueryTestUtil.createMockResultSet(false); - when(future.getUninterruptibly()).thenReturn(rs); - - try { - doReturn(rs).when(future).getUninterruptibly(anyLong(), any(TimeUnit.class)); - } catch (TimeoutException te) { - throw new IllegalArgumentException("Mocked cluster doesn't time out"); - } - - if (exceptionToThrow != null) { - when(mockSession.execute(anyString(), any(), any())).thenThrow(exceptionToThrow); - when(mockSession.execute(anyString())).thenThrow(exceptionToThrow); - } else { - when(mockSession.execute(anyString(),any(), any())).thenReturn(rs); - when(mockSession.execute(anyString())).thenReturn(rs); - } - } catch (Exception e) { - fail(e.getMessage()); - } - return mockCluster; - } - - public void setExceptionToThrow(Exception e) { - this.exceptionToThrow = e; - } - } - - private static class MockQueryCassandraTwoRounds extends MockQueryCassandra { - - private Exception exceptionToThrow = null; - - @Override - protected Cluster createCluster(List contactPoints, SSLContext sslContext, - String username, String password, String compressionType) { - Cluster mockCluster = mock(Cluster.class); - try { - Metadata mockMetadata = mock(Metadata.class); - when(mockMetadata.getClusterName()).thenReturn("cluster1"); - when(mockCluster.getMetadata()).thenReturn(mockMetadata); - Session mockSession = mock(Session.class); - when(mockCluster.connect()).thenReturn(mockSession); - when(mockCluster.connect(anyString())).thenReturn(mockSession); - Configuration config = Configuration.builder().build(); - when(mockCluster.getConfiguration()).thenReturn(config); - ResultSetFuture future = mock(ResultSetFuture.class); - ResultSet rs = CassandraQueryTestUtil.createMockResultSet(true); - when(future.getUninterruptibly()).thenReturn(rs); - - try { - doReturn(rs).when(future).getUninterruptibly(anyLong(), any(TimeUnit.class)); - } catch (TimeoutException te) { - throw new IllegalArgumentException("Mocked cluster doesn't time out"); - } - - if (exceptionToThrow != null) { - when(mockSession.execute(anyString(), any(), any())).thenThrow(exceptionToThrow); - when(mockSession.execute(anyString())).thenThrow(exceptionToThrow); - } else { - when(mockSession.execute(anyString(),any(), any())).thenReturn(rs); - when(mockSession.execute(anyString())).thenReturn(rs); - } - } catch (Exception e) { - fail(e.getMessage()); - } - return mockCluster; - } - - public void setExceptionToThrow(Exception e) { - this.exceptionToThrow = e; - } - } - -} - diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/pom.xml deleted file mode 100644 index 4195bfe32fd7..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - nifi-cassandra-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-cassandra-services-api-nar - nar - - - - org.apache.nifi - nifi-standard-shared-nar - nar - - - org.apache.nifi - nifi-cassandra-services-api - 2.0.0-SNAPSHOT - compile - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/LICENSE deleted file mode 100644 index 106720e0f851..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,266 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -APACHE NIFI SUBCOMPONENTS: - -The Apache NiFi project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -This product bundles 'asm' which is available under a 3-Clause BSD style license. -For details see http://asm.ow2.org/asmdex-license.html - - Copyright (c) 2012 France Télécom - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles 'JNR x86asm' under an MIT -style license. - - Copyright (C) 2010 Wayne Meissner - Copyright (c) 2008-2009, Petr Kobalicek - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 2d9d2aa14743..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,227 +0,0 @@ -nifi-cassandra-services-api-nar -Copyright 2016-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -****************** -Apache Software License v2 -****************** - - (ASLv2) The Netty Project - The following NOTICE information applies: - Copyright 2014 The Netty Project - ------------------------------------------------------------------------------- - This product contains the extensions to Java Collections Framework which has - been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - - This product contains a modified version of Robert Harder's Public Domain - Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - - This product contains a modified portion of 'Webbit', an event based - WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - - This product contains a modified portion of 'SLF4J', a simple logging - facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - - This product contains a modified portion of 'Apache Harmony', an open source - Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - - This product contains a modified portion of 'jbzip2', a Java bzip2 compression - and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - - This product contains a modified portion of 'libdivsufsort', a C API library to construct - the suffix array and the Burrows-Wheeler transformed string for any input string of - a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - - This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - - This product optionally depends on 'JZlib', a re-implementation of zlib in - pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - - This product optionally depends on 'Compress-LZF', a Java library for encoding and - decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - - This product optionally depends on 'lz4', a LZ4 Java compression - and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - - This product optionally depends on 'lzma-java', a LZMA Java compression - and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - - This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression - and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - - This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data - interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - - This product optionally depends on 'Bouncy Castle Crypto APIs' to generate - a temporary self-signed X.509 certificate when the JVM does not provide the - equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - - This product optionally depends on 'Snappy', a compression library produced - by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - - This product optionally depends on 'JBoss Marshalling', an alternative Java - serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - - This product optionally depends on 'Caliper', Google's micro- - benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - - This product optionally depends on 'Apache Log4J', a logging framework, which - can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - - This product optionally depends on 'Aalto XML', an ultra-high performance - non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - - This product contains a modified version of 'HPACK', a Java implementation of - the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - - This product contains a modified portion of 'Apache Commons Lang', a Java library - provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - This product contains a forked and modified version of Tomcat Native - - * LICENSE: - * ASL2 - * HOMEPAGE: - * http://tomcat.apache.org/native-doc/ - * https://svn.apache.org/repos/asf/tomcat/native/ - - (ASLv2) Guava - The following NOTICE information applies: - Guava - Copyright 2015 The Guava Authors - - (ASLv2) Dropwizard Metrics - The following NOTICE information applies: - Copyright (c) 2010-2013 Coda Hale, Yammer.com - - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - - ************************ - Eclipse Public License 1.0 - ************************ - - The following binary components are provided under the Eclipse Public License 1.0. See project link for details. - - (EPL 2.0)(GPL 2)(LGPL 2.1) JNR Posix ( jnr.posix ) https://github.com/jnr/jnr-posix/blob/master/LICENSE.txt - diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/pom.xml deleted file mode 100644 index 0ad093bbc757..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - nifi-cassandra-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-cassandra-services-api - jar - - - - org.apache.nifi - nifi-api - provided - - - - com.datastax.cassandra - cassandra-driver-core - ${cassandra.sdk.version} - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/src/main/java/org/apache/nifi/cassandra/CassandraSessionProviderService.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/src/main/java/org/apache/nifi/cassandra/CassandraSessionProviderService.java deleted file mode 100644 index fc2d22204af8..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-api/src/main/java/org/apache/nifi/cassandra/CassandraSessionProviderService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.cassandra; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.Session; -import org.apache.nifi.controller.ControllerService; - -public interface CassandraSessionProviderService extends ControllerService { - /** - * Obtains a Cassandra session instance - * @return {@link Session} - */ - Session getCassandraSession(); - - /** - * Obtains a Cassandra cluster instance - * @return {@link Cluster} - */ - Cluster getCluster(); -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/pom.xml deleted file mode 100644 index 3957bb752cfa..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - nifi-cassandra-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-cassandra-services-nar - nar - - - - - - com.google.guava - guava - provided - - - - - - - org.apache.nifi - nifi-cassandra-services-api-nar - 2.0.0-SNAPSHOT - nar - - - org.apache.nifi - nifi-cassandra-services - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-cassandra-distributedmapcache-service - 2.0.0-SNAPSHOT - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/LICENSE deleted file mode 100644 index 04416de4c91f..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,352 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -APACHE NIFI SUBCOMPONENTS: - -The Apache NiFi project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -This product bundles 'libffi' which is available under an MIT style license. - libffi - Copyright (c) 1996-2014 Anthony Green, Red Hat, Inc and others. - see https://github.com/java-native-access/jna/blob/master/native/libffi/LICENSE - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -This product bundles 'asm' which is available under a 3-Clause BSD style license. -For details see http://asm.ow2.org/asmdex-license.html - - Copyright (c) 2012 France Télécom - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. - - The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' - under an MIT style license. - - Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - -The binary distribution of this product bundles 'JNR x86asm' under an MIT -style license. - - Copyright (C) 2010 Wayne Meissner - Copyright (c) 2008-2009, Petr Kobalicek - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - -This product bundles 'logback' which is dual-licensed under the EPL v1.0 -and the LGPL 2.1. - - Logback: the reliable, generic, fast and flexible logging framework. - - Copyright (C) 1999-2017, QOS.ch. All rights reserved. - - This program and the accompanying materials are dual-licensed under - either the terms of the Eclipse Public License v1.0 as published by - the Eclipse Foundation or (per the licensee's choosing) under the - terms of the GNU Lesser General Public License version 2.1 as - published by the Free Software Foundation. - -The binary distribution of this product bundles 'ANTLR 3' which is available -under a "3-clause BSD" license. For details see http://www.antlr.org/license.html - - Copyright (c) 2012 Terence Parr and Sam Harwell - All rights reserved. - Redistribution and use in source and binary forms, with or without modification, are permitted - provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, this list of - conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of - conditions and the following disclaimer in the documentation and/or other materials - provided with the distribution. - - Neither the name of the author nor the names of its contributors may be used to endorse - or promote products derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF - THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index f2261add8685..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,292 +0,0 @@ -nifi-cassandra-services-nar -Copyright 2016-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -****************** -Apache Software License v2 -****************** - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) DataStax Java Driver for Apache Cassandra - Core - The following NOTICE information applies: - DataStax Java Driver for Apache Cassandra - Core - Copyright (C) 2012-2017 DataStax Inc. - - (ASLv2) Jackson JSON processor - The following NOTICE information applies: - # Jackson JSON processor - - Jackson is a high-performance, Free/Open Source JSON processing library. - It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has - been in development since 2007. - It is currently developed by a community of developers, as well as supported - commercially by FasterXML.com. - - ## Licensing - - Jackson core and extension components may licensed under different licenses. - To find the details that apply to this artifact see the accompanying LICENSE file. - For more information, including possible other licensing options, contact - FasterXML.com (http://fasterxml.com). - - ## Credits - - A list of contributors may be found from CREDITS file, which is included - in some artifacts (usually source distributions); but is always available - from the source code management (SCM) system project uses. - - (ASLv2) Apache Commons Codec - The following NOTICE information applies: - Apache Commons Codec - Copyright 2002-2014 The Apache Software Foundation - - src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java - contains test data from http://aspell.net/test/orig/batch0.tab. - Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - - =============================================================================== - - The content of package org.apache.commons.codec.language.bm has been translated - from the original php source code available at http://stevemorse.org/phoneticinfo.htm - with permission from the original authors. - Original source copyright: - Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - - (ASLv2) Apache Commons Lang - The following NOTICE information applies: - Apache Commons Lang - Copyright 2001-2017 The Apache Software Foundation - - This product includes software from the Spring Framework, - under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - - (ASLv2) Guava - The following NOTICE information applies: - Guava - Copyright 2015 The Guava Authors - - (ASLv2) JSON-SMART - The following NOTICE information applies: - Copyright 2011 JSON-SMART authors - - (ASLv2) Dropwizard Metrics - The following NOTICE information applies: - Copyright (c) 2010-2013 Coda Hale, Yammer.com - - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - - (ASLv2) The Netty Project - The following NOTICE information applies: - Copyright 2014 The Netty Project - ------------------------------------------------------------------------------- - This product contains the extensions to Java Collections Framework which has - been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - - This product contains a modified version of Robert Harder's Public Domain - Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - - This product contains a modified portion of 'Webbit', an event based - WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - - This product contains a modified portion of 'SLF4J', a simple logging - facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - - This product contains a modified portion of 'Apache Harmony', an open source - Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - - This product contains a modified portion of 'jbzip2', a Java bzip2 compression - and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - - This product contains a modified portion of 'libdivsufsort', a C API library to construct - the suffix array and the Burrows-Wheeler transformed string for any input string of - a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - - This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - - This product optionally depends on 'JZlib', a re-implementation of zlib in - pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - - This product optionally depends on 'Compress-LZF', a Java library for encoding and - decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - - This product optionally depends on 'lz4', a LZ4 Java compression - and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - - This product optionally depends on 'lzma-java', a LZMA Java compression - and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - - This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression - and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - - This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data - interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - - This product optionally depends on 'Bouncy Castle Crypto APIs' to generate - a temporary self-signed X.509 certificate when the JVM does not provide the - equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - - This product optionally depends on 'Snappy', a compression library produced - by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - - This product optionally depends on 'JBoss Marshalling', an alternative Java - serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - - This product optionally depends on 'Caliper', Google's micro- - benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - - This product optionally depends on 'Apache Log4J', a logging framework, which - can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - - This product optionally depends on 'Aalto XML', an ultra-high performance - non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - - This product contains a modified version of 'HPACK', a Java implementation of - the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - - This product contains a modified portion of 'Apache Commons Lang', a Java library - provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - This product contains a forked and modified version of Tomcat Native - - * LICENSE: - * ASL2 - * HOMEPAGE: - * http://tomcat.apache.org/native-doc/ - * https://svn.apache.org/repos/asf/tomcat/native/ - - (ASLv2) Objenesis - The following NOTICE information applies: - Objenesis - Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita - -************************ -Eclipse Public License 1.0 -************************ - -The following binary components are provided under the Eclipse Public License 1.0. See project link for details. - - (EPL 2.0)(GPL 2)(LGPL 2.1) JNR Posix ( jnr.posix ) https://github.com/jnr/jnr-posix/blob/master/LICENSE.txt - (EPL 1.0)(LGPL 2.1) Logback Classic (ch.qos.logback:logback-classic:jar:1.2.6 - http://logback.qos.ch/) - (EPL 1.0)(LGPL 2.1) Logback Core (ch.qos.logback:logback-core:jar:1.2.6 - http://logback.qos.ch/) diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/pom.xml deleted file mode 100644 index f2029682d188..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/pom.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - nifi-cassandra-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-cassandra-services - jar - - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-utils - - - org.apache.nifi - nifi-cassandra-services-api - provided - - - - com.datastax.cassandra - cassandra-driver-core - ${cassandra.sdk.version} - provided - - - - org.apache.nifi - nifi-ssl-context-service-api - - - org.apache.nifi - nifi-framework-api - - - org.apache.nifi - nifi-mock - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/java/org/apache/nifi/service/CassandraSessionProvider.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/java/org/apache/nifi/service/CassandraSessionProvider.java deleted file mode 100644 index 9a394539f570..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/java/org/apache/nifi/service/CassandraSessionProvider.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.service; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.ConsistencyLevel; -import com.datastax.driver.core.Metadata; -import com.datastax.driver.core.ProtocolOptions; -import com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions; -import com.datastax.driver.core.SSLOptions; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.SocketOptions; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; -import javax.net.ssl.SSLContext; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnDisabled; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.cassandra.CassandraSessionProviderService; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.controller.AbstractControllerService; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.security.util.ClientAuth; -import org.apache.nifi.ssl.SSLContextService; - -@Tags({"cassandra", "dbcp", "database", "connection", "pooling"}) -@CapabilityDescription("Provides connection session for Cassandra processors to work with Apache Cassandra.") -public class CassandraSessionProvider extends AbstractControllerService implements CassandraSessionProviderService { - - public static final int DEFAULT_CASSANDRA_PORT = 9042; - - // Common descriptors - public static final PropertyDescriptor CONTACT_POINTS = new PropertyDescriptor.Builder() - .name("Cassandra Contact Points") - .description("Contact points are addresses of Cassandra nodes. The list of contact points should be " - + "comma-separated and in hostname:port format. Example node1:port,node2:port,...." - + " The default client port for Cassandra is 9042, but the port(s) must be explicitly specified.") - .required(true) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.HOSTNAME_PORT_LIST_VALIDATOR) - .build(); - - public static final PropertyDescriptor KEYSPACE = new PropertyDescriptor.Builder() - .name("Keyspace") - .description("The Cassandra Keyspace to connect to. If no keyspace is specified, the query will need to " + - "include the keyspace name before any table reference, in case of 'query' native processors or " + - "if the processor supports the 'Table' property, the keyspace name has to be provided with the " + - "table name in the form of .
") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() - .name("SSL Context Service") - .description("The SSL Context Service used to provide client certificate information for TLS/SSL " - + "connections.") - .required(false) - .identifiesControllerService(SSLContextService.class) - .build(); - - public static final PropertyDescriptor CLIENT_AUTH = new PropertyDescriptor.Builder() - .name("Client Auth") - .description("Client authentication policy when connecting to secure (TLS/SSL) cluster. " - + "Possible values are REQUIRED, WANT, NONE. This property is only used when an SSL Context " - + "has been defined and enabled.") - .required(false) - .allowableValues(ClientAuth.values()) - .defaultValue("REQUIRED") - .build(); - - public static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder() - .name("Username") - .description("Username to access the Cassandra cluster") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder() - .name("Password") - .description("Password to access the Cassandra cluster") - .required(false) - .sensitive(true) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - public static final PropertyDescriptor CONSISTENCY_LEVEL = new PropertyDescriptor.Builder() - .name("Consistency Level") - .description("The strategy for how many replicas must respond before results are returned.") - .required(true) - .allowableValues(ConsistencyLevel.values()) - .defaultValue("ONE") - .build(); - - static final PropertyDescriptor COMPRESSION_TYPE = new PropertyDescriptor.Builder() - .name("Compression Type") - .description("Enable compression at transport-level requests and responses") - .required(false) - .allowableValues(ProtocolOptions.Compression.values()) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .defaultValue("NONE") - .build(); - - static final PropertyDescriptor READ_TIMEOUT_MS = new PropertyDescriptor.Builder() - .name("read-timeout-ms") - .displayName("Read Timout (ms)") - .description("Read timeout (in milliseconds). 0 means no timeout. If no value is set, the underlying default will be used.") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) - .build(); - - static final PropertyDescriptor CONNECT_TIMEOUT_MS = new PropertyDescriptor.Builder() - .name("connect-timeout-ms") - .displayName("Connect Timeout (ms)") - .description("Connection timeout (in milliseconds). 0 means no timeout. If no value is set, the underlying default will be used.") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) - .build(); - - private List properties; - private Cluster cluster; - private Session cassandraSession; - - @Override - public void init(final ControllerServiceInitializationContext context) { - List props = new ArrayList<>(); - - props.add(CONTACT_POINTS); - props.add(CLIENT_AUTH); - props.add(CONSISTENCY_LEVEL); - props.add(COMPRESSION_TYPE); - props.add(KEYSPACE); - props.add(USERNAME); - props.add(PASSWORD); - props.add(PROP_SSL_CONTEXT_SERVICE); - props.add(READ_TIMEOUT_MS); - props.add(CONNECT_TIMEOUT_MS); - - properties = props; - } - - @Override - public List getSupportedPropertyDescriptors() { - return properties; - } - - @OnEnabled - public void onEnabled(final ConfigurationContext context) { - connectToCassandra(context); - } - - @OnDisabled - public void onDisabled(){ - if (cassandraSession != null) { - cassandraSession.close(); - cassandraSession = null; - } - if (cluster != null) { - cluster.close(); - cluster = null; - } - } - - @Override - public Cluster getCluster() { - if (cluster != null) { - return cluster; - } else { - throw new ProcessException("Unable to get the Cassandra cluster detail."); - } - } - - @Override - public Session getCassandraSession() { - if (cassandraSession != null) { - return cassandraSession; - } else { - throw new ProcessException("Unable to get the Cassandra session."); - } - } - - private void connectToCassandra(ConfigurationContext context) { - if (cluster == null) { - ComponentLog log = getLogger(); - final String contactPointList = context.getProperty(CONTACT_POINTS).evaluateAttributeExpressions().getValue(); - final String consistencyLevel = context.getProperty(CONSISTENCY_LEVEL).getValue(); - final String compressionType = context.getProperty(COMPRESSION_TYPE).getValue(); - - List contactPoints = getContactPoints(contactPointList); - - // Set up the client for secure (SSL/TLS communications) if configured to do so - final SSLContextService sslService = - context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); - final SSLContext sslContext; - - if (sslService == null) { - sslContext = null; - } else { - sslContext = sslService.createContext();; - } - - final String username, password; - PropertyValue usernameProperty = context.getProperty(USERNAME).evaluateAttributeExpressions(); - PropertyValue passwordProperty = context.getProperty(PASSWORD).evaluateAttributeExpressions(); - - if (usernameProperty != null && passwordProperty != null) { - username = usernameProperty.getValue(); - password = passwordProperty.getValue(); - } else { - username = null; - password = null; - } - - final Integer readTimeoutMillis = context.getProperty(READ_TIMEOUT_MS).evaluateAttributeExpressions().asInteger(); - final Integer connectTimeoutMillis = context.getProperty(CONNECT_TIMEOUT_MS).evaluateAttributeExpressions().asInteger(); - - // Create the cluster and connect to it - Cluster newCluster = createCluster(contactPoints, sslContext, username, password, compressionType, readTimeoutMillis, connectTimeoutMillis); - PropertyValue keyspaceProperty = context.getProperty(KEYSPACE).evaluateAttributeExpressions(); - final Session newSession; - if (keyspaceProperty != null) { - newSession = newCluster.connect(keyspaceProperty.getValue()); - } else { - newSession = newCluster.connect(); - } - newCluster.getConfiguration().getQueryOptions().setConsistencyLevel(ConsistencyLevel.valueOf(consistencyLevel)); - Metadata metadata = newCluster.getMetadata(); - log.info("Connected to Cassandra cluster: {}", new Object[]{metadata.getClusterName()}); - - cluster = newCluster; - cassandraSession = newSession; - } - } - - private List getContactPoints(String contactPointList) { - - if (contactPointList == null) { - return null; - } - - final String[] contactPointStringList = contactPointList.split(","); - List contactPoints = new ArrayList<>(); - - for (String contactPointEntry : contactPointStringList) { - String[] addresses = contactPointEntry.split(":"); - final String hostName = addresses[0].trim(); - final int port = (addresses.length > 1) ? Integer.parseInt(addresses[1].trim()) : DEFAULT_CASSANDRA_PORT; - - contactPoints.add(new InetSocketAddress(hostName, port)); - } - - return contactPoints; - } - - private Cluster createCluster(final List contactPoints, final SSLContext sslContext, - final String username, final String password, final String compressionType, - final Integer readTimeoutMillis, final Integer connectTimeoutMillis) { - - Cluster.Builder builder = Cluster.builder().addContactPointsWithPorts(contactPoints); - if (sslContext != null) { - final SSLOptions sslOptions = RemoteEndpointAwareJdkSSLOptions.builder() - .withSSLContext(sslContext) - .build(); - builder = builder.withSSL(sslOptions); - } - - if (username != null && password != null) { - builder = builder.withCredentials(username, password); - } - - if (ProtocolOptions.Compression.SNAPPY.name().equals(compressionType)) { - builder = builder.withCompression(ProtocolOptions.Compression.SNAPPY); - } else if (ProtocolOptions.Compression.LZ4.name().equals(compressionType)) { - builder = builder.withCompression(ProtocolOptions.Compression.LZ4); - } - - SocketOptions socketOptions = new SocketOptions(); - if (readTimeoutMillis != null) { - socketOptions.setReadTimeoutMillis(readTimeoutMillis); - } - if (connectTimeoutMillis != null) { - socketOptions.setConnectTimeoutMillis(connectTimeoutMillis); - } - - builder.withSocketOptions(socketOptions); - - return builder.build(); - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService deleted file mode 100644 index 045f90625fc1..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.nifi.service.CassandraSessionProvider \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/MockCassandraProcessor.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/MockCassandraProcessor.java deleted file mode 100644 index 4891fe5a07a2..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/MockCassandraProcessor.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.service; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; - -import java.util.Collections; -import java.util.List; - -/** - * Mock Cassandra processor for testing CassandraSessionProvider - */ -public class MockCassandraProcessor extends AbstractProcessor{ - private static PropertyDescriptor CASSANDRA_SESSION_PROVIDER = new PropertyDescriptor.Builder() - .name("cassandra-session-provider") - .displayName("Cassandra Session Provider") - .required(true) - .description("Controller Service to obtain a Cassandra connection session") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .identifiesControllerService(CassandraSessionProvider.class) - .build(); - - @Override - public List getSupportedPropertyDescriptors() { - return Collections.singletonList(CASSANDRA_SESSION_PROVIDER); - } - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - - } -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/TestCassandraSessionProvider.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/TestCassandraSessionProvider.java deleted file mode 100644 index d6dbb5179992..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-services/src/test/java/org/apache/nifi/service/TestCassandraSessionProvider.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.service; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestCassandraSessionProvider { - - private static TestRunner runner; - private static CassandraSessionProvider sessionProvider; - - @BeforeAll - public static void setup() throws InitializationException { - MockCassandraProcessor mockCassandraProcessor = new MockCassandraProcessor(); - sessionProvider = new CassandraSessionProvider(); - - runner = TestRunners.newTestRunner(mockCassandraProcessor); - runner.addControllerService("cassandra-session-provider", sessionProvider); - } - - @Test - public void testGetPropertyDescriptors() { - List properties = sessionProvider.getPropertyDescriptors(); - - assertEquals(10, properties.size()); - assertTrue(properties.contains(CassandraSessionProvider.CLIENT_AUTH)); - assertTrue(properties.contains(CassandraSessionProvider.CONSISTENCY_LEVEL)); - assertTrue(properties.contains(CassandraSessionProvider.CONTACT_POINTS)); - assertTrue(properties.contains(CassandraSessionProvider.KEYSPACE)); - assertTrue(properties.contains(CassandraSessionProvider.PASSWORD)); - assertTrue(properties.contains(CassandraSessionProvider.PROP_SSL_CONTEXT_SERVICE)); - assertTrue(properties.contains(CassandraSessionProvider.USERNAME)); - } - -} diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/pom.xml b/nifi-nar-bundles/nifi-cassandra-bundle/pom.xml deleted file mode 100644 index 0107c8de154e..000000000000 --- a/nifi-nar-bundles/nifi-cassandra-bundle/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - - - nifi-standard-shared-bom - org.apache.nifi - 2.0.0-SNAPSHOT - ../nifi-standard-shared-bundle/nifi-standard-shared-bom - - - - 3.11.5 - 19.0 - - - nifi-cassandra-bundle - pom - - - nifi-cassandra-processors - nifi-cassandra-nar - nifi-cassandra-distributedmapcache-service - nifi-cassandra-services-api - nifi-cassandra-services-api-nar - nifi-cassandra-services - nifi-cassandra-services-nar - - - - - - org.apache.nifi - nifi-cassandra-processors - 2.0.0-SNAPSHOT - - - com.google.guava - guava - ${cassandra.guava.version} - - - - diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml index 3149499600b0..29fce6292f89 100755 --- a/nifi-nar-bundles/pom.xml +++ b/nifi-nar-bundles/pom.xml @@ -61,7 +61,6 @@ nifi-splunk-bundle nifi-jms-bundle nifi-beats-bundle - nifi-cassandra-bundle nifi-hive-bundle nifi-site-to-site-reporting-bundle nifi-mqtt-bundle @@ -231,11 +230,6 @@ nifi-http-context-map 2.0.0-SNAPSHOT - - org.apache.nifi - nifi-cassandra-services-api - 2.0.0-SNAPSHOT - org.apache.nifi nifi-volatile-provenance-repository From d370b470b8953ce6f5c1edb6a0623d21c08512ed Mon Sep 17 00:00:00 2001 From: Peter Turcsanyi Date: Mon, 12 Feb 2024 23:38:46 +0100 Subject: [PATCH 59/73] NIFI-12782 Migrated GCS processors' Proxy properties to ProxyConfigurationService Extracted proxy service migration code into a common util module because the same logic was already used in AWS module, and it is also reusable in other components for proxy property migration. Signed-off-by: Pierre Villard This closes #8400. --- .../nifi-aws-abstract-processors/pom.xml | 5 ++ ...stractAWSCredentialsProviderProcessor.java | 32 +------ .../aws/v2/AbstractAwsProcessor.java | 32 +------ .../nifi-migration-utils/pom.xml | 39 +++++++++ .../nifi/migration/ProxyServiceMigration.java | 69 +++++++++++++++ .../migration/ProxyServiceMigrationTest.java | 84 +++++++++++++++++++ nifi-nar-bundles/nifi-extension-utils/pom.xml | 5 +- .../nifi-gcp-processors/pom.xml | 5 ++ .../processors/gcp/AbstractGCPProcessor.java | 84 ++++--------------- .../AbstractGCPubSubWithProxyProcessor.java | 20 +---- 10 files changed, 229 insertions(+), 146 deletions(-) create mode 100644 nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/pom.xml create mode 100644 nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/main/java/org/apache/nifi/migration/ProxyServiceMigration.java create mode 100644 nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/test/java/org/apache/nifi/migration/ProxyServiceMigrationTest.java diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/pom.xml index fcfc8e709636..34cd7e9c0f9b 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/pom.xml +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/pom.xml @@ -142,6 +142,11 @@ org.apache.nifi nifi-proxy-configuration-api + + org.apache.nifi + nifi-migration-utils + 2.0.0-SNAPSHOT + com.github.ben-manes.caffeine caffeine diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSCredentialsProviderProcessor.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSCredentialsProviderProcessor.java index 5215186e366c..e3fdefde9f73 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSCredentialsProviderProcessor.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/AbstractAWSCredentialsProviderProcessor.java @@ -37,6 +37,7 @@ import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.migration.PropertyConfiguration; +import org.apache.nifi.migration.ProxyServiceMigration; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.Relationship; @@ -49,9 +50,7 @@ import javax.net.ssl.SSLContext; import java.net.Proxy; -import java.net.Proxy.Type; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -69,7 +68,6 @@ public abstract class AbstractAWSCredentialsProviderProcessor extends AbstractProcessor implements VerifiableProcessor { private static final String CREDENTIALS_SERVICE_CLASSNAME = "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService"; - private static final String PROXY_SERVICE_CLASSNAME = "org.apache.nifi.proxy.StandardProxyConfigurationService"; // Obsolete property names private static final String OBSOLETE_ACCESS_KEY = "Access Key"; @@ -85,11 +83,6 @@ public abstract class AbstractAWSCredentialsProviderProcessor proxyProperties = new HashMap<>(); - proxyProperties.put(PROXY_SERVICE_TYPE, Type.HTTP.name()); - proxyProperties.put(PROXY_SERVICE_HOST, config.getRawPropertyValue(OBSOLETE_PROXY_HOST).get()); - - // Map any optional proxy configs - config.getRawPropertyValue(OBSOLETE_PROXY_PORT).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_PORT, value)); - config.getRawPropertyValue(OBSOLETE_PROXY_USERNAME).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_USERNAME, value)); - config.getRawPropertyValue(OBSOLETE_PROXY_PASSWORD).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_PASSWORD, value)); - - final String serviceId = config.createControllerService(PROXY_SERVICE_CLASSNAME, proxyProperties); - config.setProperty(PROXY_CONFIGURATION_SERVICE, serviceId); - } - - config.removeProperty(OBSOLETE_PROXY_HOST); - config.removeProperty(OBSOLETE_PROXY_PORT); - config.removeProperty(OBSOLETE_PROXY_USERNAME); - config.removeProperty(OBSOLETE_PROXY_PASSWORD); - } - protected ClientConfiguration createConfiguration(final ProcessContext context) { return createConfiguration(context, context.getMaxConcurrentTasks()); } diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/v2/AbstractAwsProcessor.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/v2/AbstractAwsProcessor.java index 7a0b6d9bda8d..156aa916d6fb 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/v2/AbstractAwsProcessor.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/java/org/apache/nifi/processors/aws/v2/AbstractAwsProcessor.java @@ -27,6 +27,7 @@ import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.migration.PropertyConfiguration; +import org.apache.nifi.migration.ProxyServiceMigration; import org.apache.nifi.processor.AbstractSessionFactoryProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -51,12 +52,10 @@ import javax.net.ssl.TrustManager; import java.net.Proxy; -import java.net.Proxy.Type; import java.net.URI; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -71,7 +70,6 @@ */ public abstract class AbstractAwsProcessor extends AbstractSessionFactoryProcessor implements VerifiableProcessor { private static final String CREDENTIALS_SERVICE_CLASSNAME = "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService"; - private static final String PROXY_SERVICE_CLASSNAME = "org.apache.nifi.proxy.StandardProxyConfigurationService"; // Obsolete property names private static final String OBSOLETE_ACCESS_KEY = "Access Key"; @@ -87,11 +85,6 @@ public abstract class AbstractAwsProcessor extends Abstract private static final String AUTH_SERVICE_SECRET_KEY = "Secret Key"; private static final String AUTH_SERVICE_CREDENTIALS_FILE = "Credentials File"; private static final String AUTH_SERVICE_ANONYMOUS_CREDENTIALS = "anonymous-credentials"; - private static final String PROXY_SERVICE_HOST = "proxy-server-host"; - private static final String PROXY_SERVICE_PORT = "proxy-server-port"; - private static final String PROXY_SERVICE_USERNAME = "proxy-user-name"; - private static final String PROXY_SERVICE_PASSWORD = "proxy-user-password"; - private static final String PROXY_SERVICE_TYPE = "proxy-type"; public static final Relationship REL_SUCCESS = new Relationship.Builder() @@ -196,7 +189,7 @@ public Set getRelationships() { @Override public void migrateProperties(final PropertyConfiguration config) { migrateAuthenticationProperties(config); - migrateProxyProperties(config); + ProxyServiceMigration.migrateProxyProperties(config, PROXY_CONFIGURATION_SERVICE, OBSOLETE_PROXY_HOST, OBSOLETE_PROXY_PORT, OBSOLETE_PROXY_USERNAME, OBSOLETE_PROXY_PASSWORD); } private void migrateAuthenticationProperties(final PropertyConfiguration config) { @@ -224,27 +217,6 @@ private void migrateAuthenticationProperties(final PropertyConfiguration config) config.removeProperty(OBSOLETE_CREDENTIALS_FILE); } - private void migrateProxyProperties(final PropertyConfiguration config) { - if (config.isPropertySet(OBSOLETE_PROXY_HOST)) { - final Map proxyProperties = new HashMap<>(); - proxyProperties.put(PROXY_SERVICE_TYPE, Type.HTTP.name()); - proxyProperties.put(PROXY_SERVICE_HOST, config.getRawPropertyValue(OBSOLETE_PROXY_HOST).get()); - - // Map any optional proxy configs - config.getRawPropertyValue(OBSOLETE_PROXY_PORT).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_PORT, value)); - config.getRawPropertyValue(OBSOLETE_PROXY_USERNAME).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_USERNAME, value)); - config.getRawPropertyValue(OBSOLETE_PROXY_PASSWORD).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_PASSWORD, value)); - - final String serviceId = config.createControllerService(PROXY_SERVICE_CLASSNAME, proxyProperties); - config.setProperty(PROXY_CONFIGURATION_SERVICE, serviceId); - } - - config.removeProperty(OBSOLETE_PROXY_HOST); - config.removeProperty(OBSOLETE_PROXY_PORT); - config.removeProperty(OBSOLETE_PROXY_USERNAME); - config.removeProperty(OBSOLETE_PROXY_PASSWORD); - } - @OnScheduled public void onScheduled(final ProcessContext context) { getClient(context); diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/pom.xml new file mode 100644 index 000000000000..16779c53de94 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/pom.xml @@ -0,0 +1,39 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-extension-utils + 2.0.0-SNAPSHOT + + + nifi-migration-utils + + + + org.apache.nifi + nifi-api + + + + org.apache.nifi + nifi-mock + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/main/java/org/apache/nifi/migration/ProxyServiceMigration.java b/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/main/java/org/apache/nifi/migration/ProxyServiceMigration.java new file mode 100644 index 000000000000..35dec2ee4165 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/main/java/org/apache/nifi/migration/ProxyServiceMigration.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.migration; + +import org.apache.nifi.components.PropertyDescriptor; + +import java.net.Proxy; +import java.util.HashMap; +import java.util.Map; + +public final class ProxyServiceMigration { + + static final String PROXY_SERVICE_CLASSNAME = "org.apache.nifi.proxy.StandardProxyConfigurationService"; + + static final String PROXY_SERVICE_TYPE = "proxy-type"; + static final String PROXY_SERVICE_HOST = "proxy-server-host"; + static final String PROXY_SERVICE_PORT = "proxy-server-port"; + static final String PROXY_SERVICE_USERNAME = "proxy-user-name"; + static final String PROXY_SERVICE_PASSWORD = "proxy-user-password"; + + private ProxyServiceMigration() {} + + /** + * Migrates component level proxy properties to ProxyConfigurationService. + * + * @param config the component's property config to be migrated + * @param proxyServiceProperty the component's property descriptor referencing ProxyConfigurationService + * @param proxyHostProperty the name of the component level Proxy Host property + * @param proxyPortProperty the name of the component level Proxy Port property + * @param proxyUsernameProperty the name of the component level Proxy Username property + * @param proxyPasswordProperty the name of the component level Proxy Password property + */ + public static void migrateProxyProperties(final PropertyConfiguration config, final PropertyDescriptor proxyServiceProperty, + final String proxyHostProperty, final String proxyPortProperty, + final String proxyUsernameProperty, final String proxyPasswordProperty) { + if (config.isPropertySet(proxyHostProperty)) { + final Map proxyProperties = new HashMap<>(); + proxyProperties.put(PROXY_SERVICE_TYPE, Proxy.Type.HTTP.name()); + proxyProperties.put(PROXY_SERVICE_HOST, config.getRawPropertyValue(proxyHostProperty).get()); + + // Map any optional proxy configs + config.getRawPropertyValue(proxyPortProperty).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_PORT, value)); + config.getRawPropertyValue(proxyUsernameProperty).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_USERNAME, value)); + config.getRawPropertyValue(proxyPasswordProperty).ifPresent(value -> proxyProperties.put(PROXY_SERVICE_PASSWORD, value)); + + final String serviceId = config.createControllerService(PROXY_SERVICE_CLASSNAME, proxyProperties); + config.setProperty(proxyServiceProperty, serviceId); + } + + config.removeProperty(proxyHostProperty); + config.removeProperty(proxyPortProperty); + config.removeProperty(proxyUsernameProperty); + config.removeProperty(proxyPasswordProperty); + } +} diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/test/java/org/apache/nifi/migration/ProxyServiceMigrationTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/test/java/org/apache/nifi/migration/ProxyServiceMigrationTest.java new file mode 100644 index 000000000000..05d3dee837a9 --- /dev/null +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-migration-utils/src/test/java/org/apache/nifi/migration/ProxyServiceMigrationTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.migration; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.util.MockPropertyConfiguration; +import org.apache.nifi.util.MockPropertyConfiguration.CreatedControllerService; +import org.apache.nifi.util.PropertyMigrationResult; +import org.junit.jupiter.api.Test; + +import java.net.Proxy; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ProxyServiceMigrationTest { + + private static final PropertyDescriptor PROXY_SERVICE = new PropertyDescriptor.Builder() + .name("proxy-service") + .build(); + + private static final String OBSOLETE_PROXY_HOST = "proxy-host"; + private static final String OBSOLETE_PROXY_PORT = "proxy-port"; + private static final String OBSOLETE_PROXY_USERNAME = "proxy-username"; + private static final String OBSOLETE_PROXY_PASSWORD = "proxy-password"; + + private static final String PROXY_HOST_VALUE = "localhost"; + private static final String PROXY_PORT_VALUE = "8888"; + private static final String PROXY_USERNAME_VALUE = "user"; + private static final String PROXY_PASSWORD_VALUE = "pass"; + + @Test + void testMigrateProxyProperties() { + final Map properties = Map.of( + OBSOLETE_PROXY_HOST, PROXY_HOST_VALUE, + OBSOLETE_PROXY_PORT, PROXY_PORT_VALUE, + OBSOLETE_PROXY_USERNAME, PROXY_USERNAME_VALUE, + OBSOLETE_PROXY_PASSWORD, PROXY_PASSWORD_VALUE + ); + final MockPropertyConfiguration config = new MockPropertyConfiguration(properties); + + ProxyServiceMigration.migrateProxyProperties(config, PROXY_SERVICE, OBSOLETE_PROXY_HOST, OBSOLETE_PROXY_PORT, OBSOLETE_PROXY_USERNAME, OBSOLETE_PROXY_PASSWORD); + + assertFalse(config.hasProperty(OBSOLETE_PROXY_HOST)); + assertFalse(config.hasProperty(OBSOLETE_PROXY_PORT)); + assertFalse(config.hasProperty(OBSOLETE_PROXY_USERNAME)); + assertFalse(config.hasProperty(OBSOLETE_PROXY_PASSWORD)); + + assertTrue(config.isPropertySet(PROXY_SERVICE)); + + PropertyMigrationResult result = config.toPropertyMigrationResult(); + assertEquals(1, result.getCreatedControllerServices().size()); + + final CreatedControllerService createdService = result.getCreatedControllerServices().iterator().next(); + + assertEquals(config.getRawPropertyValue(PROXY_SERVICE).get(), createdService.id()); + assertEquals(ProxyServiceMigration.PROXY_SERVICE_CLASSNAME, createdService.implementationClassName()); + + assertEquals(Map.of( + ProxyServiceMigration.PROXY_SERVICE_TYPE, Proxy.Type.HTTP.name(), + ProxyServiceMigration.PROXY_SERVICE_HOST, PROXY_HOST_VALUE, + ProxyServiceMigration.PROXY_SERVICE_PORT, PROXY_PORT_VALUE, + ProxyServiceMigration.PROXY_SERVICE_USERNAME, PROXY_USERNAME_VALUE, + ProxyServiceMigration.PROXY_SERVICE_PASSWORD, PROXY_PASSWORD_VALUE + ), + createdService.serviceProperties()); + } +} diff --git a/nifi-nar-bundles/nifi-extension-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/pom.xml index 186c64c47e1f..d864171ed816 100644 --- a/nifi-nar-bundles/nifi-extension-utils/pom.xml +++ b/nifi-nar-bundles/nifi-extension-utils/pom.xml @@ -28,6 +28,7 @@ nifi-bin-manager + nifi-conflict-resolution nifi-database-utils nifi-database-test-utils nifi-dbcp-base @@ -38,14 +39,14 @@ nifi-hadoop-utils nifi-kerberos-test-utils nifi-listed-entity + nifi-migration-utils nifi-prometheus-utils nifi-put-pattern + nifi-record-path-property nifi-record-utils nifi-reporting-utils nifi-resource-transfer nifi-service-utils nifi-syslog-utils - nifi-conflict-resolution - nifi-record-path-property diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml index 24e2f86ec933..c6b216e80fe0 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml @@ -87,6 +87,11 @@ nifi-conflict-resolution 2.0.0-SNAPSHOT + + org.apache.nifi + nifi-migration-utils + 2.0.0-SNAPSHOT + org.apache.nifi nifi-mock diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java index e1faf54d8373..0650451a5b1c 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java @@ -21,7 +21,6 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.TransportOptions; import com.google.cloud.http.HttpTransportOptions; -import java.net.Proxy; import java.util.Collections; import java.util.List; import java.util.Map; @@ -32,6 +31,8 @@ import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.gcp.credentials.service.GCPCredentialsService; import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.migration.PropertyConfiguration; +import org.apache.nifi.migration.ProxyServiceMigration; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.util.StandardValidators; @@ -45,6 +46,12 @@ public abstract class AbstractGCPProcessor< CloudService extends Service, CloudServiceOptions extends ServiceOptions> extends AbstractProcessor { + // Obsolete property names + private static final String OBSOLETE_PROXY_HOST = "gcp-proxy-host"; + private static final String OBSOLETE_PROXY_PORT = "gcp-proxy-port"; + private static final String OBSOLETE_PROXY_USERNAME = "gcp-proxy-user-name"; + private static final String OBSOLETE_PROXY_PASSWORD = "gcp-proxy-user-password"; + public static final PropertyDescriptor PROJECT_ID = new PropertyDescriptor .Builder().name("gcp-project-id") .displayName("Project ID") @@ -63,47 +70,6 @@ public abstract class AbstractGCPProcessor< .addValidator(StandardValidators.INTEGER_VALIDATOR) .build(); - public static final PropertyDescriptor PROXY_HOST = new PropertyDescriptor - .Builder().name("gcp-proxy-host") - .displayName("Proxy host") - .description(""" - IP or hostname of the proxy to be used. - You might need to set the following properties in bootstrap for https proxy usage: - -Djdk.http.auth.tunneling.disabledSchemes= - -Djdk.http.auth.proxying.disabledSchemes=""") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - public static final PropertyDescriptor PROXY_PORT = new PropertyDescriptor - .Builder().name("gcp-proxy-port") - .displayName("Proxy port") - .description("Proxy port number") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .addValidator(StandardValidators.PORT_VALIDATOR) - .build(); - - public static final PropertyDescriptor HTTP_PROXY_USERNAME = new PropertyDescriptor - .Builder().name("gcp-proxy-user-name") - .displayName("HTTP Proxy Username") - .description("HTTP Proxy Username") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .required(false) - .build(); - - public static final PropertyDescriptor HTTP_PROXY_PASSWORD = new PropertyDescriptor - .Builder().name("gcp-proxy-user-password") - .displayName("HTTP Proxy Password") - .description("HTTP Proxy Password") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) - .required(false) - .sensitive(true) - .build(); - public static final PropertyDescriptor GCP_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() .name("GCP Credentials Provider Service") .description("The Controller Service used to obtain Google Cloud Platform credentials.") @@ -111,6 +77,8 @@ public abstract class AbstractGCPProcessor< .identifiesControllerService(GCPCredentialsService.class) .build(); + public static final PropertyDescriptor PROXY_CONFIGURATION_SERVICE = ProxyConfiguration.createProxyConfigPropertyDescriptor(false, ProxyAwareTransportFactory.PROXY_SPECS); + protected volatile CloudService cloudService; protected CloudService getCloudService() { @@ -122,14 +90,16 @@ public List getSupportedPropertyDescriptors() { return List.of(PROJECT_ID, GCP_CREDENTIALS_PROVIDER_SERVICE, RETRY_COUNT, - PROXY_HOST, - PROXY_PORT, - HTTP_PROXY_USERNAME, - HTTP_PROXY_PASSWORD, - ProxyConfiguration.createProxyConfigPropertyDescriptor(true, ProxyAwareTransportFactory.PROXY_SPECS) + PROXY_CONFIGURATION_SERVICE ); } + + @Override + public void migrateProperties(final PropertyConfiguration config) { + ProxyServiceMigration.migrateProxyProperties(config, PROXY_CONFIGURATION_SERVICE, OBSOLETE_PROXY_HOST, OBSOLETE_PROXY_PORT, OBSOLETE_PROXY_USERNAME, OBSOLETE_PROXY_PASSWORD); + } + /** * Verifies the cloud service configuration. This is in a separate method rather than implementing VerifiableProcessor due to type erasure. * @param context The process context @@ -206,25 +176,7 @@ public void onScheduled(final ProcessContext context) { * @return Transport options object with proxy configuration */ protected TransportOptions getTransportOptions(ProcessContext context) { - final ProxyConfiguration proxyConfiguration = ProxyConfiguration.getConfiguration(context, () -> { - if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) { - final String proxyHost = context.getProperty(PROXY_HOST).evaluateAttributeExpressions().getValue(); - final Integer proxyPort = context.getProperty(PROXY_PORT).evaluateAttributeExpressions().asInteger(); - if (proxyHost != null && proxyPort != null && proxyPort > 0) { - final ProxyConfiguration componentProxyConfig = new ProxyConfiguration(); - final String proxyUser = context.getProperty(HTTP_PROXY_USERNAME).evaluateAttributeExpressions().getValue(); - final String proxyPassword = context.getProperty(HTTP_PROXY_PASSWORD).evaluateAttributeExpressions().getValue(); - componentProxyConfig.setProxyType(Proxy.Type.HTTP); - componentProxyConfig.setProxyServerHost(proxyHost); - componentProxyConfig.setProxyServerPort(proxyPort); - componentProxyConfig.setProxyUserName(proxyUser); - componentProxyConfig.setProxyUserPassword(proxyPassword); - return componentProxyConfig; - } - } - - return ProxyConfiguration.DIRECT_CONFIGURATION; - }); + final ProxyConfiguration proxyConfiguration = ProxyConfiguration.getConfiguration(context); final ProxyAwareTransportFactory transportFactory = new ProxyAwareTransportFactory(proxyConfiguration); return HttpTransportOptions.newBuilder().setHttpTransportFactory(transportFactory).build(); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java index 172010df4acd..636a9b772992 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java @@ -28,7 +28,6 @@ import javax.annotation.Nullable; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processors.gcp.ProxyAwareTransportFactory; import org.apache.nifi.proxy.ProxyConfiguration; public abstract class AbstractGCPubSubWithProxyProcessor extends AbstractGCPubSubProcessor { @@ -36,28 +35,13 @@ public abstract class AbstractGCPubSubWithProxyProcessor extends AbstractGCPubSu public List getSupportedPropertyDescriptors() { return List.of( PROJECT_ID, - ProxyConfiguration.createProxyConfigPropertyDescriptor(true, ProxyAwareTransportFactory.PROXY_SPECS), + PROXY_CONFIGURATION_SERVICE, GCP_CREDENTIALS_PROVIDER_SERVICE ); } protected TransportChannelProvider getTransportChannelProvider(ProcessContext context) { - final ProxyConfiguration proxyConfiguration = ProxyConfiguration.getConfiguration(context, () -> { - final String proxyHost = context.getProperty(PROXY_HOST).evaluateAttributeExpressions().getValue(); - final Integer proxyPort = context.getProperty(PROXY_PORT).evaluateAttributeExpressions().asInteger(); - if (proxyHost != null && proxyPort != null && proxyPort > 0) { - final ProxyConfiguration componentProxyConfig = new ProxyConfiguration(); - final String proxyUser = context.getProperty(HTTP_PROXY_USERNAME).evaluateAttributeExpressions().getValue(); - final String proxyPassword = context.getProperty(HTTP_PROXY_PASSWORD).evaluateAttributeExpressions().getValue(); - componentProxyConfig.setProxyType(Proxy.Type.HTTP); - componentProxyConfig.setProxyServerHost(proxyHost); - componentProxyConfig.setProxyServerPort(proxyPort); - componentProxyConfig.setProxyUserName(proxyUser); - componentProxyConfig.setProxyUserPassword(proxyPassword); - return componentProxyConfig; - } - return ProxyConfiguration.DIRECT_CONFIGURATION; - }); + final ProxyConfiguration proxyConfiguration = ProxyConfiguration.getConfiguration(context); return TopicAdminSettings.defaultGrpcTransportProviderBuilder() .setChannelConfigurator(managedChannelBuilder -> managedChannelBuilder.proxyDetector( From 6b061452e6bbaa6b43561239072bc2b0e0247fe3 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Wed, 20 Mar 2024 17:23:07 -0500 Subject: [PATCH 60/73] NIFI-12926 Upgraded Jackson from 2.16.2 to 2.17.0 This closes #8538 Signed-off-by: David Handermann --- .../api/config/JsonContentConversionExceptionMapperTest.java | 3 --- pom.xml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/config/JsonContentConversionExceptionMapperTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/config/JsonContentConversionExceptionMapperTest.java index fb9431cab68e..32ca5f92de7c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/config/JsonContentConversionExceptionMapperTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/config/JsonContentConversionExceptionMapperTest.java @@ -16,7 +16,6 @@ */ package org.apache.nifi.web.api.config; -import com.fasterxml.jackson.core.JsonLocation; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.exc.InvalidFormatException; @@ -32,7 +31,6 @@ import static com.fasterxml.jackson.databind.JsonMappingException.wrapWithPath; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class JsonContentConversionExceptionMapperTest { @@ -43,7 +41,6 @@ public class JsonContentConversionExceptionMapperTest { @BeforeEach public void setUp() { - when(mockParser.getTokenLocation()).thenReturn(new JsonLocation(null, 100, 1, 1)); jsonCCEM = new JsonContentConversionExceptionMapper(); } diff --git a/pom.xml b/pom.xml index 67f63cf42e8a..2381f94a4845 100644 --- a/pom.xml +++ b/pom.xml @@ -130,7 +130,7 @@ 2.9.0 10.17.1.0 12.0.6 - 2.16.2 + 2.17.0 1.11.3 4.0.4 1.3.2 From 99b49b631b68b36df12c1ff10a5b4e5fb4728613 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Wed, 20 Mar 2024 18:48:47 -0500 Subject: [PATCH 61/73] NIFI-12927 Upgraded Spring Security from 6.2.2 to 6.2.3 This closes #8539 Signed-off-by: David Handermann --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2381f94a4845..c91980d907cd 100644 --- a/pom.xml +++ b/pom.xml @@ -152,7 +152,7 @@ 4.1.106.Final 6.0.0 6.0.18 - 6.2.2 + 6.2.3 2.2.20 2.2.224 3.9.2 From 2974d9c65e46640be1d5585fd2b805eadc06905b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:18:45 +0000 Subject: [PATCH 62/73] NIFI-12931 Upgraded Commons Configuration from 2.9.0 to 2.10.1 This closes #8541 Signed-off-by: David Handermann --- .../nifi-lookup-services-bundle/nifi-lookup-services/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index d0a252d02744..afe30e312938 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -51,7 +51,7 @@ org.apache.commons commons-configuration2 - 2.9.0 + 2.10.1 commons-logging From 884dd8596a58104728986a1d6dcefdba457ff4f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 20:29:04 +0000 Subject: [PATCH 63/73] NIFI-12932 Upgraded webpack-dev-middleware from 5.3.3 to 5.3.4 This closes #8543 Signed-off-by: David Handermann --- .../nifi-web-frontend/src/main/nifi/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json index f44df9e0e1fd..fe11eba5178e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-frontend/src/main/nifi/package-lock.json @@ -17125,9 +17125,9 @@ } }, "node_modules/webpack-dev-server/node_modules/webpack-dev-middleware": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz", - "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", "dev": true, "dependencies": { "colorette": "^2.0.10", From 8c0829d711da9d38ef87302c4d2f12ced9e7167f Mon Sep 17 00:00:00 2001 From: Mark Bathori Date: Sun, 24 Mar 2024 20:31:13 +0100 Subject: [PATCH 64/73] NIFI-12938: Upgrade Iceberg from 1.4.3 to 1.5.0 Signed-off-by: Pierre Villard This closes #8552. --- nifi-nar-bundles/nifi-iceberg-bundle/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml index 313daffe3caf..f6c96fb45be1 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml @@ -25,7 +25,7 @@ pom - 1.4.3 + 1.5.0 3.1.3 From 635eb9ed2a7f5b74e3b0914eda75c0d3ae302563 Mon Sep 17 00:00:00 2001 From: Peter Turcsanyi Date: Sat, 23 Mar 2024 18:58:52 +0100 Subject: [PATCH 65/73] NIFI-12895 Added Timeout property to GetSmbFile and PutSmbFile Signed-off-by: Pierre Villard This closes #8551. --- .../main/java/org/apache/nifi/processors/smb/GetSmbFile.java | 2 ++ .../main/java/org/apache/nifi/processors/smb/PutSmbFile.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/GetSmbFile.java b/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/GetSmbFile.java index c938ce83f3d7..b4089f5287b1 100644 --- a/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/GetSmbFile.java +++ b/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/GetSmbFile.java @@ -81,6 +81,7 @@ import java.util.regex.Pattern; import static org.apache.nifi.smb.common.SmbProperties.SMB_DIALECT; +import static org.apache.nifi.smb.common.SmbProperties.TIMEOUT; import static org.apache.nifi.smb.common.SmbProperties.USE_ENCRYPTION; import static org.apache.nifi.smb.common.SmbUtils.buildSmbClient; @@ -256,6 +257,7 @@ protected void init(final ProcessorInitializationContext context) { descriptors.add(IGNORE_HIDDEN_FILES); descriptors.add(SMB_DIALECT); descriptors.add(USE_ENCRYPTION); + descriptors.add(TIMEOUT); this.descriptors = Collections.unmodifiableList(descriptors); final Set relationships = new HashSet(); diff --git a/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/PutSmbFile.java b/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/PutSmbFile.java index b7ee444ab986..468828f81988 100644 --- a/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/PutSmbFile.java +++ b/nifi-nar-bundles/nifi-smb-bundle/nifi-smb-processors/src/main/java/org/apache/nifi/processors/smb/PutSmbFile.java @@ -65,6 +65,7 @@ import java.util.concurrent.TimeUnit; import static org.apache.nifi.smb.common.SmbProperties.SMB_DIALECT; +import static org.apache.nifi.smb.common.SmbProperties.TIMEOUT; import static org.apache.nifi.smb.common.SmbProperties.USE_ENCRYPTION; import static org.apache.nifi.smb.common.SmbUtils.buildSmbClient; @@ -193,6 +194,7 @@ protected void init(final ProcessorInitializationContext context) { descriptors.add(RENAME_SUFFIX); descriptors.add(SMB_DIALECT); descriptors.add(USE_ENCRYPTION); + descriptors.add(TIMEOUT); this.descriptors = Collections.unmodifiableList(descriptors); final Set relationships = new HashSet(); From 8eb013a813ad5e8452a3eacd9774068a7bb4d3b3 Mon Sep 17 00:00:00 2001 From: Peter Turcsanyi Date: Fri, 22 Mar 2024 13:13:29 +0100 Subject: [PATCH 66/73] NIFI-12933 Rearranged properties on GCP processors Also used current API methods for relationships/properties collections Signed-off-by: Pierre Villard This closes #8545. --- .../processors/gcp/AbstractGCPProcessor.java | 9 ----- .../bigquery/AbstractBigQueryProcessor.java | 17 +-------- .../processors/gcp/bigquery/PutBigQuery.java | 12 ++---- .../AbstractGCPubSubWithProxyProcessor.java | 10 ----- .../gcp/pubsub/ConsumeGCPubSub.java | 35 +++++++++++------- .../gcp/pubsub/PublishGCPubSub.java | 37 ++++++++++--------- .../gcp/pubsub/lite/ConsumeGCPubSubLite.java | 34 +++++++++-------- .../gcp/pubsub/lite/PublishGCPubSubLite.java | 25 +++++++------ .../gcp/storage/AbstractGCSProcessor.java | 15 +------- .../gcp/storage/DeleteGCSObject.java | 19 ++++++---- .../gcp/storage/FetchGCSObject.java | 26 ++++++++----- .../processors/gcp/storage/ListGCSBucket.java | 31 +++++++++------- .../processors/gcp/storage/PutGCSObject.java | 33 ++++++++++------- 13 files changed, 146 insertions(+), 157 deletions(-) diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java index 0650451a5b1c..db6fc1e722dd 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java @@ -85,15 +85,6 @@ protected CloudService getCloudService() { return cloudService; } - @Override - public List getSupportedPropertyDescriptors() { - return List.of(PROJECT_ID, - GCP_CREDENTIALS_PROVIDER_SERVICE, - RETRY_COUNT, - PROXY_CONFIGURATION_SERVICE - ); - } - @Override public void migrateProperties(final PropertyConfiguration config) { diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/AbstractBigQueryProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/AbstractBigQueryProcessor.java index c2652b29f20d..2012c4c47428 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/AbstractBigQueryProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/AbstractBigQueryProcessor.java @@ -39,10 +39,8 @@ import org.apache.nifi.util.StringUtils; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -63,8 +61,7 @@ public abstract class AbstractBigQueryProcessor extends AbstractGCPProcessor relationships = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE))); + public static final Set RELATIONSHIPS = Set.of(REL_SUCCESS, REL_FAILURE); public static final PropertyDescriptor DATASET = new PropertyDescriptor.Builder() .name(BigQueryAttributes.DATASET_ATTR) @@ -98,17 +95,7 @@ public abstract class AbstractBigQueryProcessor extends AbstractGCPProcessor getRelationships() { - return relationships; - } - - @Override - public List getSupportedPropertyDescriptors() { - final List descriptors = new ArrayList<>(); - descriptors.addAll(super.getSupportedPropertyDescriptors()); - descriptors.add(DATASET); - descriptors.add(TABLE_NAME); - descriptors.add(IGNORE_UNKNOWN); - return Collections.unmodifiableList(descriptors); + return RELATIONSHIPS; } @Override diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java index 8536f232c700..cb51362a08c8 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/bigquery/PutBigQuery.java @@ -62,7 +62,6 @@ import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.processors.gcp.ProxyAwareTransportFactory; import org.apache.nifi.processors.gcp.bigquery.proto.ProtoUtils; import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.serialization.RecordReader; @@ -80,7 +79,6 @@ import java.time.LocalTime; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -88,10 +86,6 @@ import java.util.concurrent.Phaser; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.toList; @TriggerSerially @Tags({"google", "google cloud", "bq", "bigquery"}) @@ -179,7 +173,7 @@ public class PutBigQuery extends AbstractBigQueryProcessor { .defaultValue("false") .build(); - private static final List DESCRIPTORS = Stream.of( + private static final List DESCRIPTORS = List.of( GCP_CREDENTIALS_PROVIDER_SERVICE, PROJECT_ID, BIGQUERY_API_ENDPOINT, @@ -190,8 +184,8 @@ public class PutBigQuery extends AbstractBigQueryProcessor { APPEND_RECORD_COUNT, RETRY_COUNT, SKIP_INVALID_ROWS, - ProxyConfiguration.createProxyConfigPropertyDescriptor(false, ProxyAwareTransportFactory.PROXY_SPECS) - ).collect(collectingAndThen(toList(), Collections::unmodifiableList)); + PROXY_CONFIGURATION_SERVICE + ); @Override public List getSupportedPropertyDescriptors() { diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java index 636a9b772992..aed2b945a511 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/AbstractGCPubSubWithProxyProcessor.java @@ -24,21 +24,11 @@ import java.net.InetSocketAddress; import java.net.Proxy; import java.net.SocketAddress; -import java.util.List; import javax.annotation.Nullable; -import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.proxy.ProxyConfiguration; public abstract class AbstractGCPubSubWithProxyProcessor extends AbstractGCPubSubProcessor { - @Override - public List getSupportedPropertyDescriptors() { - return List.of( - PROJECT_ID, - PROXY_CONFIGURATION_SERVICE, - GCP_CREDENTIALS_PROVIDER_SERVICE - ); - } protected TransportChannelProvider getTransportChannelProvider(ProcessContext context) { final ProxyConfiguration proxyConfiguration = ProxyConfiguration.getConfiguration(context); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/ConsumeGCPubSub.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/ConsumeGCPubSub.java index 5e93eeebf33b..377b9275f3ea 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/ConsumeGCPubSub.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/ConsumeGCPubSub.java @@ -95,11 +95,32 @@ public class ConsumeGCPubSub extends AbstractGCPubSubWithProxyProcessor { .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + PROJECT_ID, + SUBSCRIPTION, + BATCH_SIZE_THRESHOLD, + API_ENDPOINT, + PROXY_CONFIGURATION_SERVICE + ); + + public static final Set RELATIONSHIPS = Set.of(REL_SUCCESS); + private SubscriberStub subscriber = null; private PullRequest pullRequest; private final AtomicReference storedException = new AtomicReference<>(); + @Override + public List getSupportedPropertyDescriptors() { + return DESCRIPTORS; + } + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + @OnScheduled public void onScheduled(ProcessContext context) { final Integer batchSize = context.getProperty(BATCH_SIZE_THRESHOLD).asInteger(); @@ -188,20 +209,6 @@ public void onStopped() { } } - @Override - public List getSupportedPropertyDescriptors() { - final List descriptors = new ArrayList<>(super.getSupportedPropertyDescriptors()); - descriptors.add(SUBSCRIPTION); - descriptors.add(BATCH_SIZE_THRESHOLD); - descriptors.add(API_ENDPOINT); - return Collections.unmodifiableList(descriptors); - } - - @Override - public Set getRelationships() { - return Collections.singleton(REL_SUCCESS); - } - @Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { if (subscriber == null) { diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/PublishGCPubSub.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/PublishGCPubSub.java index 4c95e466ef28..0d43ef06d131 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/PublishGCPubSub.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/PublishGCPubSub.java @@ -75,10 +75,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -173,22 +171,29 @@ public class PublishGCPubSub extends AbstractGCPubSubWithProxyProcessor { .description("FlowFiles are routed to this relationship if the Google Cloud Pub/Sub operation fails but attempting the operation again may succeed.") .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + PROJECT_ID, + TOPIC_NAME, + MESSAGE_DERIVATION_STRATEGY, + RECORD_READER, + RECORD_WRITER, + MAX_BATCH_SIZE, + MAX_MESSAGE_SIZE, + BATCH_SIZE_THRESHOLD, + BATCH_BYTES_THRESHOLD, + BATCH_DELAY_THRESHOLD, + API_ENDPOINT, + PROXY_CONFIGURATION_SERVICE + ); + + public static final Set RELATIONSHIPS = Set.of(REL_SUCCESS, REL_FAILURE, REL_RETRY); + protected Publisher publisher = null; @Override public List getSupportedPropertyDescriptors() { - final List descriptors = new ArrayList<>(super.getSupportedPropertyDescriptors()); - descriptors.add(MAX_BATCH_SIZE); - descriptors.add(MESSAGE_DERIVATION_STRATEGY); - descriptors.add(RECORD_READER); - descriptors.add(RECORD_WRITER); - descriptors.add(MAX_MESSAGE_SIZE); - descriptors.add(TOPIC_NAME); - descriptors.add(BATCH_SIZE_THRESHOLD); - descriptors.add(BATCH_BYTES_THRESHOLD); - descriptors.add(BATCH_DELAY_THRESHOLD); - descriptors.add(API_ENDPOINT); - return Collections.unmodifiableList(descriptors); + return DESCRIPTORS; } @Override @@ -205,9 +210,7 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String @Override public Set getRelationships() { - return Collections.unmodifiableSet( - new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE, REL_RETRY)) - ); + return RELATIONSHIPS; } @Override diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/ConsumeGCPubSubLite.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/ConsumeGCPubSubLite.java index e223f06d43a9..9cab4f638344 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/ConsumeGCPubSubLite.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/ConsumeGCPubSubLite.java @@ -52,9 +52,7 @@ import org.apache.nifi.processors.gcp.pubsub.AbstractGCPubSubProcessor; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -116,9 +114,28 @@ public class ConsumeGCPubSubLite extends AbstractGCPubSubProcessor implements Ve .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + SUBSCRIPTION, + BYTES_OUTSTANDING, + MESSAGES_OUTSTANDING + ); + + public static final Set RELATIONSHIPS = Set.of(REL_SUCCESS); + private Subscriber subscriber = null; private BlockingQueue messages = new LinkedBlockingQueue<>(); + @Override + public List getSupportedPropertyDescriptors() { + return DESCRIPTORS; + } + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + @Override protected Collection customValidate(final ValidationContext validationContext) { final Collection results = new ArrayList(1); @@ -168,19 +185,6 @@ public void onStopped() { } } - @Override - public List getSupportedPropertyDescriptors() { - return Collections.unmodifiableList(Arrays.asList(SUBSCRIPTION, - GCP_CREDENTIALS_PROVIDER_SERVICE, - BYTES_OUTSTANDING, - MESSAGES_OUTSTANDING)); - } - - @Override - public Set getRelationships() { - return Collections.singleton(REL_SUCCESS); - } - @Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { if (subscriber == null) { diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/PublishGCPubSubLite.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/PublishGCPubSubLite.java index 17ec36e67912..931f53581749 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/PublishGCPubSubLite.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/pubsub/lite/PublishGCPubSubLite.java @@ -61,11 +61,8 @@ import java.io.ByteArrayOutputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -112,16 +109,22 @@ public class PublishGCPubSubLite extends AbstractGCPubSubProcessor implements Ve .addValidator(StandardValidators.NON_BLANK_VALIDATOR) .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + TOPIC_NAME, + ORDERING_KEY, + BATCH_SIZE_THRESHOLD, + BATCH_BYTES_THRESHOLD, + BATCH_DELAY_THRESHOLD + ); + + public static final Set RELATIONSHIPS = Set.of(REL_SUCCESS, REL_FAILURE); + private Publisher publisher = null; @Override public List getSupportedPropertyDescriptors() { - return Collections.unmodifiableList(Arrays.asList(TOPIC_NAME, - GCP_CREDENTIALS_PROVIDER_SERVICE, - ORDERING_KEY, - BATCH_SIZE_THRESHOLD, - BATCH_BYTES_THRESHOLD, - BATCH_DELAY_THRESHOLD)); + return DESCRIPTORS; } @Override @@ -138,9 +141,7 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String @Override public Set getRelationships() { - return Collections.unmodifiableSet( - new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE)) - ); + return RELATIONSHIPS; } @Override diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java index 213ff6d31eea..cf343389f89a 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java @@ -39,10 +39,7 @@ import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -62,12 +59,11 @@ public abstract class AbstractGCSProcessor extends AbstractGCPProcessor relationships = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE))); + public static final Set RELATIONSHIPS = Set.of(REL_SUCCESS, REL_FAILURE); @Override public Set getRelationships() { - return relationships; + return RELATIONSHIPS; } // https://cloud.google.com/storage/docs/request-endpoints#storage-set-client-endpoint-java @@ -81,13 +77,6 @@ public Set getRelationships() { .required(false) .build(); - @Override - public List getSupportedPropertyDescriptors() { - final List propertyDescriptors = new ArrayList<>(super.getSupportedPropertyDescriptors()); - propertyDescriptors.add(STORAGE_API_URL); - return Collections.unmodifiableList(propertyDescriptors); - } - @Override public List verify(final ProcessContext context, final ComponentLog verificationLogger, final Map attributes) { final List results = new ArrayList<>(verifyCloudService(context, verificationLogger, attributes)); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/DeleteGCSObject.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/DeleteGCSObject.java index ec8ed1bf4270..0c34d5878aa1 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/DeleteGCSObject.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/DeleteGCSObject.java @@ -31,7 +31,6 @@ import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -77,14 +76,20 @@ public class DeleteGCSObject extends AbstractGCSProcessor { .required(false) .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + PROJECT_ID, + BUCKET, + KEY, + GENERATION, + RETRY_COUNT, + STORAGE_API_URL, + PROXY_CONFIGURATION_SERVICE + ); + @Override public List getSupportedPropertyDescriptors() { - final List descriptors = new ArrayList<>(); - descriptors.addAll(super.getSupportedPropertyDescriptors()); - descriptors.add(BUCKET); - descriptors.add(KEY); - descriptors.add(GENERATION); - return Collections.unmodifiableList(descriptors); + return DESCRIPTORS; } @Override diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/FetchGCSObject.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/FetchGCSObject.java index ee9a6d6b5a35..12376abf7a80 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/FetchGCSObject.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/FetchGCSObject.java @@ -170,7 +170,7 @@ public class FetchGCSObject extends AbstractGCSProcessor { public static final PropertyDescriptor KEY = new PropertyDescriptor .Builder().name("gcs-key") - .displayName("Name") + .displayName("Key") .description(KEY_DESC) .required(true) .defaultValue("${" + CoreAttributes.FILENAME.key() + "}") @@ -217,17 +217,23 @@ public class FetchGCSObject extends AbstractGCSProcessor { .required(false) .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + PROJECT_ID, + BUCKET, + KEY, + GENERATION, + ENCRYPTION_KEY, + RANGE_START, + RANGE_LENGTH, + RETRY_COUNT, + STORAGE_API_URL, + PROXY_CONFIGURATION_SERVICE + ); + @Override public List getSupportedPropertyDescriptors() { - final List descriptors = new ArrayList<>(); - descriptors.add(BUCKET); - descriptors.add(KEY); - descriptors.addAll(super.getSupportedPropertyDescriptors()); - descriptors.add(GENERATION); - descriptors.add(ENCRYPTION_KEY); - descriptors.add(RANGE_START); - descriptors.add(RANGE_LENGTH); - return Collections.unmodifiableList(descriptors); + return DESCRIPTORS; } @Override diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java index 25583b87ce4c..aed69ba1ee9f 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java @@ -250,27 +250,32 @@ public class ListGCSBucket extends AbstractGCSProcessor { .identifiesControllerService(RecordSetWriterFactory.class) .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + PROJECT_ID, + BUCKET, + PREFIX, + LISTING_STRATEGY, + TRACKING_STATE_CACHE, + INITIAL_LISTING_TARGET, + TRACKING_TIME_WINDOW, + RECORD_WRITER, + USE_GENERATIONS, + RETRY_COUNT, + STORAGE_API_URL, + PROXY_CONFIGURATION_SERVICE + ); @Override public List getSupportedPropertyDescriptors() { - final List descriptors = new ArrayList<>(); - descriptors.add(LISTING_STRATEGY); - descriptors.add(TRACKING_STATE_CACHE); - descriptors.add(INITIAL_LISTING_TARGET); - descriptors.add(TRACKING_TIME_WINDOW); - descriptors.add(BUCKET); - descriptors.add(RECORD_WRITER); - descriptors.addAll(super.getSupportedPropertyDescriptors()); - descriptors.add(PREFIX); - descriptors.add(USE_GENERATIONS); - return Collections.unmodifiableList(descriptors); + return DESCRIPTORS; } - private static final Set relationships = Collections.singleton(REL_SUCCESS); + private static final Set RELATIONSHIPS = Set.of(REL_SUCCESS); @Override public Set getRelationships() { - return relationships; + return RELATIONSHIPS; } // State tracking diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java index 438fac19b888..606389650446 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/PutGCSObject.java @@ -289,21 +289,28 @@ public class PutGCSObject extends AbstractGCSProcessor { .allowableValues(CD_INLINE, CD_ATTACHMENT) .build(); + private static final List DESCRIPTORS = List.of( + GCP_CREDENTIALS_PROVIDER_SERVICE, + PROJECT_ID, + BUCKET, + KEY, + RESOURCE_TRANSFER_SOURCE, + FILE_RESOURCE_SERVICE, + CONTENT_TYPE, + CRC32C, + ACL, + ENCRYPTION_KEY, + OVERWRITE, + CONTENT_DISPOSITION_TYPE, + GZIPCONTENT, + STORAGE_API_URL, + RETRY_COUNT, + PROXY_CONFIGURATION_SERVICE + ); + @Override public List getSupportedPropertyDescriptors() { - final List descriptors = new ArrayList<>(super.getSupportedPropertyDescriptors()); - descriptors.add(BUCKET); - descriptors.add(KEY); - descriptors.add(RESOURCE_TRANSFER_SOURCE); - descriptors.add(FILE_RESOURCE_SERVICE); - descriptors.add(CONTENT_TYPE); - descriptors.add(CRC32C); - descriptors.add(ACL); - descriptors.add(ENCRYPTION_KEY); - descriptors.add(OVERWRITE); - descriptors.add(CONTENT_DISPOSITION_TYPE); - descriptors.add(GZIPCONTENT); - return Collections.unmodifiableList(descriptors); + return DESCRIPTORS; } @Override From bffc34252158103d0de727783c85c1cac727e5f4 Mon Sep 17 00:00:00 2001 From: Peter Turcsanyi Date: Thu, 21 Mar 2024 08:47:28 +0100 Subject: [PATCH 67/73] NIFI-12928 Added Simple Write strategy in PutAzureDataLakeStorage Signed-off-by: Pierre Villard This closes #8540. --- .../storage/PutAzureDataLakeStorage.java | 148 ++++++++++++----- .../azure/storage/utils/WritingStrategy.java | 49 ++++++ .../additionalDetails.html | 44 ++++- .../AbstractAzureDataLakeStorageIT.java | 2 + .../storage/ITPutAzureDataLakeStorage.java | 155 ++++++++++++------ 5 files changed, 298 insertions(+), 100 deletions(-) create mode 100644 nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/WritingStrategy.java diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java index 7651ad6a99bd..d385eeb27dfb 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/PutAzureDataLakeStorage.java @@ -40,6 +40,7 @@ import org.apache.nifi.processors.azure.AbstractAzureDataLakeStorageProcessor; import org.apache.nifi.processors.azure.storage.utils.AzureStorageUtils; import org.apache.nifi.processors.azure.storage.utils.AzureStorageUtils.DirectoryValidator; +import org.apache.nifi.processors.azure.storage.utils.WritingStrategy; import org.apache.nifi.processors.transfer.ResourceTransferSource; import org.apache.nifi.util.StringUtils; @@ -99,6 +100,15 @@ public class PutAzureDataLakeStorage extends AbstractAzureDataLakeStorageProcess .allowableValues(FAIL_RESOLUTION, REPLACE_RESOLUTION, IGNORE_RESOLUTION) .build(); + protected static final PropertyDescriptor WRITING_STRATEGY = new PropertyDescriptor.Builder() + .name("writing-strategy") + .displayName("Writing Strategy") + .description("Defines the approach for writing the Azure file.") + .required(true) + .allowableValues(WritingStrategy.class) + .defaultValue(WritingStrategy.WRITE_AND_RENAME) + .build(); + public static final PropertyDescriptor BASE_TEMPORARY_PATH = new PropertyDescriptor.Builder() .name("base-temporary-path") .displayName("Base Temporary Path") @@ -108,6 +118,7 @@ public class PutAzureDataLakeStorage extends AbstractAzureDataLakeStorageProcess .defaultValue("") .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .addValidator(new DirectoryValidator("Base Temporary Path")) + .dependsOn(WRITING_STRATEGY, WritingStrategy.WRITE_AND_RENAME) .build(); private static final List PROPERTIES = List.of( @@ -115,6 +126,7 @@ public class PutAzureDataLakeStorage extends AbstractAzureDataLakeStorageProcess FILESYSTEM, DIRECTORY, FILE, + WRITING_STRATEGY, BASE_TEMPORARY_PATH, CONFLICT_RESOLUTION, RESOURCE_TRANSFER_SOURCE, @@ -137,41 +149,44 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final long startNanos = System.nanoTime(); try { final String fileSystem = evaluateFileSystemProperty(FILESYSTEM, context, flowFile); - final String originalDirectory = evaluateDirectoryProperty(DIRECTORY, context, flowFile); - final String tempPath = evaluateDirectoryProperty(BASE_TEMPORARY_PATH, context, flowFile); - final String tempDirectory = createPath(tempPath, TEMP_FILE_DIRECTORY); + final String directory = evaluateDirectoryProperty(DIRECTORY, context, flowFile); final String fileName = evaluateFileProperty(context, flowFile); final DataLakeFileSystemClient fileSystemClient = getFileSystemClient(context, flowFile, fileSystem); - final DataLakeDirectoryClient directoryClient = fileSystemClient.getDirectoryClient(originalDirectory); + final DataLakeDirectoryClient directoryClient = fileSystemClient.getDirectoryClient(directory); - final String tempFilePrefix = UUID.randomUUID().toString(); - final DataLakeDirectoryClient tempDirectoryClient = fileSystemClient.getDirectoryClient(tempDirectory); + final WritingStrategy writingStrategy = context.getProperty(WRITING_STRATEGY).asAllowableValue(WritingStrategy.class); final String conflictResolution = context.getProperty(CONFLICT_RESOLUTION).getValue(); final ResourceTransferSource resourceTransferSource = ResourceTransferSource.valueOf(context.getProperty(RESOURCE_TRANSFER_SOURCE).getValue()); final Optional fileResourceFound = getFileResource(resourceTransferSource, context, flowFile.getAttributes()); final long transferSize = fileResourceFound.map(FileResource::getSize).orElse(flowFile.getSize()); - final DataLakeFileClient tempFileClient = tempDirectoryClient.createFile(tempFilePrefix + fileName, true); - if (transferSize > 0) { - final FlowFile sourceFlowFile = flowFile; - try ( - final InputStream inputStream = new BufferedInputStream( - fileResourceFound.map(FileResource::getInputStream) - .orElseGet(() -> session.read(sourceFlowFile)) - ) - ) { - uploadContent(tempFileClient, inputStream, transferSize); - } catch (final Exception e) { - removeTempFile(tempFileClient); - throw e; + final DataLakeFileClient fileClient; + + if (writingStrategy == WritingStrategy.WRITE_AND_RENAME) { + final String tempPath = evaluateDirectoryProperty(BASE_TEMPORARY_PATH, context, flowFile); + final String tempDirectory = createPath(tempPath, TEMP_FILE_DIRECTORY); + final String tempFilePrefix = UUID.randomUUID().toString(); + + final DataLakeDirectoryClient tempDirectoryClient = fileSystemClient.getDirectoryClient(tempDirectory); + final DataLakeFileClient tempFileClient = tempDirectoryClient.createFile(tempFilePrefix + fileName, true); + + uploadFile(session, flowFile, fileResourceFound, transferSize, tempFileClient); + + createDirectoryIfNotExists(directoryClient); + + fileClient = renameFile(tempFileClient, directoryClient.getDirectoryPath(), fileName, conflictResolution); + } else { + fileClient = createFile(directoryClient, fileName, conflictResolution); + + if (fileClient != null) { + uploadFile(session, flowFile, fileResourceFound, transferSize, fileClient); } } - createDirectoryIfNotExists(directoryClient); - final String fileUrl = renameFile(tempFileClient, directoryClient.getDirectoryPath(), fileName, conflictResolution); - if (fileUrl != null) { - final Map attributes = createAttributeMap(fileSystem, originalDirectory, fileName, fileUrl, transferSize); + if (fileClient != null) { + final String fileUrl = fileClient.getFileUrl(); + final Map attributes = createAttributeMap(fileSystem, directory, fileName, fileUrl, transferSize); flowFile = session.putAllAttributes(flowFile, attributes); final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); @@ -186,12 +201,12 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } } - private DataLakeFileSystemClient getFileSystemClient(ProcessContext context, FlowFile flowFile, String fileSystem) { + private DataLakeFileSystemClient getFileSystemClient(final ProcessContext context, final FlowFile flowFile, final String fileSystem) { final DataLakeServiceClient storageClient = getStorageClient(context, flowFile); return storageClient.getFileSystemClient(fileSystem); } - private Map createAttributeMap(String fileSystem, String originalDirectory, String fileName, String fileUrl, long length) { + private Map createAttributeMap(final String fileSystem, final String originalDirectory, final String fileName, final String fileUrl, final long length) { final Map attributes = new HashMap<>(); attributes.put(ATTR_NAME_FILESYSTEM, fileSystem); attributes.put(ATTR_NAME_DIRECTORY, originalDirectory); @@ -201,14 +216,29 @@ private Map createAttributeMap(String fileSystem, String origina return attributes; } - private void createDirectoryIfNotExists(DataLakeDirectoryClient directoryClient) { + private void createDirectoryIfNotExists(final DataLakeDirectoryClient directoryClient) { if (!directoryClient.getDirectoryPath().isEmpty() && !directoryClient.exists()) { directoryClient.create(); } } + private void uploadFile(final ProcessSession session, final FlowFile flowFile, final Optional fileResourceFound, + final long transferSize, final DataLakeFileClient fileClient) throws Exception { + if (transferSize > 0) { + try (final InputStream inputStream = new BufferedInputStream( + fileResourceFound.map(FileResource::getInputStream) + .orElseGet(() -> session.read(flowFile))) + ) { + uploadContent(fileClient, inputStream, transferSize); + } catch (final Exception e) { + removeFile(fileClient); + throw e; + } + } + } + //Visible for testing - static void uploadContent(DataLakeFileClient fileClient, InputStream in, long length) { + static void uploadContent(final DataLakeFileClient fileClient, final InputStream in, final long length) { long chunkStart = 0; long chunkSize; @@ -229,19 +259,41 @@ static void uploadContent(DataLakeFileClient fileClient, InputStream in, long le } /** - * This method serves as a "commit" for the upload process. Upon upload, a 0-byte file is created, then the payload is appended to it. - * Because of that, a work-in-progress file is available for readers before the upload is complete. It is not an efficient approach in - * case of conflicts because FlowFiles are uploaded unnecessarily, but it is a calculated risk because consistency is more important. + * Creates the file on Azure for 'Simple Write' strategy. Upon upload, a 0-byte file is created, then the payload is appended to it. + * Because of that, a work-in-progress file is available for readers before the upload is complete. + * + * @param directoryClient directory client of the uploaded file's parent directory + * @param fileName name of the uploaded file + * @param conflictResolution conflict resolution strategy + * @return the file client of the uploaded file or {@code null} if the file already exists and conflict resolution strategy is 'ignore' + * @throws ProcessException if the file already exists and the conflict resolution strategy is 'fail'; also in case of other errors + */ + DataLakeFileClient createFile(DataLakeDirectoryClient directoryClient, final String fileName, final String conflictResolution) { + final String destinationPath = createPath(directoryClient.getDirectoryPath(), fileName); + + try { + final boolean overwrite = conflictResolution.equals(REPLACE_RESOLUTION); + return directoryClient.createFile(fileName, overwrite); + } catch (DataLakeStorageException dataLakeStorageException) { + return handleDataLakeStorageException(dataLakeStorageException, destinationPath, conflictResolution); + } + } + + /** + * This method serves as a "commit" for the upload process in case of 'Write and Rename' strategy. In order to prevent work-in-progress files from being available for readers, + * a temporary file is written first, and then renamed/moved to its final destination. It is not an efficient approach in case of conflicts because FlowFiles are uploaded unnecessarily, + * but it is a calculated risk because consistency is more important for 'Write and Rename' strategy. *

* Visible for testing * - * @param sourceFileClient client of the temporary file + * @param sourceFileClient file client of the temporary file * @param destinationDirectory final location of the uploaded file * @param destinationFileName final name of the uploaded file * @param conflictResolution conflict resolution strategy - * @return URL of the uploaded file + * @return the file client of the uploaded file or {@code null} if the file already exists and conflict resolution strategy is 'ignore' + * @throws ProcessException if the file already exists and the conflict resolution strategy is 'fail'; also in case of other errors */ - String renameFile(final DataLakeFileClient sourceFileClient, final String destinationDirectory, final String destinationFileName, final String conflictResolution) { + DataLakeFileClient renameFile(final DataLakeFileClient sourceFileClient, final String destinationDirectory, final String destinationFileName, final String conflictResolution) { final String destinationPath = createPath(destinationDirectory, destinationFileName); try { @@ -249,18 +301,24 @@ String renameFile(final DataLakeFileClient sourceFileClient, final String destin if (!conflictResolution.equals(REPLACE_RESOLUTION)) { destinationCondition.setIfNoneMatch("*"); } - return sourceFileClient.renameWithResponse(null, destinationPath, null, destinationCondition, null, null).getValue().getFileUrl(); + return sourceFileClient.renameWithResponse(null, destinationPath, null, destinationCondition, null, null).getValue(); } catch (DataLakeStorageException dataLakeStorageException) { - removeTempFile(sourceFileClient); - if (dataLakeStorageException.getStatusCode() == 409 && conflictResolution.equals(IGNORE_RESOLUTION)) { - getLogger().info("File [{}] already exists. Remote file not modified due to {} being set to '{}'.", - destinationPath, CONFLICT_RESOLUTION.getDisplayName(), conflictResolution); - return null; - } else if (dataLakeStorageException.getStatusCode() == 409 && conflictResolution.equals(FAIL_RESOLUTION)) { - throw new ProcessException(String.format("File [%s] already exists.", destinationPath), dataLakeStorageException); - } else { - throw new ProcessException(String.format("Renaming File [%s] failed", destinationPath), dataLakeStorageException); - } + removeFile(sourceFileClient); + + return handleDataLakeStorageException(dataLakeStorageException, destinationPath, conflictResolution); + } + } + + private DataLakeFileClient handleDataLakeStorageException(final DataLakeStorageException dataLakeStorageException, final String destinationPath, final String conflictResolution) { + final boolean fileAlreadyExists = dataLakeStorageException.getStatusCode() == 409; + if (fileAlreadyExists && conflictResolution.equals(IGNORE_RESOLUTION)) { + getLogger().info("File [{}] already exists. Remote file not modified due to {} being set to '{}'.", + destinationPath, CONFLICT_RESOLUTION.getDisplayName(), conflictResolution); + return null; + } else if (fileAlreadyExists && conflictResolution.equals(FAIL_RESOLUTION)) { + throw new ProcessException(String.format("File [%s] already exists.", destinationPath), dataLakeStorageException); + } else { + throw new ProcessException(String.format("File operation failed [%s]", destinationPath), dataLakeStorageException); } } @@ -270,7 +328,7 @@ private String createPath(final String baseDirectory, final String path) { : path; } - private void removeTempFile(final DataLakeFileClient fileClient) { + private void removeFile(final DataLakeFileClient fileClient) { try { fileClient.delete(); } catch (Exception e) { diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/WritingStrategy.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/WritingStrategy.java new file mode 100644 index 000000000000..6fda36623d85 --- /dev/null +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/WritingStrategy.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.azure.storage.utils; + +import org.apache.nifi.components.DescribedValue; + +public enum WritingStrategy implements DescribedValue { + + WRITE_AND_RENAME("Write and Rename", "The processor writes the Azure file into a temporary directory and then renames/moves it to the final destination." + + " This prevents other processes from reading partially written files."), + SIMPLE_WRITE("Simple Write", "The processor writes the Azure file directly to the destination. This might result in the reading of partially written files."); + + private final String displayName; + private final String description; + + WritingStrategy(String displayName, String description) { + this.displayName = displayName; + this.description = description; + } + + @Override + public String getValue() { + return name(); + } + + @Override + public String getDisplayName() { + return displayName; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/docs/org.apache.nifi.processors.azure.storage.PutAzureDataLakeStorage/additionalDetails.html b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/docs/org.apache.nifi.processors.azure.storage.PutAzureDataLakeStorage/additionalDetails.html index 2469ceafaa20..4758d4c19efc 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/docs/org.apache.nifi.processors.azure.storage.PutAzureDataLakeStorage/additionalDetails.html +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/docs/org.apache.nifi.processors.azure.storage.PutAzureDataLakeStorage/additionalDetails.html @@ -26,31 +26,57 @@ This processor is responsible for uploading files to Azure Data Lake Storage Gen2.

-

File uploading and cleanup process

+

File uploading and cleanup process in case of "Write and Rename" strategy

New file upload

  1. A temporary file is created with random prefix under the given path in '_nifitempdirectory'.
  2. Content is appended to temp file.
    3. -
  3. Temp file is renamed to its original name, the original file is overwritten.
    4. -
  4. In case of appending or renaming failure the temp file is deleted, the original file remains intact.
    5. -
  5. In case of temporary file deletion failure both temp file and original file remain on the server.
    6. +
  6. Temp file is moved to the final destination directory and renamed to its original name.
    7. +
  7. In case of appending or renaming failure, the temp file is deleted.
    8. +
  8. In case of temporary file deletion failure, the temp file remains on the server.

Existing file upload

    -
  • Processors with "fail" conflict resolution strategy will be directed to "Failure" relationship.
    • -
  • Processors with "ignore" conflict resolution strategy will be directed to "Success" relationship.
    • +
  • Processors with "fail" conflict resolution strategy will direct the FlowFile to "Failure" relationship.
    • +
  • Processors with "ignore" conflict resolution strategy will direct the FlowFile to "Success" relationship.
  • Processors with "replace" conflict resolution strategy:
    1. A temporary file is created with random prefix under the given path in '_nifitempdirectory'.
    2. Content is appended to temp file.
      3. -
    3. Temp file is renamed to its original name, the original file is overwritten.
      4. -
    4. In case of appending or renaming failure the temp file is deleted, the original file remains intact.
      5. -
    5. In case of temporary file deletion failure both temp file and original file remain on the server.
      6. +
    6. Temp file is moved to the final destination directory and renamed to its original name, the original file is overwritten.
      7. +
    7. In case of appending or renaming failure, the temp file is deleted and the original file remains intact.
      8. +
    8. In case of temporary file deletion failure, both temp file and original file remain on the server.
      9. +
    +
+ +

File uploading and cleanup process in case of "Simple Write" strategy

+ +

New file upload

+ +
    +
  1. An empty file is created at its final destination.
    2. +
  2. Content is appended to the file.
    3. +
  3. In case of appending failure, the file is deleted.
    4. +
  4. In case of file deletion failure, the file remains on the server.
    5. +
+ +

Existing file upload

+ +
    +
  • Processors with "fail" conflict resolution strategy will direct the FlowFile to "Failure" relationship.
    • +
  • Processors with "ignore" conflict resolution strategy will direct the FlowFile to "Success" relationship.
    • +
  • Processors with "replace" conflict resolution strategy:
    • + +
      +
    1. An empty file is created at its final destination, the original file is overwritten.
      2. +
    2. Content is appended to the file.
      3. +
    3. In case of appending failure, the file is deleted and the original file is not restored.
      4. +
    4. In case of file deletion failure, the file remains on the server.
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/AbstractAzureDataLakeStorageIT.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/AbstractAzureDataLakeStorageIT.java index 8e94485a1fbc..fc5f2e62322d 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/AbstractAzureDataLakeStorageIT.java +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/AbstractAzureDataLakeStorageIT.java @@ -26,6 +26,7 @@ import org.apache.nifi.processors.azure.storage.utils.AzureStorageUtils; import org.apache.nifi.services.azure.storage.ADLSCredentialsControllerService; import org.apache.nifi.services.azure.storage.ADLSCredentialsService; +import org.apache.nifi.services.azure.storage.AzureStorageCredentialsType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -53,6 +54,7 @@ protected String getDefaultEndpointSuffix() { protected void setUpCredentials() throws Exception { ADLSCredentialsService service = new ADLSCredentialsControllerService(); runner.addControllerService("ADLSCredentials", service); + runner.setProperty(service, AzureStorageUtils.CREDENTIALS_TYPE, AzureStorageCredentialsType.ACCOUNT_KEY); runner.setProperty(service, ADLSCredentialsControllerService.ACCOUNT_NAME, getAccountName()); runner.setProperty(service, AzureStorageUtils.ACCOUNT_KEY, getAccountKey()); runner.enableControllerService(service); diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java index 4a59a9b3a4a1..b4708375b928 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureDataLakeStorage.java @@ -23,13 +23,15 @@ import org.apache.nifi.fileresource.service.api.FileResourceService; import org.apache.nifi.processor.Processor; import org.apache.nifi.processors.azure.storage.utils.AzureStorageUtils; +import org.apache.nifi.processors.azure.storage.utils.WritingStrategy; import org.apache.nifi.processors.transfer.ResourceTransferProperties; import org.apache.nifi.processors.transfer.ResourceTransferSource; import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.provenance.ProvenanceEventType; import org.apache.nifi.util.MockFlowFile; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -69,27 +71,23 @@ public void setUp() { runner.setProperty(AzureStorageUtils.FILE, FILE_NAME); } - @Test - public void testPutFileToExistingDirectory() throws Exception { - fileSystemClient.createDirectory(DIRECTORY); - - runProcessor(FILE_DATA); + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToExistingDirectory(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); - assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); - } - - @Test - public void testPutFileToExistingDirectoryUsingProxyConfigurationService() throws Exception { fileSystemClient.createDirectory(DIRECTORY); - configureProxyService(); runProcessor(FILE_DATA); assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); } - @Test - public void testPutFileToExistingDirectoryWithReplaceResolution() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToExistingDirectoryWithReplaceResolution(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + fileSystemClient.createDirectory(DIRECTORY); runner.setProperty(PutAzureDataLakeStorage.CONFLICT_RESOLUTION, PutAzureDataLakeStorage.REPLACE_RESOLUTION); @@ -99,8 +97,11 @@ public void testPutFileToExistingDirectoryWithReplaceResolution() throws Excepti assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); } - @Test - public void testPutFileToExistingDirectoryWithIgnoreResolution() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToExistingDirectoryWithIgnoreResolution(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + fileSystemClient.createDirectory(DIRECTORY); runner.setProperty(PutAzureDataLakeStorage.CONFLICT_RESOLUTION, PutAzureDataLakeStorage.IGNORE_RESOLUTION); @@ -110,15 +111,21 @@ public void testPutFileToExistingDirectoryWithIgnoreResolution() throws Exceptio assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); } - @Test - public void testPutFileToNonExistingDirectory() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToNonExistingDirectory(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + runProcessor(FILE_DATA); assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); } - @Test - public void testPutFileToDeepDirectory() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToDeepDirectory(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + String baseDirectory = "dir1/dir2"; String fullDirectory = baseDirectory + "/dir3/dir4"; fileSystemClient.createDirectory(baseDirectory); @@ -129,8 +136,11 @@ public void testPutFileToDeepDirectory() throws Exception { assertSuccess(fullDirectory, FILE_NAME, FILE_DATA); } - @Test - public void testPutFileToRootDirectory() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToRootDirectory(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + String rootDirectory = ""; runner.setProperty(AzureStorageUtils.DIRECTORY, rootDirectory); @@ -139,8 +149,11 @@ public void testPutFileToRootDirectory() throws Exception { assertSuccess(rootDirectory, FILE_NAME, FILE_DATA); } - @Test - public void testPutEmptyFile() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutEmptyFile(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + byte[] fileData = new byte[0]; runProcessor(fileData); @@ -148,8 +161,11 @@ public void testPutEmptyFile() throws Exception { assertSuccess(DIRECTORY, FILE_NAME, fileData); } - @Test - public void testPutBigFile() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutBigFile(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + Random random = new Random(); byte[] fileData = new byte[120_000_000]; random.nextBytes(fileData); @@ -159,8 +175,11 @@ public void testPutBigFile() throws Exception { assertSuccess(DIRECTORY, FILE_NAME, fileData); } - @Test - public void testPutFileWithNonExistingFileSystem() { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileWithNonExistingFileSystem(WritingStrategy writingStrategy) { + setWritingStrategy(writingStrategy); + runner.setProperty(AzureStorageUtils.FILESYSTEM, "dummy"); runProcessor(FILE_DATA); @@ -168,8 +187,11 @@ public void testPutFileWithNonExistingFileSystem() { assertFailure(); } - @Test - public void testPutFileWithInvalidFileName() { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileWithInvalidFileName(WritingStrategy writingStrategy) { + setWritingStrategy(writingStrategy); + runner.setProperty(AzureStorageUtils.FILE, "/file1"); runProcessor(FILE_DATA); @@ -177,8 +199,11 @@ public void testPutFileWithInvalidFileName() { assertFailure(); } - @Test - public void testPutFileWithSpacesInDirectoryAndFileName() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileWithSpacesInDirectoryAndFileName(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + String directory = "dir 1"; String fileName = "file 1"; runner.setProperty(AzureStorageUtils.DIRECTORY, directory); @@ -189,8 +214,11 @@ public void testPutFileWithSpacesInDirectoryAndFileName() throws Exception { assertSuccess(directory, fileName, FILE_DATA); } - @Test - public void testPutFileToExistingFileWithFailResolution() { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToExistingFileWithFailResolution(WritingStrategy writingStrategy) { + setWritingStrategy(writingStrategy); + fileSystemClient.createFile(String.format("%s/%s", DIRECTORY, FILE_NAME)); runProcessor(FILE_DATA); @@ -198,8 +226,11 @@ public void testPutFileToExistingFileWithFailResolution() { assertFailure(); } - @Test - public void testPutFileToExistingFileWithReplaceResolution() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToExistingFileWithReplaceResolution(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + fileSystemClient.createFile(String.format("%s/%s", DIRECTORY, FILE_NAME)); runner.setProperty(PutAzureDataLakeStorage.CONFLICT_RESOLUTION, PutAzureDataLakeStorage.REPLACE_RESOLUTION); @@ -209,8 +240,11 @@ public void testPutFileToExistingFileWithReplaceResolution() throws Exception { assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); } - @Test - public void testPutFileToExistingFileWithIgnoreResolution() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileToExistingFileWithIgnoreResolution(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + String azureFileContent = "AzureFileContent"; createDirectoryAndUploadFile(DIRECTORY, FILE_NAME, azureFileContent); @@ -221,8 +255,11 @@ public void testPutFileToExistingFileWithIgnoreResolution() throws Exception { assertSuccessWithIgnoreResolution(DIRECTORY, FILE_NAME, FILE_DATA, azureFileContent.getBytes(StandardCharsets.UTF_8)); } - @Test - public void testPutFileWithEL() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileWithEL(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + Map attributes = createAttributesMap(); setELProperties(); @@ -231,8 +268,11 @@ public void testPutFileWithEL() throws Exception { assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); } - @Test - public void testPutFileWithELButFilesystemIsNotSpecified() { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileWithELButFilesystemIsNotSpecified(WritingStrategy writingStrategy) { + setWritingStrategy(writingStrategy); + Map attributes = createAttributesMap(); attributes.remove(EL_FILESYSTEM); setELProperties(); @@ -242,8 +282,11 @@ public void testPutFileWithELButFilesystemIsNotSpecified() { assertFailure(); } - @Test - public void testPutFileWithELButFileNameIsNotSpecified() { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileWithELButFileNameIsNotSpecified(WritingStrategy writingStrategy) { + setWritingStrategy(writingStrategy); + Map attributes = createAttributesMap(); attributes.remove(EL_FILE_NAME); setELProperties(); @@ -253,8 +296,11 @@ public void testPutFileWithELButFileNameIsNotSpecified() { assertFailure(); } - @Test - public void testPutFileFromLocalFile() throws Exception { + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileFromLocalFile(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + String attributeName = "file.path"; String serviceId = FileResourceService.class.getSimpleName(); @@ -280,6 +326,19 @@ public void testPutFileFromLocalFile() throws Exception { assertProvenanceEvents(); } + @ParameterizedTest + @EnumSource(WritingStrategy.class) + public void testPutFileUsingProxy(WritingStrategy writingStrategy) throws Exception { + setWritingStrategy(writingStrategy); + + fileSystemClient.createDirectory(DIRECTORY); + configureProxyService(); + + runProcessor(FILE_DATA); + + assertSuccess(DIRECTORY, FILE_NAME, FILE_DATA); + } + private Map createAttributesMap() { Map attributes = new HashMap<>(); @@ -290,6 +349,10 @@ private Map createAttributesMap() { return attributes; } + private void setWritingStrategy(WritingStrategy writingStrategy) { + runner.setProperty(PutAzureDataLakeStorage.WRITING_STRATEGY, writingStrategy); + } + private void setELProperties() { runner.setProperty(AzureStorageUtils.FILESYSTEM, String.format("${%s}", EL_FILESYSTEM)); runner.setProperty(AzureStorageUtils.DIRECTORY, String.format("${%s}", EL_DIRECTORY)); From 15d2b49e774bce5ffeb0d976e9101e3f4170504f Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Mon, 25 Mar 2024 08:39:55 -0500 Subject: [PATCH 68/73] NIFI-12942 Upgraded Jetty from 12.0.6 to 12.0.7 Signed-off-by: Pierre Villard This closes #8554. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c91980d907cd..9004ee27d332 100644 --- a/pom.xml +++ b/pom.xml @@ -129,7 +129,7 @@ 2.0.12 2.9.0 10.17.1.0 - 12.0.6 + 12.0.7 2.17.0 1.11.3 4.0.4 From a9e246956cee822e407b20e45930e05c3fd72de6 Mon Sep 17 00:00:00 2001 From: Peter Turcsanyi Date: Fri, 22 Mar 2024 22:32:27 +0100 Subject: [PATCH 69/73] NIFI-12936 ListGCSBucket resets its tracking state after configuration change Signed-off-by: Pierre Villard This closes #8550. --- .../processors/gcp/storage/ListGCSBucket.java | 60 ++- .../gcp/storage/ListGCSBucketTest.java | 451 ++++++++++-------- 2 files changed, 298 insertions(+), 213 deletions(-) diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java index aed69ba1ee9f..97c6783a0772 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java @@ -278,40 +278,62 @@ public Set getRelationships() { return RELATIONSHIPS; } - // State tracking + private static final Set TRACKING_RESET_PROPERTIES = Set.of( + BUCKET, + PREFIX, + LISTING_STRATEGY + ); + + // used by Tracking Timestamps tracking strategy public static final String CURRENT_TIMESTAMP = "currentTimestamp"; public static final String CURRENT_KEY_PREFIX = "key-"; private volatile long currentTimestamp = 0L; private final Set currentKeys = Collections.synchronizedSet(new HashSet<>()); - private volatile boolean justElectedPrimaryNode = false; - private volatile boolean resetEntityTrackingState = false; + // used by Tracking Entities tracking strategy private volatile ListedEntityTracker listedEntityTracker; + private volatile boolean justElectedPrimaryNode = false; + private volatile boolean resetTracking = false; + @OnPrimaryNodeStateChange public void onPrimaryNodeChange(final PrimaryNodeState newState) { justElectedPrimaryNode = (newState == PrimaryNodeState.ELECTED_PRIMARY_NODE); } + @Override + public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) { + if (isConfigurationRestored() && TRACKING_RESET_PROPERTIES.contains(descriptor)) { + resetTracking = true; + } + } + @OnScheduled - public void initListedEntityTracker(ProcessContext context) { - final boolean isTrackingEntityStrategy = BY_ENTITIES.getValue().equals(context.getProperty(LISTING_STRATEGY).getValue()); - if (listedEntityTracker != null && (resetEntityTrackingState || !isTrackingEntityStrategy)) { + public void initTrackingStrategy(ProcessContext context) throws IOException { + final String listingStrategy = context.getProperty(LISTING_STRATEGY).getValue(); + final boolean isTrackingTimestampsStrategy = BY_TIMESTAMPS.getValue().equals(listingStrategy); + final boolean isTrackingEntityStrategy = BY_ENTITIES.getValue().equals(listingStrategy); + + if (resetTracking || !isTrackingTimestampsStrategy) { + context.getStateManager().clear(Scope.CLUSTER); + currentTimestamp = 0L; + currentKeys.clear(); + } + + if (listedEntityTracker != null && (resetTracking || !isTrackingEntityStrategy)) { try { listedEntityTracker.clearListedEntities(); + listedEntityTracker = null; } catch (IOException e) { throw new RuntimeException("Failed to reset previously listed entities", e); } } - resetEntityTrackingState = false; - if (isTrackingEntityStrategy) { - if (listedEntityTracker == null) { - listedEntityTracker = createListedEntityTracker(); - } - } else { - listedEntityTracker = null; + if (isTrackingEntityStrategy && listedEntityTracker == null) { + listedEntityTracker = createListedEntityTracker(); } + + resetTracking = false; } protected ListedEntityTracker createListedEntityTracker() { @@ -1027,4 +1049,16 @@ public int getCount() { return count; } } + + long getCurrentTimestamp() { + return currentTimestamp; + } + + ListedEntityTracker getListedEntityTracker() { + return listedEntityTracker; + } + + boolean isResetTracking() { + return resetTracking; + } } diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java index 82852acbfaab..f220c1be6f25 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java @@ -17,33 +17,24 @@ package org.apache.nifi.processors.gcp.storage; import com.google.api.gax.paging.Page; +import com.google.cloud.PageImpl; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import org.apache.nifi.components.AllowableValue; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateMap; +import org.apache.nifi.distributed.cache.client.DistributedMapCacheClient; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.state.MockStateManager; import org.apache.nifi.util.LogMessage; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.Captor; @@ -51,6 +42,21 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import static org.apache.nifi.processors.gcp.storage.StorageAttributes.BUCKET_ATTR; import static org.apache.nifi.processors.gcp.storage.StorageAttributes.CACHE_CONTROL_ATTR; import static org.apache.nifi.processors.gcp.storage.StorageAttributes.COMPONENT_COUNT_ATTR; @@ -75,11 +81,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -87,6 +95,7 @@ */ @MockitoSettings(strictness = Strictness.LENIENT) public class ListGCSBucketTest extends AbstractGCSTest { + private static final String PREFIX = "test-prefix"; private static final Boolean USE_GENERATIONS = true; @@ -113,16 +122,29 @@ public class ListGCSBucketTest extends AbstractGCSTest { private static final Long CREATE_TIME = 1234L; private static final Long UPDATE_TIME = 4567L; private final static Long GENERATION = 5L; + private static final long TIMESTAMP = 1234567890; @Mock Storage storage; + @Mock + Page mockBlobPage; + @Captor ArgumentCaptor argumentCaptor; - @Override - public ListGCSBucket getProcessor() { - return new ListGCSBucket() { + private TestRunner runner; + + private ListGCSBucket processor; + + private MockStateManager mockStateManager; + + @Mock + private DistributedMapCacheClient mockCache; + + @BeforeEach + public void beforeEach() throws Exception { + processor = new ListGCSBucket() { @Override protected Storage getCloudService() { return storage; @@ -133,21 +155,25 @@ protected Storage getCloudService(final ProcessContext context) { return storage; } }; + + runner = buildNewRunner(processor); + runner.setProperty(ListGCSBucket.BUCKET, BUCKET); + runner.assertValid(); + + mockStateManager = runner.getStateManager(); + } + + @Override + public ListGCSBucket getProcessor() { + return processor; } @Override protected void addRequiredPropertiesToRunner(TestRunner runner) { - runner.setProperty(ListGCSBucket.BUCKET, BUCKET); } @Test public void testRestoreFreshState() throws Exception { - reset(storage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - assertFalse(runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).getStateVersion().isPresent(), "Cluster StateMap should be fresh (version -1L)"); assertTrue(processor.getStateKeys().isEmpty()); @@ -157,17 +183,10 @@ public void testRestoreFreshState() throws Exception { assertEquals(0L, processor.getStateTimestamp()); assertTrue(processor.getStateKeys().isEmpty()); - } @Test public void testRestorePreviousState() throws Exception { - reset(storage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - final Map state = new LinkedHashMap<>(); state.put(ListGCSBucket.CURRENT_TIMESTAMP, String.valueOf(4L)); state.put(ListGCSBucket.CURRENT_KEY_PREFIX + "0", "test-key-0"); @@ -188,12 +207,6 @@ public void testRestorePreviousState() throws Exception { @Test public void testPersistState() throws Exception { - reset(storage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - assertFalse( runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).getStateVersion().isPresent(), "Cluster StateMap should be fresh" @@ -215,13 +228,7 @@ public void testPersistState() throws Exception { } @Test - public void testFailedPersistState() throws Exception { - reset(storage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testFailedPersistState() { runner.getStateManager().setFailOnStateSet(Scope.CLUSTER, true); final Set keys = new HashSet<>(Arrays.asList("test-key-0", "test-key-1")); @@ -241,60 +248,10 @@ public void testFailedPersistState() throws Exception { // We could do more specific things like check the contents of the LogMessage, // but that seems too nitpicky. - - } - - @Mock - Page mockBlobPage; - - private Blob buildMockBlob(final String bucket, final String key, final long updateTime) { - final Blob blob = mock(Blob.class); - when(blob.getBucket()).thenReturn(bucket); - when(blob.getName()).thenReturn(key); - when(blob.getUpdateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); - when(blob.getCreateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); - return blob; - } - - private Blob buildMockBlobWithoutBucket(final String bucket, final String key, final long updateTime) { - final Blob blob = mock(Blob.class); - when(blob.getName()).thenReturn(key); - when(blob.getUpdateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); - when(blob.getCreateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); - return blob; - } - - private OffsetDateTime offsetDateTime(final long value) { - final Instant instant = Instant.ofEpochMilli(value); - final LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); - return OffsetDateTime.of(localDateTime, ZoneOffset.UTC); - } - - private void verifyConfigVerification(final TestRunner runner, final ListGCSBucket processor, final int expectedCount) { - final List verificationResults = processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap()); - assertEquals(3, verificationResults.size()); - final ConfigVerificationResult cloudServiceResult = verificationResults.get(0); - assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, cloudServiceResult.getOutcome()); - - final ConfigVerificationResult iamPermissionsResult = verificationResults.get(1); - assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, iamPermissionsResult.getOutcome()); - - final ConfigVerificationResult listingResult = verificationResults.get(2); - assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, listingResult.getOutcome()); - - assertTrue( - listingResult.getExplanation().matches(String.format(".*finding %s blobs.*", expectedCount)), - String.format("Expected %s blobs to be counted, but explanation was: %s", expectedCount, listingResult.getExplanation())); } @Test - public void testSuccessfulList() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testSuccessfulList() { final Iterable mockList = Arrays.asList( buildMockBlob("blob-bucket-1", "blob-key-1", 2L), buildMockBlob("blob-bucket-2", "blob-key-2", 3L) @@ -331,11 +288,7 @@ public void testSuccessfulList() throws Exception { } @Test - public void testNoTrackingListing() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); + public void testNoTrackingListing() { runner.setProperty(ListGCSBucket.LISTING_STRATEGY, ListGCSBucket.NO_TRACKING); runner.assertValid(); @@ -381,12 +334,6 @@ public void testNoTrackingListing() throws Exception { @Test public void testOldValues() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - final Iterable mockList = Collections.singletonList( buildMockBlob("blob-bucket-1", "blob-key-1", 2L) ); @@ -409,16 +356,8 @@ public void testOldValues() throws Exception { assertEquals("2", runner.getStateManager().getState(Scope.CLUSTER).get(ListGCSBucket.CURRENT_TIMESTAMP)); } - - @Test public void testEmptyList() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - final Iterable mockList = Collections.emptyList(); when(mockBlobPage.getValues()).thenReturn(mockList); @@ -438,12 +377,6 @@ public void testEmptyList() throws Exception { @Test public void testListWithStateAndFilesComingInAlphabeticalOrder() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - final Map state = new LinkedHashMap<>(); state.put(ListGCSBucket.CURRENT_TIMESTAMP, String.valueOf(1L)); state.put(ListGCSBucket.CURRENT_KEY_PREFIX + "0", "blob-key-1"); @@ -482,12 +415,6 @@ public void testListWithStateAndFilesComingInAlphabeticalOrder() throws Exceptio @Test public void testListWithStateAndFilesComingNotInAlphabeticalOrder() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - final Map state = new LinkedHashMap<>(); state.put(ListGCSBucket.CURRENT_TIMESTAMP, String.valueOf(1L)); state.put(ListGCSBucket.CURRENT_KEY_PREFIX + "0", "blob-key-2"); @@ -531,12 +458,6 @@ public void testListWithStateAndFilesComingNotInAlphabeticalOrder() throws Excep @Test public void testListWithStateAndNewFilesComingWithTheSameTimestamp() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - final Map state = new LinkedHashMap<>(); state.put(ListGCSBucket.CURRENT_TIMESTAMP, String.valueOf(1L)); state.put(ListGCSBucket.CURRENT_KEY_PREFIX + "0", "blob-key-2"); @@ -586,12 +507,6 @@ public void testListWithStateAndNewFilesComingWithTheSameTimestamp() throws Exce @Test public void testListWithStateAndNewFilesComingWithTheCurrentTimestamp() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - final Map state = new LinkedHashMap<>(); state.put(ListGCSBucket.CURRENT_TIMESTAMP, String.valueOf(1L)); state.put(ListGCSBucket.CURRENT_KEY_PREFIX + "0", "blob-key-2"); @@ -639,13 +554,7 @@ public void testListWithStateAndNewFilesComingWithTheCurrentTimestamp() throws E } @Test - public void testAttributesSet() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testAttributesSet() { final Blob blob = buildMockBlob("test-bucket-1", "test-key-1", 2L); when(blob.getSize()).thenReturn(SIZE); when(blob.getCacheControl()).thenReturn(CACHE_CONTROL); @@ -705,13 +614,7 @@ public void testAttributesSet() throws Exception { } @Test - public void testAclOwnerUser() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testAclOwnerUser() { final Blob blob = buildMockBlob("test-bucket-1", "test-key-1", 2L); final Acl.User mockUser = mock(Acl.User.class); when(mockUser.getEmail()).thenReturn(OWNER_USER_EMAIL); @@ -734,15 +637,8 @@ public void testAclOwnerUser() throws Exception { assertEquals("user", flowFile.getAttribute(OWNER_TYPE_ATTR)); } - @Test - public void testAclOwnerGroup() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testAclOwnerGroup() { final Blob blob = buildMockBlob("test-bucket-1", "test-key-1", 2L); final Acl.Group mockGroup = mock(Acl.Group.class); when(mockGroup.getEmail()).thenReturn(OWNER_GROUP_EMAIL); @@ -765,16 +661,8 @@ public void testAclOwnerGroup() throws Exception { assertEquals("group", flowFile.getAttribute(OWNER_TYPE_ATTR)); } - - @Test - public void testAclOwnerDomain() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testAclOwnerDomain() { final Blob blob = buildMockBlob("test-bucket-1", "test-key-1", 2L); final Acl.Domain mockDomain = mock(Acl.Domain.class); when(mockDomain.getDomain()).thenReturn(OWNER_DOMAIN); @@ -796,16 +684,8 @@ public void testAclOwnerDomain() throws Exception { assertEquals("domain", flowFile.getAttribute(OWNER_TYPE_ATTR)); } - - @Test - public void testAclOwnerProject() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testAclOwnerProject() { final Blob blob = buildMockBlob("test-bucket-1", "test-key-1", 2L); final Acl.Project mockProject = mock(Acl.Project.class); when(mockProject.getProjectId()).thenReturn(OWNER_PROJECT_ID); @@ -828,15 +708,8 @@ public void testAclOwnerProject() throws Exception { assertEquals("project", flowFile.getAttribute(OWNER_TYPE_ATTR)); } - @Test - public void testYieldOnBadStateRestore() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - runner.assertValid(); - + public void testYieldOnBadStateRestore() { final Iterable mockList = Collections.emptyList(); runner.getStateManager().setFailOnStateGet(Scope.CLUSTER, true); @@ -848,12 +721,7 @@ public void testYieldOnBadStateRestore() throws Exception { } @Test - public void testListOptionsPrefix() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - + public void testListOptionsPrefix() { runner.setProperty(ListGCSBucket.PREFIX, PREFIX); runner.assertValid(); @@ -869,14 +737,8 @@ public void testListOptionsPrefix() throws Exception { assertEquals(Storage.BlobListOption.prefix(PREFIX), argumentCaptor.getValue()); } - @Test - public void testListOptionsVersions() throws Exception { - reset(storage, mockBlobPage); - final ListGCSBucket processor = getProcessor(); - final TestRunner runner = buildNewRunner(processor); - addRequiredPropertiesToRunner(runner); - + public void testListOptionsVersions() { runner.setProperty(ListGCSBucket.USE_GENERATIONS, String.valueOf(USE_GENERATIONS)); runner.assertValid(); @@ -892,4 +754,193 @@ public void testListOptionsVersions() throws Exception { Storage.BlobListOption option = argumentCaptor.getValue(); assertEquals(Storage.BlobListOption.versions(true), option); } + + @Test + void testResetTimestampTrackingWhenBucketModified() throws Exception { + setUpResetTrackingTest(ListGCSBucket.BY_TIMESTAMPS); + + assertFalse(processor.isResetTracking()); + + runner.run(); + + assertEquals(TIMESTAMP, processor.getCurrentTimestamp()); + + runner.setProperty(ListGCSBucket.BUCKET, "otherBucket"); + + assertTrue(processor.isResetTracking()); + + runner.run(); + + assertEquals(0, processor.getCurrentTimestamp()); + mockStateManager.assertStateNotSet(ListGCSBucket.CURRENT_TIMESTAMP, Scope.CLUSTER); + + assertFalse(processor.isResetTracking()); + } + + @Test + void testResetTimestampTrackingWhenPrefixModified() throws Exception { + setUpResetTrackingTest(ListGCSBucket.BY_TIMESTAMPS); + + assertFalse(processor.isResetTracking()); + + runner.run(); + + assertEquals(TIMESTAMP, processor.getCurrentTimestamp()); + + runner.setProperty(ListGCSBucket.PREFIX, "prefix2"); + + assertTrue(processor.isResetTracking()); + + runner.run(); + + assertEquals(0, processor.getCurrentTimestamp()); + mockStateManager.assertStateNotSet(ListGCSBucket.CURRENT_TIMESTAMP, Scope.CLUSTER); + + assertFalse(processor.isResetTracking()); + } + + @Test + void testResetTimestampTrackingWhenStrategyModified() throws Exception { + setUpResetTrackingTest(ListGCSBucket.BY_TIMESTAMPS); + + assertFalse(processor.isResetTracking()); + + runner.run(); + + assertEquals(TIMESTAMP, processor.getCurrentTimestamp()); + + runner.setProperty(ListGCSBucket.LISTING_STRATEGY, ListGCSBucket.NO_TRACKING); + + assertTrue(processor.isResetTracking()); + + runner.run(); + + assertEquals(0, processor.getCurrentTimestamp()); + mockStateManager.assertStateNotSet(ListGCSBucket.CURRENT_TIMESTAMP, Scope.CLUSTER); + + assertFalse(processor.isResetTracking()); + } + + @Test + void testResetEntityTrackingWhenBucketModified() throws Exception { + setUpResetTrackingTest(ListGCSBucket.BY_ENTITIES); + + assertFalse(processor.isResetTracking()); + + runner.run(); + + assertNotNull(processor.getListedEntityTracker()); + + runner.setProperty(ListGCSBucket.BUCKET, "otherBucket"); + + assertTrue(processor.isResetTracking()); + + runner.run(); + + assertNotNull(processor.getListedEntityTracker()); + verify(mockCache).remove(eq("ListedEntities::" + processor.getIdentifier()), any()); + + assertFalse(processor.isResetTracking()); + } + + @Test + void testResetEntityTrackingWhenPrefixModified() throws Exception { + setUpResetTrackingTest(ListGCSBucket.BY_ENTITIES); + + assertFalse(processor.isResetTracking()); + + runner.run(); + + assertNotNull(processor.getListedEntityTracker()); + + runner.setProperty(ListGCSBucket.PREFIX, "prefix2"); + + assertTrue(processor.isResetTracking()); + + runner.run(); + + assertNotNull(processor.getListedEntityTracker()); + verify(mockCache).remove(eq("ListedEntities::" + processor.getIdentifier()), any()); + + assertFalse(processor.isResetTracking()); + } + + @Test + void testResetEntityTrackingWhenStrategyModified() throws Exception { + setUpResetTrackingTest(ListGCSBucket.BY_ENTITIES); + + assertFalse(processor.isResetTracking()); + + runner.run(); + + assertNotNull(processor.getListedEntityTracker()); + + runner.setProperty(ListGCSBucket.LISTING_STRATEGY, ListGCSBucket.NO_TRACKING); + + assertTrue(processor.isResetTracking()); + + runner.run(); + + assertNull(processor.getListedEntityTracker()); + verify(mockCache).remove(eq("ListedEntities::" + processor.getIdentifier()), any()); + + assertFalse(processor.isResetTracking()); + } + + private void setUpResetTrackingTest(AllowableValue listingStrategy) throws Exception { + runner.setProperty(ListGCSBucket.LISTING_STRATEGY, listingStrategy); + runner.setProperty(ListGCSBucket.PREFIX, "prefix1"); + + if (listingStrategy == ListGCSBucket.BY_TIMESTAMPS) { + mockStateManager.setState(Map.of(ListGCSBucket.CURRENT_TIMESTAMP, Long.toString(TIMESTAMP), ListGCSBucket.CURRENT_KEY_PREFIX + "0", "file"), Scope.CLUSTER); + } else if (listingStrategy == ListGCSBucket.BY_ENTITIES) { + String serviceId = "DistributedMapCacheClient"; + when(mockCache.getIdentifier()).thenReturn(serviceId); + runner.addControllerService(serviceId, mockCache); + runner.enableControllerService(mockCache); + runner.setProperty(ListGCSBucket.TRACKING_STATE_CACHE, serviceId); + } + + when(storage.list(anyString(), any(Storage.BlobListOption.class))).thenReturn(new PageImpl<>(null, null, null)); + } + + private Blob buildMockBlob(final String bucket, final String key, final long updateTime) { + final Blob blob = mock(Blob.class); + when(blob.getBucket()).thenReturn(bucket); + when(blob.getName()).thenReturn(key); + when(blob.getUpdateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); + when(blob.getCreateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); + return blob; + } + + private Blob buildMockBlobWithoutBucket(final String bucket, final String key, final long updateTime) { + final Blob blob = mock(Blob.class); + when(blob.getName()).thenReturn(key); + when(blob.getUpdateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); + when(blob.getCreateTimeOffsetDateTime()).thenReturn(offsetDateTime(updateTime)); + return blob; + } + + private OffsetDateTime offsetDateTime(final long value) { + final Instant instant = Instant.ofEpochMilli(value); + final LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); + return OffsetDateTime.of(localDateTime, ZoneOffset.UTC); + } + + private void verifyConfigVerification(final TestRunner runner, final ListGCSBucket processor, final int expectedCount) { + final List verificationResults = processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap()); + assertEquals(3, verificationResults.size()); + final ConfigVerificationResult cloudServiceResult = verificationResults.get(0); + assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, cloudServiceResult.getOutcome()); + + final ConfigVerificationResult iamPermissionsResult = verificationResults.get(1); + assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, iamPermissionsResult.getOutcome()); + + final ConfigVerificationResult listingResult = verificationResults.get(2); + assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, listingResult.getOutcome()); + + assertTrue( + listingResult.getExplanation().matches(String.format(".*finding %s blobs.*", expectedCount)), + String.format("Expected %s blobs to be counted, but explanation was: %s", expectedCount, listingResult.getExplanation())); + } } \ No newline at end of file From dd9d1c978f7d4854ac0d53140f72f7c96c4e1d71 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Mon, 25 Mar 2024 09:06:27 -0500 Subject: [PATCH 70/73] NIFI-12943 Upgraded Hadoop from 3.3.6 to 3.4.0 This closes #8556 Signed-off-by: Joseph Witt --- .../nifi-extension-utils/nifi-hadoop-utils/pom.xml | 9 +++++++++ .../nifi-record-utils/nifi-hadoop-record-utils/pom.xml | 9 +++++++++ .../nifi-hadoop-bundle/nifi-hdfs-processors/pom.xml | 9 +++++++++ .../nifi-hadoop-libraries-nar/pom.xml | 9 +++++++++ nifi-nar-bundles/nifi-hive-bundle/pom.xml | 5 +++++ nifi-nar-bundles/nifi-iceberg-bundle/pom.xml | 5 +++++ .../nifi-parquet-bundle/nifi-parquet-processors/pom.xml | 9 +++++++++ .../nifi-hadoop-dbcp-service/pom.xml | 5 +++++ pom.xml | 2 +- 9 files changed, 61 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/pom.xml index cf1d9242a36a..d1e4968f83cd 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/pom.xml +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/pom.xml @@ -76,8 +76,17 @@ commons-logging commons-logging + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcprov-jdk18on + org.apache.hadoop hadoop-auth diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/pom.xml index 978495d3b37e..e01ed8aa80de 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/pom.xml +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/pom.xml @@ -64,8 +64,17 @@ commons-logging commons-logging + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcprov-jdk18on + org.slf4j jcl-over-slf4j diff --git a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/pom.xml b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/pom.xml index 357f057dbce7..420086edde12 100644 --- a/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/pom.xml +++ b/nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/pom.xml @@ -67,8 +67,17 @@ commons-logging commons-logging + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcprov-jdk18on + org.apache.hadoop hadoop-hdfs diff --git a/nifi-nar-bundles/nifi-hadoop-libraries-bundle/nifi-hadoop-libraries-nar/pom.xml b/nifi-nar-bundles/nifi-hadoop-libraries-bundle/nifi-hadoop-libraries-nar/pom.xml index 2d2a8c1f492d..a0d2c466a41e 100644 --- a/nifi-nar-bundles/nifi-hadoop-libraries-bundle/nifi-hadoop-libraries-nar/pom.xml +++ b/nifi-nar-bundles/nifi-hadoop-libraries-bundle/nifi-hadoop-libraries-nar/pom.xml @@ -73,8 +73,17 @@ org.eclipse.jetty jetty-rewrite + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcprov-jdk18on + org.slf4j log4j-over-slf4j diff --git a/nifi-nar-bundles/nifi-hive-bundle/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/pom.xml index 48c41e008998..1c7544f072d9 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-hive-bundle/pom.xml @@ -245,6 +245,11 @@ org.eclipse.jetty jetty-rewrite + + + org.bouncycastle + bcprov-jdk15on + diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml index f6c96fb45be1..fdcbc880f053 100644 --- a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml @@ -147,6 +147,11 @@ org.eclipse.jetty jetty-rewrite + + + org.bouncycastle + bcprov-jdk15on + diff --git a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/pom.xml b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/pom.xml index fe09b2e5fd90..e8cb2ced4667 100644 --- a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/pom.xml +++ b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/pom.xml @@ -77,8 +77,17 @@ org.eclipse.jetty.websocket websocket-client + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcprov-jdk18on + com.fasterxml.jackson.core jackson-databind diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml index 4f23161056b6..37a03706729d 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-hadoop-dbcp-service-bundle/nifi-hadoop-dbcp-service/pom.xml @@ -87,6 +87,11 @@ commons-logging commons-logging + + + org.bouncycastle + bcprov-jdk15on + diff --git a/pom.xml b/pom.xml index 9004ee27d332..fa5b6f86dc1c 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ 2.5.0 4.0.20 3.2.5 - 3.3.6 + 3.4.0 1.2.1 2.1.5 1.9.21 From 258715539e7cf136152625aeafee5a2f8673eacd Mon Sep 17 00:00:00 2001 From: Ricardo Ferreira Date: Thu, 21 Mar 2024 16:34:21 +0000 Subject: [PATCH 71/73] NIFI-12944 - Add PeerAddress as Attribute into the flowfile Signed-off-by: Pierre Villard This closes #8557. --- .../snmp/operations/SNMPTrapReceiver.java | 10 ++++++-- .../snmp/operations/SNMPTrapReceiverTest.java | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/main/java/org/apache/nifi/snmp/operations/SNMPTrapReceiver.java b/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/main/java/org/apache/nifi/snmp/operations/SNMPTrapReceiver.java index d2eaf19528fa..c93bb83e4681 100644 --- a/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/main/java/org/apache/nifi/snmp/operations/SNMPTrapReceiver.java +++ b/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/main/java/org/apache/nifi/snmp/operations/SNMPTrapReceiver.java @@ -25,6 +25,7 @@ import org.snmp4j.CommandResponderEvent; import org.snmp4j.PDU; import org.snmp4j.PDUv1; +import org.snmp4j.smi.Address; import java.util.Map; @@ -46,7 +47,7 @@ public void processPdu(final CommandResponderEvent event) { final PDU pdu = event.getPDU(); if (isValidTrapPdu(pdu)) { final ProcessSession processSession = processSessionFactory.createSession(); - final FlowFile flowFile = createFlowFile(processSession, pdu); + final FlowFile flowFile = createFlowFile(processSession,event); processSession.getProvenanceReporter().create(flowFile, event.getPeerAddress() + "/" + pdu.getRequestID()); if (pdu.getErrorStatus() == PDU.noError) { processSession.transfer(flowFile, REL_SUCCESS); @@ -59,14 +60,19 @@ public void processPdu(final CommandResponderEvent event) { } } - private FlowFile createFlowFile(final ProcessSession processSession, final PDU pdu) { + private FlowFile createFlowFile(final ProcessSession processSession, final CommandResponderEvent event) { FlowFile flowFile = processSession.create(); final Map attributes; + final PDU pdu = event.getPDU(); + final Address peerAddress = event.getPeerAddress(); if (pdu instanceof PDUv1) { attributes = SNMPUtils.getV1TrapPduAttributeMap((PDUv1) pdu); } else { attributes = SNMPUtils.getPduAttributeMap(pdu); } + if (peerAddress.isValid()) { + processSession.putAttribute(flowFile, SNMPUtils.SNMP_PROP_PREFIX + "peerAddress", peerAddress.toString()); + } flowFile = processSession.putAllAttributes(flowFile, attributes); return flowFile; } diff --git a/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/test/java/org/apache/nifi/snmp/operations/SNMPTrapReceiverTest.java b/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/test/java/org/apache/nifi/snmp/operations/SNMPTrapReceiverTest.java index eeca40eefe38..0d273c3921f7 100644 --- a/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/test/java/org/apache/nifi/snmp/operations/SNMPTrapReceiverTest.java +++ b/nifi-nar-bundles/nifi-snmp-bundle/nifi-snmp-processors/src/test/java/org/apache/nifi/snmp/operations/SNMPTrapReceiverTest.java @@ -29,6 +29,7 @@ import org.snmp4j.CommandResponderEvent; import org.snmp4j.PDU; import org.snmp4j.PDUv1; +import org.snmp4j.smi.Address; import org.snmp4j.smi.OID; import org.snmp4j.smi.VariableBinding; @@ -95,9 +96,15 @@ void testTrapReceiverCreatesTrapPduV1FlowFile() { when(mockV1Pdu.getType()).thenReturn(PDU.V1TRAP); when(mockV1Pdu.getEnterprise()).thenReturn(new OID("1.3.6.1.2.1.1.1.0")); when(mockV1Pdu.getSpecificTrap()).thenReturn(4); + + final Address mockAddress = mock(Address.class); + when(mockAddress.toString()).thenReturn("127.0.0.1/62"); + when(mockAddress.isValid()).thenReturn(true); + final Vector vbs = new Vector<>(); doReturn(vbs).when(mockV1Pdu).getVariableBindings(); when(mockEvent.getPDU()).thenReturn(mockV1Pdu); + when(mockEvent.getPeerAddress()).thenReturn(mockAddress); when(mockProcessSessionFactory.createSession()).thenReturn(mockProcessSession); snmpTrapReceiver.processPdu(mockEvent); @@ -107,6 +114,8 @@ void testTrapReceiverCreatesTrapPduV1FlowFile() { assertEquals("1.3.6.1.2.1.1.1.0", flowFile.getAttribute("snmp$enterprise")); assertEquals(String.valueOf(4), flowFile.getAttribute("snmp$specificTrapType")); + assertEquals("127.0.0.1/62", flowFile.getAttribute("snmp$peerAddress")); + } @Test @@ -117,8 +126,15 @@ void testTrapReceiverCreatesTrapPduV2FlowFile() { when(mockPdu.getErrorIndex()).thenReturn(123); when(mockPdu.getErrorStatusText()).thenReturn("test error status text"); final Vector vbs = new Vector<>(); + + final Address mockAddress = mock(Address.class); + when(mockAddress.toString()).thenReturn("127.0.0.1/62"); + when(mockAddress.isValid()).thenReturn(true); + doReturn(vbs).when(mockPdu).getVariableBindings(); when(mockEvent.getPDU()).thenReturn(mockPdu); + when(mockEvent.getPeerAddress()).thenReturn(mockAddress); + when(mockProcessSessionFactory.createSession()).thenReturn(mockProcessSession); snmpTrapReceiver.processPdu(mockEvent); @@ -128,6 +144,7 @@ void testTrapReceiverCreatesTrapPduV2FlowFile() { assertEquals(String.valueOf(123), flowFile.getAttribute("snmp$errorIndex")); assertEquals("test error status text", flowFile.getAttribute("snmp$errorStatusText")); + assertEquals("127.0.0.1/62", flowFile.getAttribute("snmp$peerAddress")); } @Test @@ -136,9 +153,14 @@ void testReceiveTrapWithErrorGetsTransferredToFailure() { when(mockPdu.getType()).thenReturn(PDU.TRAP); when(mockPdu.getErrorStatus()).thenReturn(PDU.badValue); + + final Address mockAddress = mock(Address.class); + when(mockAddress.isValid()).thenReturn(false); + final Vector vbs = new Vector<>(); doReturn(vbs).when(mockPdu).getVariableBindings(); when(mockEvent.getPDU()).thenReturn(mockPdu); + when(mockEvent.getPeerAddress()).thenReturn(mockAddress); when(mockProcessSessionFactory.createSession()).thenReturn(mockProcessSession); snmpTrapReceiver.processPdu(mockEvent); @@ -149,5 +171,6 @@ void testReceiveTrapWithErrorGetsTransferredToFailure() { final FlowFile flowFile = flowFiles.get(0); assertEquals(String.valueOf(PDU.badValue), flowFile.getAttribute("snmp$errorStatus")); + assertEquals(null, flowFile.getAttribute("snmp$peerAddress")); } } From 08ff54f5fb712d01f372fd25a2d904d605e271ba Mon Sep 17 00:00:00 2001 From: Jim Steinebrey Date: Tue, 19 Mar 2024 11:54:38 -0400 Subject: [PATCH 72/73] Add auto commit property to QueryDatabaseTable and QueryDatabaseTable processors to allow disabling auto commit so PostgreSQL Fetch Size will work NIFI-1931 Add proper default value for auto commit (false) to PostgreSQLDatabaseAdapter to allow FETCH_SIZE to be honored on reads. NIFI-1931 Added customValidate code to check the auto commit property setting against the db adapter's required auto commit setting and give validation error message if they do not match. NIFI-1931 Added automated test to check the Auto Commit customValidate error message. NIFI-1931 remove clearDefaultValue() because it is not needed since required = false a;ready defaults it to null. This closes #8534 Signed-off-by: Matt Burgess --- .../standard/AbstractQueryDatabaseTable.java | 71 +++++++++++- .../standard/QueryDatabaseTable.java | 1 + .../standard/QueryDatabaseTableRecord.java | 1 + .../standard/db/DatabaseAdapter.java | 13 +++ .../db/impl/PostgreSQLDatabaseAdapter.java | 18 ++++ .../standard/QueryDatabaseTableIT.java | 78 ++++++++++++++ .../standard/QueryDatabaseTableRecordIT.java | 78 ++++++++++++++ .../QueryDatabaseTableRecordTest.java | 97 +++++++++++++++-- .../standard/QueryDatabaseTableTest.java | 101 ++++++++++++++++-- .../impl/TestPostgreSQLDatabaseAdapter.java | 16 +++ 10 files changed, 456 insertions(+), 18 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableIT.java create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordIT.java diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractQueryDatabaseTable.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractQueryDatabaseTable.java index e5fc6745d609..7f7a870fb73d 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractQueryDatabaseTable.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractQueryDatabaseTable.java @@ -90,16 +90,34 @@ public abstract class AbstractQueryDatabaseTable extends AbstractDatabaseFetchPr "TRANSACTION_SERIALIZABLE" ); + private static final String FETCH_SIZE_NAME = "Fetch Size"; + private static final String AUTO_COMMIT_NAME = "Set Auto Commit"; + public static final PropertyDescriptor FETCH_SIZE = new PropertyDescriptor.Builder() - .name("Fetch Size") + .name(FETCH_SIZE_NAME) .description("The number of result rows to be fetched from the result set at a time. This is a hint to the database driver and may not be " - + "honored and/or exact. If the value specified is zero, then the hint is ignored.") + + "honored and/or exact. If the value specified is zero, then the hint is ignored. " + + "If using PostgreSQL, then '" + AUTO_COMMIT_NAME + "' must be equal to 'false' to cause '" + FETCH_SIZE_NAME + "' to take effect.") .defaultValue("0") .required(true) .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) .build(); + public static final PropertyDescriptor AUTO_COMMIT = new PropertyDescriptor.Builder() + .name(AUTO_COMMIT_NAME) + .description("Allows enabling or disabling the auto commit functionality of the DB connection. Default value is 'No value set'. " + + "'No value set' will leave the db connection's auto commit mode unchanged. " + + "For some JDBC drivers such as PostgreSQL driver, it is required to disable the auto commit functionality " + + "to get the '" + FETCH_SIZE_NAME + "' setting to take effect. " + + "When auto commit is enabled, PostgreSQL driver ignores '" + FETCH_SIZE_NAME + "' setting and loads all rows of the result set to memory at once. " + + "This could lead for a large amount of memory usage when executing queries which fetch large data sets. " + + "More Details of this behaviour in PostgreSQL driver can be found in https://jdbc.postgresql.org//documentation/head/query.html.") + .allowableValues("true", "false") + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .required(false) + .build(); + public static final PropertyDescriptor MAX_ROWS_PER_FLOW_FILE = new PropertyDescriptor.Builder() .name("qdbt-max-rows") .displayName("Max Rows Per Flow File") @@ -196,6 +214,23 @@ protected Collection customValidate(ValidationContext validati .build()); } + final Boolean propertyAutoCommit = validationContext.getProperty(AUTO_COMMIT).evaluateAttributeExpressions().asBoolean(); + final Integer fetchSize = validationContext.getProperty(FETCH_SIZE).evaluateAttributeExpressions().asInteger(); + final DatabaseAdapter dbAdapter = dbAdapters.get(validationContext.getProperty(DB_TYPE).getValue()); + final Boolean adapterAutoCommit = dbAdapter == null + ? null + : dbAdapter.getAutoCommitForReads(fetchSize).orElse(null); + if (adapterAutoCommit != null && propertyAutoCommit != null + && propertyAutoCommit != adapterAutoCommit ) { + results.add(new ValidationResult.Builder().valid(false) + .subject(AUTO_COMMIT.getDisplayName()) + .input(String.valueOf(propertyAutoCommit)) + .explanation(String.format("'%s' must be set to '%s' because '%s' %s requires it to be '%s'", + AUTO_COMMIT.getDisplayName(), adapterAutoCommit, + dbAdapter.getName(), DB_TYPE.getDisplayName(), adapterAutoCommit)) + .build()); + } + return results; } @@ -304,7 +339,7 @@ public void onTrigger(final ProcessContext context, final ProcessSessionFactory } } } catch (final Exception e) { - logger.error("Unable to execute SQL select query {} due to {}", new Object[]{selectMaxQuery, e}); + logger.error("Unable to execute SQL select query {} due to {}", selectMaxQuery, e); context.yield(); } } @@ -343,6 +378,24 @@ public void onTrigger(final ProcessContext context, final ProcessSessionFactory if (logger.isDebugEnabled()) { logger.debug("Executing query {}", new Object[] { selectQuery }); } + + final boolean originalAutoCommit = con.getAutoCommit(); + final Boolean propertyAutoCommitValue = context.getProperty(AUTO_COMMIT).evaluateAttributeExpressions().asBoolean(); + // If user sets AUTO_COMMIT property to non-null (i.e. true or false), then the property value overrides the dbAdapter's value + final Boolean setAutoCommitValue = + dbAdapter == null || propertyAutoCommitValue != null + ? propertyAutoCommitValue + : dbAdapter.getAutoCommitForReads(fetchSize).orElse(null); + if (setAutoCommitValue != null && originalAutoCommit != setAutoCommitValue) { + try { + con.setAutoCommit(setAutoCommitValue); + logger.debug("Driver connection changed to setAutoCommit({})", setAutoCommitValue); + } catch (Exception ex) { + logger.debug("Failed to setAutoCommit({}) due to {}: {}", + setAutoCommitValue, ex.getClass().getName(), ex.getMessage()); + } + } + try (final ResultSet resultSet = st.executeQuery(selectQuery)) { int fragmentIndex=0; // Max values will be updated in the state property map by the callback @@ -441,12 +494,22 @@ public void onTrigger(final ProcessContext context, final ProcessSessionFactory } } catch (final SQLException e) { throw e; + } finally { + if (con.getAutoCommit() != originalAutoCommit) { + try { + con.setAutoCommit(originalAutoCommit); + logger.debug("Driver connection reset to original setAutoCommit({})", originalAutoCommit); + } catch (Exception ex) { + logger.debug("Failed to setAutoCommit({}) due to {}: {}", + originalAutoCommit, ex.getClass().getName(), ex.getMessage()); + } + } } session.transfer(resultSetFlowFiles, REL_SUCCESS); } catch (final ProcessException | SQLException e) { - logger.error("Unable to execute SQL select query {} due to {}", new Object[]{selectQuery, e}); + logger.error("Unable to execute SQL select query {} due to {}", selectQuery, e); if (!resultSetFlowFiles.isEmpty()) { session.remove(resultSetFlowFiles); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java index 24123729b869..51fbc41409ff 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java @@ -109,6 +109,7 @@ public QueryDatabaseTable() { pds.add(INITIAL_LOAD_STRATEGY); pds.add(QUERY_TIMEOUT); pds.add(FETCH_SIZE); + pds.add(AUTO_COMMIT); pds.add(MAX_ROWS_PER_FLOW_FILE); pds.add(OUTPUT_BATCH_SIZE); pds.add(MAX_FRAGMENTS); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecord.java index 5838d7e46c69..2004649976ae 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecord.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecord.java @@ -208,6 +208,7 @@ public QueryDatabaseTableRecord() { pds.add(INITIAL_LOAD_STRATEGY); pds.add(QUERY_TIMEOUT); pds.add(FETCH_SIZE); + pds.add(AUTO_COMMIT); pds.add(MAX_ROWS_PER_FLOW_FILE); pds.add(OUTPUT_BATCH_SIZE); pds.add(MAX_FRAGMENTS); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/DatabaseAdapter.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/DatabaseAdapter.java index ab661998edf4..65b43ff8b7f8 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/DatabaseAdapter.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/DatabaseAdapter.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; /** @@ -211,6 +212,18 @@ default List getAlterTableStatements(String tableName, List getAutoCommitForReads(Integer fetchSize) { + return Optional.empty(); + } + default String getSQLForDataType(int sqlType) { return JDBCType.valueOf(sqlType).getName(); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/impl/PostgreSQLDatabaseAdapter.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/impl/PostgreSQLDatabaseAdapter.java index 5e488186009f..8ba7b64b22f9 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/impl/PostgreSQLDatabaseAdapter.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/db/impl/PostgreSQLDatabaseAdapter.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import static java.sql.Types.CHAR; @@ -160,6 +161,23 @@ public List getAlterTableStatements(final String tableName, final List getAutoCommitForReads(Integer fetchSize) { + if (fetchSize != null && fetchSize != 0) { + return Optional.of(Boolean.FALSE); + } + return Optional.empty(); + } + @Override public String getSQLForDataType(int sqlType) { switch (sqlType) { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableIT.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableIT.java new file mode 100644 index 000000000000..602a87e9455d --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableIT.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.standard; + +import org.apache.nifi.dbcp.DBCPConnectionPool; +import org.apache.nifi.dbcp.utils.DBCPProperties; +import org.apache.nifi.processors.standard.db.DatabaseAdapter; +import org.apache.nifi.processors.standard.db.impl.PostgreSQLDatabaseAdapter; +import org.apache.nifi.reporting.InitializationException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; + +import java.io.IOException; +import java.sql.SQLException; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class QueryDatabaseTableIT extends QueryDatabaseTableTest { + private static PostgreSQLContainer postgres; + + @BeforeAll + public static void setupBeforeClass() { + postgres = new PostgreSQLContainer<>("postgres:9.6.12") + .withInitScript("PutDatabaseRecordIT/create-person-table.sql"); + postgres.start(); + } + + @AfterAll + public static void cleanUpAfterClass() { + if (postgres != null) { + postgres.close(); + postgres = null; + } + } + + @Override + public DatabaseAdapter createDatabaseAdapter() { + return new PostgreSQLDatabaseAdapter(); + } + + @Override + public void createDbcpControllerService() throws InitializationException { + final DBCPConnectionPool connectionPool = new DBCPConnectionPool(); + runner.addControllerService("dbcp", connectionPool); + runner.setProperty(connectionPool, DBCPProperties.DATABASE_URL, postgres.getJdbcUrl()); + runner.setProperty(connectionPool, DBCPProperties.DB_USER, postgres.getUsername()); + runner.setProperty(connectionPool, DBCPProperties.DB_PASSWORD, postgres.getPassword()); + runner.setProperty(connectionPool, DBCPProperties.DB_DRIVERNAME, postgres.getDriverClassName()); + runner.enableControllerService(connectionPool); + } + + @Test + public void testAddedRowsAutoCommitTrue() throws SQLException, IOException { + // this test in the base class is not valid for PostgreSQL so check the validation error message. + final AssertionError assertionError = assertThrows(AssertionError.class, super::testAddedRowsAutoCommitTrue); + assertThat(assertionError.getMessage(), equalTo("Processor has 1 validation failures:\n" + + "'Set Auto Commit' validated against 'true' is invalid because 'Set Auto Commit' " + + "must be set to 'false' because 'PostgreSQL' Database Type requires it to be 'false'\n")); + } +} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordIT.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordIT.java new file mode 100644 index 000000000000..4a98f0d48d10 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordIT.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.standard; + +import org.apache.nifi.dbcp.DBCPConnectionPool; +import org.apache.nifi.dbcp.utils.DBCPProperties; +import org.apache.nifi.processors.standard.db.DatabaseAdapter; +import org.apache.nifi.processors.standard.db.impl.PostgreSQLDatabaseAdapter; +import org.apache.nifi.reporting.InitializationException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; + +import java.io.IOException; +import java.sql.SQLException; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class QueryDatabaseTableRecordIT extends QueryDatabaseTableRecordTest { + private static PostgreSQLContainer postgres; + + @BeforeAll + public static void setupBeforeClass() { + postgres = new PostgreSQLContainer<>("postgres:9.6.12") + .withInitScript("PutDatabaseRecordIT/create-person-table.sql"); + postgres.start(); + } + + @AfterAll + public static void cleanUpAfterClass() { + if (postgres != null) { + postgres.close(); + postgres = null; + } + } + + @Override + public DatabaseAdapter createDatabaseAdapter() { + return new PostgreSQLDatabaseAdapter(); + } + + @Override + public void createDbcpControllerService() throws InitializationException { + final DBCPConnectionPool connectionPool = new DBCPConnectionPool(); + runner.addControllerService("dbcp", connectionPool); + runner.setProperty(connectionPool, DBCPProperties.DATABASE_URL, postgres.getJdbcUrl()); + runner.setProperty(connectionPool, DBCPProperties.DB_USER, postgres.getUsername()); + runner.setProperty(connectionPool, DBCPProperties.DB_PASSWORD, postgres.getPassword()); + runner.setProperty(connectionPool, DBCPProperties.DB_DRIVERNAME, postgres.getDriverClassName()); + runner.enableControllerService(connectionPool); + } + + @Test + public void testAddedRowsAutoCommitTrue() throws SQLException, IOException { + // this test in the base class is not valid for PostgreSQL so check the validation error message. + final AssertionError assertionError = assertThrows(AssertionError.class, super::testAddedRowsAutoCommitTrue); + assertThat(assertionError.getMessage(), equalTo("Processor has 1 validation failures:\n" + + "'Set Auto Commit' validated against 'true' is invalid because 'Set Auto Commit' " + + "must be set to 'false' because 'PostgreSQL' Database Type requires it to be 'false'\n")); + } +} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordTest.java index 91b288df16e4..7e7de06992a3 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordTest.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableRecordTest.java @@ -71,7 +71,7 @@ public class QueryDatabaseTableRecordTest { private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); MockQueryDatabaseTableRecord processor; - private TestRunner runner; + protected TestRunner runner; private final static String DB_LOCATION = "target/db_qdt"; private DatabaseAdapter dbAdapter; private HashMap origDbAdapters; @@ -109,18 +109,25 @@ public static void cleanUpAfterClass() throws Exception { System.clearProperty("derby.stream.error.file"); } + public DatabaseAdapter createDatabaseAdapter() { + return new GenericDatabaseAdapter(); + } - @BeforeEach - public void setup() throws InitializationException, IOException { + public void createDbcpControllerService() throws InitializationException { final DBCPService dbcp = new DBCPServiceSimpleImpl(); final Map dbcpProperties = new HashMap<>(); + runner.addControllerService("dbcp", dbcp, dbcpProperties); + runner.enableControllerService(dbcp); + } + + @BeforeEach + public void setup() throws InitializationException, IOException { origDbAdapters = new HashMap<>(QueryDatabaseTableRecord.dbAdapters); - dbAdapter = new GenericDatabaseAdapter(); + dbAdapter = createDatabaseAdapter(); QueryDatabaseTableRecord.dbAdapters.put(dbAdapter.getName(), dbAdapter); processor = new MockQueryDatabaseTableRecord(); runner = TestRunners.newTestRunner(processor); - runner.addControllerService("dbcp", dbcp, dbcpProperties); - runner.enableControllerService(dbcp); + createDbcpControllerService(); runner.setProperty(QueryDatabaseTableRecord.DBCP_SERVICE, "dbcp"); runner.setProperty(QueryDatabaseTableRecord.DB_TYPE, dbAdapter.getName()); runner.getStateManager().clear(Scope.CLUSTER); @@ -371,6 +378,82 @@ public void testAddedRows() throws SQLException, IOException { runner.clearTransferState(); } + @Test + public void testAddedRowsAutoCommitTrue() throws SQLException, IOException { + + // load test data to database + final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection(); + Statement stmt = con.createStatement(); + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(QueryDatabaseTableRecord.TABLE_NAME, "TEST_QUERY_DB_TABLE"); + runner.setIncomingConnection(false); + runner.setProperty(QueryDatabaseTableRecord.MAX_VALUE_COLUMN_NAMES, "ID"); + runner.setProperty(QueryDatabaseTableRecord.MAX_ROWS_PER_FLOW_FILE, "2"); + runner.setProperty(QueryDatabaseTableRecord.FETCH_SIZE, "2"); + runner.setProperty(QueryDatabaseTable.AUTO_COMMIT, "true"); + + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTableRecord.REL_SUCCESS, 2); + + MockFlowFile flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTableRecord.REL_SUCCESS).get(0); + assertEquals("TEST_QUERY_DB_TABLE", flowFile.getAttribute(QueryDatabaseTableRecord.RESULT_TABLENAME)); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + flowFile.assertAttributeEquals("record.count", "2"); + + flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTableRecord.REL_SUCCESS).get(1); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + flowFile.assertAttributeEquals("record.count", "1"); + } + + @Test + public void testAddedRowsAutoCommitFalse() throws SQLException, IOException { + + // load test data to database + final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection(); + Statement stmt = con.createStatement(); + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(QueryDatabaseTableRecord.TABLE_NAME, "TEST_QUERY_DB_TABLE"); + runner.setIncomingConnection(false); + runner.setProperty(QueryDatabaseTableRecord.MAX_VALUE_COLUMN_NAMES, "ID"); + runner.setProperty(QueryDatabaseTableRecord.MAX_ROWS_PER_FLOW_FILE, "2"); + runner.setProperty(QueryDatabaseTableRecord.FETCH_SIZE, "2"); + runner.setProperty(QueryDatabaseTable.AUTO_COMMIT, "false"); + + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTableRecord.REL_SUCCESS, 2); + + MockFlowFile flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTableRecord.REL_SUCCESS).get(0); + assertEquals("TEST_QUERY_DB_TABLE", flowFile.getAttribute(QueryDatabaseTableRecord.RESULT_TABLENAME)); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + flowFile.assertAttributeEquals("record.count", "2"); + + flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTableRecord.REL_SUCCESS).get(1); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + flowFile.assertAttributeEquals("record.count", "1"); + } + @Test public void testAddedRowsTwoTables() throws SQLException { @@ -1415,7 +1498,7 @@ public Connection getConnection() throws ProcessException { } @Stateful(scopes = Scope.CLUSTER, description = "Mock for QueryDatabaseTableRecord processor") - private static class MockQueryDatabaseTableRecord extends QueryDatabaseTableRecord { + protected static class MockQueryDatabaseTableRecord extends QueryDatabaseTableRecord { void putColumnType(String colName, Integer colType) { columnTypeMap.put(colName, colType); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java index 17ce74bebbb4..8f360eeb5004 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java @@ -74,7 +74,7 @@ public class QueryDatabaseTableTest { MockQueryDatabaseTable processor; - private TestRunner runner; + protected TestRunner runner; private final static String DB_LOCATION = "target/db_qdt"; private DatabaseAdapter dbAdapter; private HashMap origDbAdapters; @@ -113,18 +113,25 @@ public static void cleanUpAfterClass() throws Exception { System.clearProperty("derby.stream.error.file"); } + public DatabaseAdapter createDatabaseAdapter() { + return new GenericDatabaseAdapter(); + } - @BeforeEach - public void setup() throws InitializationException, IOException { + public void createDbcpControllerService() throws InitializationException { final DBCPService dbcp = new DBCPServiceSimpleImpl(); final Map dbcpProperties = new HashMap<>(); + runner.addControllerService("dbcp", dbcp, dbcpProperties); + runner.enableControllerService(dbcp); + } + + @BeforeEach + public void setup() throws InitializationException, IOException { origDbAdapters = new HashMap<>(QueryDatabaseTable.dbAdapters); - dbAdapter = new GenericDatabaseAdapter(); + dbAdapter = createDatabaseAdapter(); QueryDatabaseTable.dbAdapters.put(dbAdapter.getName(), dbAdapter); processor = new MockQueryDatabaseTable(); runner = TestRunners.newTestRunner(processor); - runner.addControllerService("dbcp", dbcp, dbcpProperties); - runner.enableControllerService(dbcp); + createDbcpControllerService(); runner.setProperty(QueryDatabaseTable.DBCP_SERVICE, "dbcp"); runner.setProperty(QueryDatabaseTable.DB_TYPE, dbAdapter.getName()); runner.getStateManager().clear(Scope.CLUSTER); @@ -373,6 +380,86 @@ public void testAddedRows() throws ClassNotFoundException, SQLException, Initial runner.clearTransferState(); } + @Test + public void testAddedRowsAutoCommitTrue() throws SQLException, IOException { + + // load test data to database + final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection(); + Statement stmt = con.createStatement(); + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(QueryDatabaseTable.TABLE_NAME, "TEST_QUERY_DB_TABLE"); + runner.setIncomingConnection(false); + runner.setProperty(QueryDatabaseTable.MAX_VALUE_COLUMN_NAMES, "ID"); + runner.setProperty(QueryDatabaseTable.MAX_ROWS_PER_FLOW_FILE, "2"); + runner.setProperty(QueryDatabaseTable.FETCH_SIZE, "2"); + runner.setProperty(QueryDatabaseTable.AUTO_COMMIT, "true"); + + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 2); + + MockFlowFile flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0); + assertEquals("TEST_QUERY_DB_TABLE", flowFile.getAttribute(QueryDatabaseTable.RESULT_TABLENAME)); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + InputStream in = new ByteArrayInputStream(flowFile.toByteArray()); + assertEquals(2, getNumberOfRecordsFromStream(in)); + + flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(1); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + in = new ByteArrayInputStream(flowFile.toByteArray()); + assertEquals(1, getNumberOfRecordsFromStream(in)); + } + + @Test + public void testAddedRowsAutoCommitFalse() throws SQLException, IOException { + + // load test data to database + final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection(); + Statement stmt = con.createStatement(); + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(QueryDatabaseTable.TABLE_NAME, "TEST_QUERY_DB_TABLE"); + runner.setIncomingConnection(false); + runner.setProperty(QueryDatabaseTable.MAX_VALUE_COLUMN_NAMES, "ID"); + runner.setProperty(QueryDatabaseTable.MAX_ROWS_PER_FLOW_FILE, "2"); + runner.setProperty(QueryDatabaseTable.FETCH_SIZE, "2"); + runner.setProperty(QueryDatabaseTable.AUTO_COMMIT, "false"); + + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 2); + + MockFlowFile flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0); + assertEquals("TEST_QUERY_DB_TABLE", flowFile.getAttribute(QueryDatabaseTable.RESULT_TABLENAME)); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + InputStream in = new ByteArrayInputStream(flowFile.toByteArray()); + assertEquals(2, getNumberOfRecordsFromStream(in)); + + flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(1); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + in = new ByteArrayInputStream(flowFile.toByteArray()); + assertEquals(1, getNumberOfRecordsFromStream(in)); + } + @Test public void testAddedRowsTwoTables() throws ClassNotFoundException, SQLException, InitializationException, IOException { @@ -1461,7 +1548,7 @@ public Connection getConnection() throws ProcessException { } @Stateful(scopes = Scope.CLUSTER, description = "Mock for QueryDatabaseTable processor") - private static class MockQueryDatabaseTable extends QueryDatabaseTable { + protected static class MockQueryDatabaseTable extends QueryDatabaseTable { void putColumnType(String colName, Integer colType) { columnTypeMap.put(colName, colType); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/db/impl/TestPostgreSQLDatabaseAdapter.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/db/impl/TestPostgreSQLDatabaseAdapter.java index ea1062186727..6d866ef90f11 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/db/impl/TestPostgreSQLDatabaseAdapter.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/db/impl/TestPostgreSQLDatabaseAdapter.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -41,6 +42,21 @@ public void testSupportsUpsert() { assertTrue(testSubject.supportsUpsert(), testSubject.getClass().getSimpleName() + " should support upsert"); } + @Test + public void getAutoCommitForReadsFetchSizeNull() { + assertEquals(Optional.empty(), testSubject.getAutoCommitForReads(null)); + } + + @Test + public void getAutoCommitForReadsFetchSizeZero() { + assertEquals(Optional.empty(), testSubject.getAutoCommitForReads(0)); + } + + @Test + public void getAutoCommitForReadsFetchSizeNonZero() { + assertEquals(Optional.of(Boolean.FALSE), testSubject.getAutoCommitForReads(1)); + } + @Test public void testGetUpsertStatementWithNullTableName() { testGetUpsertStatement(null, Arrays.asList("notEmpty"), Arrays.asList("notEmpty"), new IllegalArgumentException("Table name cannot be null or blank")); From 407dd4d4bcb0cbf6fec037ebe2ad99551b4ce067 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Mon, 25 Mar 2024 12:31:24 -0500 Subject: [PATCH 73/73] NIFI-12947 Upgraded MIME4J to 0.8.11 Signed-off-by: Pierre Villard This closes #8561. --- nifi-code-coverage/pom.xml | 7 +++++++ nifi-nar-bundles/nifi-media-bundle/pom.xml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/nifi-code-coverage/pom.xml b/nifi-code-coverage/pom.xml index f9cec5b9d14c..a74b90f3be87 100644 --- a/nifi-code-coverage/pom.xml +++ b/nifi-code-coverage/pom.xml @@ -31,6 +31,7 @@ 1.6.0 1.24.0 2.12.0 + 0.8.11 @@ -125,6 +126,12 @@ sshd-osgi ${org.apache.sshd.version} + + + org.apache.james + apache-mime4j-core + ${mime4j.version} + diff --git a/nifi-nar-bundles/nifi-media-bundle/pom.xml b/nifi-nar-bundles/nifi-media-bundle/pom.xml index 6555b16cc42f..1c0c6a3cb41c 100644 --- a/nifi-nar-bundles/nifi-media-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-media-bundle/pom.xml @@ -27,6 +27,7 @@ 5.2.5 + 0.8.11 @@ -58,6 +59,11 @@ poi-ooxml ${poi.version} + + org.apache.james + apache-mime4j-core + ${mime4j.version} +