Skip to content

Commit 2e197f1

Browse files
authored
[Spark]Add VacuumProtocolCheck ReaderWriter Table Feature (#2730)
## Description Add a new VacuumProtocolCheck ReaderWriter Table Feature so that Vacuum command on older DBR client and OSS clients fail. This is in follow-up to #2557 where protocol-check was added during the vacuum-write flow. ## How was this patch tested? UTs ## Does this PR introduce _any_ user-facing changes? No
1 parent 60914cd commit 2e197f1

File tree

2 files changed

+42
-1
lines changed

2 files changed

+42
-1
lines changed

spark/src/main/scala/org/apache/spark/sql/delta/TableFeature.scala

+15-1
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,8 @@ object TableFeature {
357357
// Row IDs are still under development and only available in testing.
358358
RowTrackingFeature,
359359
InCommitTimestampTableFeature,
360-
TypeWideningTableFeature)
360+
TypeWideningTableFeature,
361+
VacuumProtocolCheckTableFeature)
361362
}
362363
val featureMap = features.map(f => f.name.toLowerCase(Locale.ROOT) -> f).toMap
363364
require(features.size == featureMap.size, "Lowercase feature names must not duplicate.")
@@ -656,6 +657,19 @@ object InCommitTimestampTableFeature
656657
}
657658
}
658659

660+
/**
661+
* A ReaderWriter table feature for VACUUM. If this feature is enabled:
662+
* A writer should follow one of the following:
663+
* 1. Non-Support for Vacuum: Writers can explicitly state that they do not support VACUUM for
664+
* any table, regardless of whether the Vacuum Protocol Check Table feature exists.
665+
* 2. Implement Writer Protocol Check: Ensure that the VACUUM implementation includes a writer
666+
* protocol check before any file deletions occur.
667+
* Readers don't need to understand or change anything new; they just need to acknowledge the
668+
* feature exists
669+
*/
670+
object VacuumProtocolCheckTableFeature
671+
extends ReaderWriterFeature(name = "vacuumProtocolCheck-dev")
672+
659673
/**
660674
* Features below are for testing only, and are being registered to the system only in the testing
661675
* environment. See [[TableFeature.allSupportedFeaturesMap]] for the registration.

spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableFeatureSuite.scala

+27
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,33 @@ class DeltaTableFeatureSuite
459459
}
460460
}
461461

462+
for(commandName <- Seq("ALTER", "CLONE", "REPLACE", "CREATE OR REPLACE")) {
463+
test(s"Vacuum Protocol Check is disabled by default but can be enabled during $commandName") {
464+
val table = "tbl"
465+
withTable(table) {
466+
spark.range(0).write.format("delta").saveAsTable(table)
467+
val log = DeltaLog.forTable(spark, TableIdentifier(table))
468+
val protocol = log.update().protocol
469+
assert(!protocol.readerAndWriterFeatureNames.contains(VacuumProtocolCheckTableFeature.name))
470+
471+
val tblProperties1 = Seq(s"'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION")
472+
sql(buildTablePropertyModifyingCommand(
473+
commandName, targetTableName = table, sourceTableName = table, tblProperties1))
474+
val newProtocol1 = log.update().protocol
475+
assert(!newProtocol1.readerAndWriterFeatureNames.contains(
476+
VacuumProtocolCheckTableFeature.name))
477+
478+
val tblProperties2 = Seq(s"'$FEATURE_PROP_PREFIX${VacuumProtocolCheckTableFeature.name}' " +
479+
s"= 'supported', 'delta.minWriterVersion' = $TABLE_FEATURES_MIN_WRITER_VERSION")
480+
sql(buildTablePropertyModifyingCommand(
481+
commandName, targetTableName = table, sourceTableName = table, tblProperties2))
482+
val newProtocol2 = log.update().protocol
483+
assert(newProtocol2.readerAndWriterFeatureNames.contains(
484+
VacuumProtocolCheckTableFeature.name))
485+
}
486+
}
487+
}
488+
462489
private def buildTablePropertyModifyingCommand(
463490
commandName: String,
464491
targetTableName: String,

0 commit comments

Comments
 (0)