Skip to content

Core|ORC|Spark: Remove deprecated functionality #6274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rdblue merged 1 commit into from
Dec 18, 2022
Merged

Core|ORC|Spark: Remove deprecated functionality #6274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions .palantir/revapi.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ acceptedBreaks:
justification: "Removing deprecated method"
"1.1.0":
org.apache.iceberg:iceberg-core:
- code: "java.class.removed"
old: "class org.apache.iceberg.rest.HTTPClientFactory"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.exception.checkedAdded"
old: "method T com.fasterxml.jackson.databind.JsonDeserializer<T>::deserialize(com.fasterxml.jackson.core.JsonParser,\
\ com.fasterxml.jackson.databind.DeserializationContext, T) throws java.io.IOException\
Expand Down Expand Up @@ -258,6 +261,99 @@ acceptedBreaks:
\ T) throws java.io.IOException, com.fasterxml.jackson.core.JacksonException\
\ @ org.apache.iceberg.rest.RESTSerializers.UpdateRequirementDeserializer"
justification: "False positive - JacksonException is a subclass of IOException"
- code: "java.method.removed"
old: "method <T> org.apache.iceberg.io.CloseableIterable<T> org.apache.iceberg.deletes.Deletes::filterDeleted(org.apache.iceberg.io.CloseableIterable<T>,\
\ java.util.function.Predicate<T>)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.AllDataFilesTable.AllDataFilesTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.AllDeleteFilesTable.AllDeleteFilesTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.AllFilesTable.AllFilesTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.AllManifestsTable.AllManifestsTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.DataFilesTable.DataFilesTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.DataTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.DeleteFilesTable.DeleteFilesTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method boolean org.apache.iceberg.BaseTableScan::colStats() @ org.apache.iceberg.FilesTable.FilesTableScan"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.lang.String[] org.apache.iceberg.hadoop.Util::blockLocations(org.apache.iceberg.io.FileIO,\
\ org.apache.iceberg.CombinedScanTask)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.util.List<org.apache.iceberg.ManifestFile> org.apache.iceberg.BaseReplacePartitions::apply(org.apache.iceberg.TableMetadata)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.util.List<org.apache.iceberg.ManifestFile> org.apache.iceberg.BaseRewriteManifests::apply(org.apache.iceberg.TableMetadata)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.util.List<org.apache.iceberg.ManifestFile> org.apache.iceberg.MergingSnapshotProducer<ThisT>::apply(org.apache.iceberg.TableMetadata)\
\ @ org.apache.iceberg.BaseOverwriteFiles"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.util.List<org.apache.iceberg.ManifestFile> org.apache.iceberg.MergingSnapshotProducer<ThisT>::apply(org.apache.iceberg.TableMetadata)\
\ @ org.apache.iceberg.StreamingDelete"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.util.Map<java.lang.Integer, ?> org.apache.iceberg.util.PartitionUtil::constantsMap(org.apache.iceberg.FileScanTask)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.util.Map<java.lang.Integer, ?> org.apache.iceberg.util.PartitionUtil::constantsMap(org.apache.iceberg.FileScanTask,\
\ java.util.function.BiFunction<org.apache.iceberg.types.Type, java.lang.Object,\
\ java.lang.Object>)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method java.util.Map<java.lang.Integer, ?> org.apache.iceberg.util.PartitionUtil::constantsMap(org.apache.iceberg.FileScanTask,\
\ org.apache.iceberg.types.Types.StructType, java.util.function.BiFunction<org.apache.iceberg.types.Type,\
\ java.lang.Object, java.lang.Object>)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method void org.apache.iceberg.BaseReplacePartitions::validate(org.apache.iceberg.TableMetadata)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method void org.apache.iceberg.ManifestReader<F extends org.apache.iceberg.ContentFile<F\
\ extends org.apache.iceberg.ContentFile<F>>>::<init>(org.apache.iceberg.io.InputFile,\
\ java.util.Map<java.lang.Integer, org.apache.iceberg.PartitionSpec>, org.apache.iceberg.InheritableMetadata,\
\ org.apache.iceberg.ManifestReader.FileType)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method void org.apache.iceberg.ManifestWriter<F extends org.apache.iceberg.ContentFile<F\
\ extends org.apache.iceberg.ContentFile<F>>>::delete(F)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method void org.apache.iceberg.ManifestWriter<F extends org.apache.iceberg.ContentFile<F\
\ extends org.apache.iceberg.ContentFile<F>>>::existing(F, long, long)"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method void org.apache.iceberg.SnapshotProducer<ThisT>::validate(org.apache.iceberg.TableMetadata)\
\ @ org.apache.iceberg.BaseOverwriteFiles"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method void org.apache.iceberg.SnapshotProducer<ThisT>::validate(org.apache.iceberg.TableMetadata)\
\ @ org.apache.iceberg.BaseRewriteManifests"
justification: "Removing deprecations for 1.2.0"
- code: "java.method.removed"
old: "method void org.apache.iceberg.SnapshotProducer<ThisT>::validate(org.apache.iceberg.TableMetadata)\
\ @ org.apache.iceberg.StreamingDelete"
justification: "Removing deprecations for 1.2.0"
org.apache.iceberg:iceberg-orc:
- code: "java.method.removed"
old: "method org.apache.iceberg.orc.ORC.WriteBuilder org.apache.iceberg.orc.ORC.WriteBuilder::config(java.lang.String,\
\ java.lang.String)"
justification: "Removing deprecations for 1.2.0"
apache-iceberg-0.14.0:
org.apache.iceberg:iceberg-api:
- code: "java.class.defaultSerializationChanged"
Expand Down
28 changes: 0 additions & 28 deletions core/src/main/java/org/apache/iceberg/BaseReplacePartitions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,20 +79,6 @@ public ReplacePartitions validateNoConflictingData() {
return this;
}

/**
* Validate the current metadata.
*
* <p>Child operations can override this to add custom validation.
*
* @param currentMetadata current table metadata to validate
* @deprecated Will be removed in 1.2.0, use {@link SnapshotProducer#validate(TableMetadata,
* Snapshot)}.
*/
@Deprecated
public void validate(TableMetadata currentMetadata) {
super.validate(currentMetadata);
}

@Override
public void validate(TableMetadata currentMetadata, Snapshot snapshot) {
if (validateConflictingData) {
Expand All @@ -114,20 +100,6 @@ public void validate(TableMetadata currentMetadata, Snapshot snapshot) {
}
}

/**
* Apply the update's changes to the base table metadata and return the new manifest list.
*
* @param base the base table metadata to apply changes to
* @return a manifest list for the new snapshot.
* @deprecated Will be removed in 1.2.0, use {@link BaseReplacePartitions#apply(TableMetadata,
* Snapshot)}.
*/
@Deprecated
@Override
public List<ManifestFile> apply(TableMetadata base) {
return super.apply(base);
}

@Override
public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
if (dataSpec().fields().size() <= 0) {
Expand Down
14 changes: 0 additions & 14 deletions core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,20 +173,6 @@ private ManifestFile copyManifest(ManifestFile manifest) {
summaryBuilder);
}

/**
* Apply the update's changes to the base table metadata and return the new manifest list.
*
* @param base the base table metadata to apply changes to
* @return a manifest list for the new snapshot.
* @deprecated Will be removed in 1.2.0, use {@link BaseRewriteManifests#apply(TableMetadata,
* Snapshot)}.
*/
@Deprecated
@Override
public List<ManifestFile> apply(TableMetadata base) {
return super.apply(base);
}

@Override
public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
List<ManifestFile> currentManifests = base.currentSnapshot().dataManifests(ops.io());
Expand Down
10 changes: 0 additions & 10 deletions core/src/main/java/org/apache/iceberg/BaseTableScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,16 +61,6 @@ protected Long snapshotId() {
return context().snapshotId();
}

/**
* @return whether column stats are returned.
* @deprecated Will be removed in 1.2.0, use {@link TableScanContext#returnColumnStats()}
* directly.
*/
@Deprecated
protected boolean colStats() {
return context().returnColumnStats();
}

protected Map<String, String> options() {
return context().options();
}
Expand Down
13 changes: 0 additions & 13 deletions core/src/main/java/org/apache/iceberg/FastAppend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,19 +141,6 @@ private ManifestFile copyManifest(ManifestFile manifest) {
summaryBuilder);
}

/**
* Apply the update's changes to the base table metadata and return the new manifest list.
*
* @param base the base table metadata to apply changes to
* @return a manifest list for the new snapshot.
* @deprecated Will be removed in 1.2.0, use {@link FastAppend#apply(TableMetadata, Snapshot)}.
*/
@Deprecated
@Override
public List<ManifestFile> apply(TableMetadata base) {
return super.apply(base);
}

@Override
public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
List<ManifestFile> newManifests = Lists.newArrayList();
Expand Down
23 changes: 0 additions & 23 deletions core/src/main/java/org/apache/iceberg/GenericManifestEntry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ class GenericManifestEntry<F extends ContentFile<F>>
private Status status = Status.EXISTING;
private Long snapshotId = null;
private Long dataSequenceNumber = null;
private Long sequenceNumber = null;
Copy link
Contributor

@rdblue rdblue Nov 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aokolnychyi, can you take a look at these changes?

private Long fileSequenceNumber = null;
private F file = null;

Expand All @@ -47,7 +46,6 @@ private GenericManifestEntry(GenericManifestEntry<F> toCopy, boolean fullCopy) {
this.status = toCopy.status;
this.snapshotId = toCopy.snapshotId;
this.dataSequenceNumber = toCopy.dataSequenceNumber;
this.sequenceNumber = toCopy.sequenceNumber;
this.fileSequenceNumber = toCopy.fileSequenceNumber;
this.file = toCopy.file().copy(fullCopy);
}
Expand All @@ -62,7 +60,6 @@ ManifestEntry<F> wrapExisting(
this.status = Status.EXISTING;
this.snapshotId = newSnapshotId;
this.dataSequenceNumber = newDataSequenceNumber;
this.sequenceNumber = newDataSequenceNumber;
this.fileSequenceNumber = newFileSequenceNumber;
this.file = newFile;
return this;
Expand All @@ -76,7 +73,6 @@ ManifestEntry<F> wrapAppend(Long newSnapshotId, Long newDataSequenceNumber, F ne
this.status = Status.ADDED;
this.snapshotId = newSnapshotId;
this.dataSequenceNumber = newDataSequenceNumber;
this.sequenceNumber = newDataSequenceNumber;
this.fileSequenceNumber = null;
this.file = newFile;
return this;
Expand All @@ -92,7 +88,6 @@ ManifestEntry<F> wrapDelete(
this.status = Status.DELETED;
this.snapshotId = newSnapshotId;
this.dataSequenceNumber = newDataSequenceNumber;
this.sequenceNumber = null;
this.fileSequenceNumber = newFileSequenceNumber;
this.file = newFile;
return this;
Expand All @@ -115,15 +110,6 @@ public Long dataSequenceNumber() {
return dataSequenceNumber;
}

@Override
public Long sequenceNumber() {
if (sequenceNumber != null) {
return sequenceNumber;
} else {
return isLive() ? dataSequenceNumber : null;
}
}

@Override
public Long fileSequenceNumber() {
return fileSequenceNumber;
Expand Down Expand Up @@ -152,14 +138,7 @@ public void setSnapshotId(long newSnapshotId) {

@Override
public void setDataSequenceNumber(long newDataSequenceNumber) {
// always reset sequenceNumber whenever dataSequenceNumber is changed
this.dataSequenceNumber = newDataSequenceNumber;
this.sequenceNumber = null;
}

@Override
public void setSequenceNumber(long newSequenceNumber) {
this.sequenceNumber = newSequenceNumber;
}

@Override
Expand All @@ -178,9 +157,7 @@ public void put(int i, Object v) {
this.snapshotId = (Long) v;
return;
case 2:
// always reset sequenceNumber whenever dataSequenceNumber is changed
this.dataSequenceNumber = (Long) v;
this.sequenceNumber = null;
return;
case 3:
this.fileSequenceNumber = (Long) v;
Expand Down
4 changes: 0 additions & 4 deletions core/src/main/java/org/apache/iceberg/InheritableMetadataFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,6 @@ public <F extends ContentFile<F>> ManifestEntry<F> apply(ManifestEntry<F> manife
manifestEntry.setDataSequenceNumber(sequenceNumber);
}

if (manifestEntry.sequenceNumber() == null) {
manifestEntry.setSequenceNumber(sequenceNumber);
}

// in v1 tables, the file sequence number is not persisted and can be safely defaulted to 0
// in v2 tables, the file sequence number should be inherited iff the entry status is ADDED
if (manifestEntry.fileSequenceNumber() == null
Expand Down
21 changes: 0 additions & 21 deletions core/src/main/java/org/apache/iceberg/ManifestEntry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,27 +103,6 @@ default boolean isLive() {
*/
void setDataSequenceNumber(long dataSequenceNumber);

/**
* Returns the data sequence number of the file if the entry status is ADDED or EXISTING.
* Otherwise, returns the sequence number of the snapshot in which the file was removed.
*
* <p>Note that usage of this method should be avoided as it behaves inconsistently for different
* entry statutes. Use {@link #dataSequenceNumber()} instead.
*
* @deprecated since 1.0.0, will be removed in 1.1.0; use {@link #dataSequenceNumber()} instead.
*/
@Deprecated
Long sequenceNumber();
Copy link
Contributor Author

@nastra nastra Nov 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aokolnychyi could you please review all places that adjust sequenceNumber -> dataSequenceNumber?


/**
* Set the sequence number for this manifest entry.
*
* @param sequenceNumber a sequence number
* @deprecated since 1.0.0, will be removed in 1.1.0; use the data sequence number instead.
*/
@Deprecated
void setSequenceNumber(long sequenceNumber);

/**
* Returns the file sequence number.
*
Expand Down
32 changes: 0 additions & 32 deletions core/src/main/java/org/apache/iceberg/ManifestReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,38 +98,6 @@ private String fileClass() {
private Evaluator lazyEvaluator = null;
private InclusiveMetricsEvaluator lazyMetricsEvaluator = null;

/**
* @deprecated Will be removed in 1.2.0, use {@link ManifestReader#ManifestReader(InputFile, int,
* Map, InheritableMetadata, FileType)}.
*/
@Deprecated
protected ManifestReader(
InputFile file,
Map<Integer, PartitionSpec> specsById,
InheritableMetadata inheritableMetadata,
FileType content) {
this.file = file;
this.inheritableMetadata = inheritableMetadata;
this.content = content;

Map<String, String> metadata = readMetadata(file);
int specId = TableMetadata.INITIAL_SPEC_ID;
String specProperty = metadata.get("partition-spec-id");
if (specProperty != null) {
specId = Integer.parseInt(specProperty);
}

if (specsById != null) {
this.spec = specsById.get(specId);
} else {
Schema schema = SchemaParser.fromJson(metadata.get("schema"));
this.spec =
PartitionSpecParser.fromJsonFields(schema, specId, metadata.get("partition-spec"));
}

this.fileSchema = new Schema(DataFile.getType(spec.partitionType()).fields());
}

protected ManifestReader(
InputFile file,
int specId,
Expand Down
Loading