Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,8 @@ public interface FileStoreCommit extends AutoCloseable {
/** Find out which committables need to be retried when recovering from the failure. */
List<ManifestCommittable> filterCommitted(List<ManifestCommittable> committables);

/** Commit from manifest committable. */
void commit(ManifestCommittable committable, Map<String, String> properties);

/** Commit from manifest committable with checkAppendFiles. */
void commit(
ManifestCommittable committable,
Map<String, String> properties,
boolean checkAppendFiles);
void commit(ManifestCommittable committable, boolean checkAppendFiles);

/**
* Overwrite from manifest committable and partition.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@
* <p>This class provides an atomic commit method to the user.
*
* <ol>
* <li>Before calling {@link #commit(ManifestCommittable, Map)}, if user cannot determine if this
* commit is done before, user should first call {@link #filterCommitted}.
* <li>Before calling {@link #commit}, if user cannot determine if this commit is done before,
* user should first call {@link #filterCommitted}.
* <li>Before committing, it will first check for conflicts by checking if all files to be removed
* currently exists, and if modified files have overlapping key ranges with existing files.
* <li>After that it use the external {@link SnapshotCommit} (if provided) or the atomic rename of
Expand Down Expand Up @@ -260,21 +260,13 @@ public List<ManifestCommittable> filterCommitted(List<ManifestCommittable> commi
}

@Override
public void commit(ManifestCommittable committable, Map<String, String> properties) {
commit(committable, properties, false);
}

@Override
public void commit(
ManifestCommittable committable,
Map<String, String> properties,
boolean checkAppendFiles) {
public void commit(ManifestCommittable committable, boolean checkAppendFiles) {
LOG.info(
"Ready to commit to table {}, number of commit messages: {}",
tableName,
committable.fileCommittables().size());
if (LOG.isDebugEnabled()) {
LOG.debug("Ready to commit\n{}", committable.toString());
LOG.debug("Ready to commit\n{}", committable);
}

long started = System.nanoTime();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
Expand Down Expand Up @@ -215,7 +214,7 @@ public void commit(ManifestCommittable committable) {
public void commitMultiple(List<ManifestCommittable> committables, boolean checkAppendFiles) {
if (overwritePartition == null) {
for (ManifestCommittable committable : committables) {
commit.commit(committable, new HashMap<>(), checkAppendFiles);
commit.commit(committable, checkAppendFiles);
}
if (!committables.isEmpty()) {
expire(committables.get(committables.size() - 1).identifier(), expireMainExecutor);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public void commit(CommitMessage... commitMessages) {
for (CommitMessage commitMessage : commitMessages) {
committable.addFileCommittable(commitMessage);
}
newCommit().commit(committable, Collections.emptyMap());
newCommit().commit(committable, false);
}

public CommitMessage removeIndexFiles(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ public List<Snapshot> commitDataWatermark(
null,
watermark,
Collections.emptyList(),
(commit, committable) -> commit.commit(committable, Collections.emptyMap()));
(commit, committable) -> commit.commit(committable, false));
}

public List<Snapshot> commitData(
Expand All @@ -237,7 +237,7 @@ public List<Snapshot> commitData(
(commit, committable) -> {
logOffsets.forEach(
(bucket, offset) -> committable.addLogOffset(bucket, offset, false));
commit.commit(committable, Collections.emptyMap());
commit.commit(committable, false);
});
}

Expand Down Expand Up @@ -291,7 +291,7 @@ public List<Snapshot> commitDataIndex(
null,
null,
Arrays.asList(indexFiles),
(commit, committable) -> commit.commit(committable, Collections.emptyMap()));
(commit, committable) -> commit.commit(committable, false));
}

public List<Snapshot> commitDataImpl(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ public void testCommitEmpty() throws Exception {
null,
null,
Collections.emptyList(),
(commit, committable) -> commit.commit(committable, Collections.emptyMap()));
(commit, committable) -> commit.commit(committable, false));
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(snapshot.id());

// commit empty new files
Expand All @@ -546,7 +546,7 @@ public void testCommitEmpty() throws Exception {
Collections.emptyList(),
(commit, committable) -> {
commit.ignoreEmptyCommit(false);
commit.commit(committable, Collections.emptyMap());
commit.commit(committable, false);
});
assertThat(store.snapshotManager().latestSnapshotId()).isEqualTo(snapshot.id() + 1);
}
Expand All @@ -567,20 +567,14 @@ public void testCommitOldSnapshotAgain() throws Exception {
null,
Collections.emptyList(),
(commit, committable) -> {
commit.commit(committable, Collections.emptyMap());
commit.commit(committable, false);
committables.add(committable);
});
}

// commit the first snapshot again, should throw exception due to conflicts
for (int i = 0; i < 3; i++) {
assertThatThrownBy(
() ->
store.newCommit()
.commit(
committables.get(0),
Collections.emptyMap(),
true))
assertThatThrownBy(() -> store.newCommit().commit(committables.get(0), true))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining("Give up committing.");
}
Expand Down Expand Up @@ -1024,15 +1018,15 @@ public void testCommitManifestWithProperties() throws Exception {

// commit with empty properties, the properties in snapshot should be null
ManifestCommittable manifestCommittable = new ManifestCommittable(0);
fileStoreCommit.commit(manifestCommittable, Collections.emptyMap());
fileStoreCommit.commit(manifestCommittable, false);
Snapshot snapshot = checkNotNull(store.snapshotManager().latestSnapshot());
assertThat(snapshot.properties()).isNull();

// commit with non-empty properties
manifestCommittable = new ManifestCommittable(0);
manifestCommittable.addProperty("k1", "v1");
manifestCommittable.addProperty("k2", "v2");
fileStoreCommit.commit(manifestCommittable, Collections.emptyMap());
fileStoreCommit.commit(manifestCommittable, false);
snapshot = checkNotNull(store.snapshotManager().latestSnapshot());
Map<String, String> expectedProps = new HashMap<>();
expectedProps.put("k1", "v1");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -111,7 +110,7 @@ public static void commitData(
}

try (FileStoreCommit commit = store.newCommit()) {
commit.commit(committable, Collections.emptyMap());
commit.commit(committable, false);
}

writers.values().stream()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ private void doCommit() throws Exception {
inc.compactIncrement()));
}

runWithRetry(committable, () -> commit.commit(committable, Collections.emptyMap()));
runWithRetry(committable, () -> commit.commit(committable, false));
}

private void doOverwrite() throws Exception {
Expand Down Expand Up @@ -193,7 +193,7 @@ private void doFinalCompact() {
inc.newFilesIncrement(),
inc.compactIncrement()));
}
commit.commit(committable, Collections.emptyMap());
commit.commit(committable, false);
break;
} catch (Exception e) {
if (LOG.isDebugEnabled()) {
Expand Down
Loading