From 92aa344e788e2d2f8bb3cd0e04ed1fc763511006 Mon Sep 17 00:00:00 2001 From: Marc D'Mello Date: Wed, 27 Nov 2024 02:01:47 +0000 Subject: [PATCH] Removed PerWriterIndexRamManager --- .../index/FlushByRamOrCountsPolicy.java | 14 +++---- .../org/apache/lucene/index/FlushPolicy.java | 12 ++---- .../org/apache/lucene/index/IndexWriter.java | 13 +++--- .../lucene/index/IndexWriterRAMManager.java | 40 +++---------------- .../apache/lucene/index/TestIndexWriter.java | 5 +-- .../index/TestIndexWriterRAMManager.java | 10 ++--- 6 files changed, 27 insertions(+), 67 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java b/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java index 13c517afdb52..174474c37234 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java @@ -55,13 +55,13 @@ && flushOnDocCount() } @Override - public void flushWriter( - IndexWriterRAMManager ramManager, - IndexWriterRAMManager.PerWriterIndexWriterRAMManager perWriterRamManager) - throws IOException { - long totalBytes = perWriterRamManager.getTotalBufferBytesUsed(); - if (totalBytes > ramManager.getRamBufferSizeMB() * 1024 * 1024) { - ramManager.flushRoundRobin(); + public void flushRamManager(IndexWriter writer) throws IOException { + IndexWriterRAMManager ramManager = writer.getConfig().indexWriterRAMManager; + if (ramManager.getRamBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH) { + long totalBytes = ramManager.updateAndGetCurrentBytesUsed(writer.ramManagerId); + if (totalBytes > ramManager.getRamBufferSizeMB() * 1024 * 1024) { + ramManager.flushRoundRobin(); + } } } diff --git a/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java b/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java index a57f6f78fd4b..a6edb87be3a2 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java @@ -59,15 +59,11 @@ public abstract void onChange( DocumentsWriterFlushControl control, DocumentsWriterPerThread perThread); /** - * Chooses which writer should be flushed. Default implementation chooses the writer with most RAM - * usage - * - * @param ramManager the {@link IndexWriterRAMManager} being used to actually flush the writers + * Flushed a writer according to the FlushPolicy. NOTE: this doesn't necessarily mean the passed + * in writer will be flushed, and in most cases, this will actually be the case as the default + * policy is a round-robin policy */ - public abstract void flushWriter( - IndexWriterRAMManager ramManager, - IndexWriterRAMManager.PerWriterIndexWriterRAMManager perWriterRamManager) - throws IOException; + public abstract void flushRamManager(IndexWriter writer) throws IOException; /** Called by DocumentsWriter to initialize the FlushPolicy */ protected synchronized void init(LiveIndexWriterConfig indexWriterConfig) { diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index eeff44f840ef..57829368f9cf 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -462,7 +462,8 @@ public void onTicketBacklog() { } }; - private final IndexWriterRAMManager.PerWriterIndexWriterRAMManager indexWriterRAMManager; + /** The id that is associated with this writer for {@link IndexWriterRAMManager} */ + public final int ramManagerId; /** * Expert: returns a readonly reader, covering all committed as well as un-committed changes to @@ -1213,9 +1214,7 @@ public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException { writeLock = null; } } - this.indexWriterRAMManager = - new IndexWriterRAMManager.PerWriterIndexWriterRAMManager( - this, config.getIndexWriterRAMManager()); + this.ramManagerId = config.indexWriterRAMManager.registerWriter(this); } /** Confirms that the incoming index sort (if any) matches the existing index sort (if any). */ @@ -1370,7 +1369,7 @@ private void shutdown() throws IOException { */ @Override public void close() throws IOException { - indexWriterRAMManager.removeWriter(); + config.indexWriterRAMManager.removeWriter(ramManagerId); if (config.getCommitOnClose()) { shutdown(); } else { @@ -2451,7 +2450,7 @@ public void rollback() throws IOException { // Ensure that only one thread actually gets to do the // closing, and make sure no commit is also in progress: if (shouldClose(true)) { - indexWriterRAMManager.removeWriter(); + config.indexWriterRAMManager.removeWriter(ramManagerId); rollbackInternal(); } } @@ -6019,7 +6018,7 @@ private long maybeProcessEvents(long seqNo) throws IOException { seqNo = -seqNo; processEvents(true); } - indexWriterRAMManager.flushIfNecessary(config.flushPolicy); + config.flushPolicy.flushRamManager(this); return seqNo; } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterRAMManager.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterRAMManager.java index 80852cef00c5..09b9e4f3bfb5 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterRAMManager.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterRAMManager.java @@ -63,7 +63,7 @@ public int flushRoundRobin() throws IOException { return idToWriter.flushRoundRobin(); } - /** Registers a writer and returns the associated ID, protected for testing */ + /** Registers a writer can returns the associated ID */ protected int registerWriter(IndexWriter writer) { int id = idGenerator.incrementAndGet(); idToWriter.addWriter(writer, id); @@ -75,42 +75,12 @@ protected void removeWriter(int id) { idToWriter.removeWriter(id); } - private void flushIfNecessary( - FlushPolicy flushPolicy, PerWriterIndexWriterRAMManager perWriterRAMManager) - throws IOException { - if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH) { - flushPolicy.flushWriter(this, perWriterRAMManager); - } - } - - private long updateAndGetCurrentBytesUsed(int id) { - return idToWriter.getTotalRamTracker(id); - } - /** - * For use in {@link IndexWriter}, manages communication with the {@link IndexWriterRAMManager} + * Will call {@link IndexWriter#ramBytesUsed()} for the writer id passed in, and then updates the + * total ram using that value and returns it */ - public static class PerWriterIndexWriterRAMManager { - - private final int id; - private final IndexWriterRAMManager manager; - - PerWriterIndexWriterRAMManager(IndexWriter writer, IndexWriterRAMManager manager) { - id = manager.registerWriter(writer); - this.manager = manager; - } - - void removeWriter() { - manager.removeWriter(id); - } - - void flushIfNecessary(FlushPolicy flushPolicy) throws IOException { - manager.flushIfNecessary(flushPolicy, this); - } - - long getTotalBufferBytesUsed() { - return manager.updateAndGetCurrentBytesUsed(id); - } + public long updateAndGetCurrentBytesUsed(int id) { + return idToWriter.getTotalRamTracker(id); } private static class LinkedIdToWriter { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index b7ec9cc46342..84627b140a58 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -3218,10 +3218,7 @@ public void onChange( } @Override - public void flushWriter( - IndexWriterRAMManager ramManager, - IndexWriterRAMManager.PerWriterIndexWriterRAMManager perWriterRamManager) - throws IOException {} + public void flushRamManager(IndexWriter writer) throws IOException {} }); try (IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { assertEquals(0, w.docWriter.flushControl.getDeleteBytesUsed()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterRAMManager.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterRAMManager.java index b54978d1f572..fa5032e15599 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterRAMManager.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterRAMManager.java @@ -369,11 +369,9 @@ private static class TestFlushPolicy extends FlushPolicy { public void onChange(DocumentsWriterFlushControl control, DocumentsWriterPerThread perThread) {} @Override - public void flushWriter( - IndexWriterRAMManager ramManager, - IndexWriterRAMManager.PerWriterIndexWriterRAMManager perWriterRamManager) - throws IOException { - long totalBytes = perWriterRamManager.getTotalBufferBytesUsed(); + public void flushRamManager(IndexWriter writer) throws IOException { + IndexWriterRAMManager ramManager = writer.getConfig().indexWriterRAMManager; + long totalBytes = ramManager.updateAndGetCurrentBytesUsed(writer.ramManagerId); if (totalBytes > ramManager.getRamBufferSizeMB() * 1024 * 1024) { int flushedId = ramManager.flushRoundRobin(); flushedWriters.add(flushedId); @@ -420,7 +418,7 @@ public int flushRoundRobin() throws IOException { } @Override - protected int registerWriter(IndexWriter writer) { + public int registerWriter(IndexWriter writer) { int id = super.registerWriter(writer); events.add(new TestEventAndId(TestEvent.ADD, id)); return id;