diff --git a/core/rio/api/src/main/java/org/eclipse/rdf4j/rio/helpers/RDFaParserSettings.java b/core/rio/api/src/main/java/org/eclipse/rdf4j/rio/helpers/RDFaParserSettings.java index 76d7ead5af1..94ef5ecd731 100644 --- a/core/rio/api/src/main/java/org/eclipse/rdf4j/rio/helpers/RDFaParserSettings.java +++ b/core/rio/api/src/main/java/org/eclipse/rdf4j/rio/helpers/RDFaParserSettings.java @@ -35,7 +35,8 @@ public class RDFaParserSettings { /** * Enables or disables vocabulary - * expansion feature. + * expansion feature. Note that although these settings are not used within RDF4J, they are in use by external + * plugins. *

* Defaults to false *

@@ -43,7 +44,6 @@ public class RDFaParserSettings { * * @see RDFa Vocabulary Expansion */ - @Deprecated(since = "4.3.0", forRemoval = true) public static final BooleanRioSetting VOCAB_EXPANSION_ENABLED = new BooleanRioSetting( "org.eclipse.rdf4j.rio.rdfa.vocab_expansion", "Vocabulary Expansion", Boolean.FALSE); diff --git a/core/sail/api/src/main/java/org/eclipse/rdf4j/common/concurrent/locks/ExclusiveReentrantLockManager.java b/core/sail/api/src/main/java/org/eclipse/rdf4j/common/concurrent/locks/ExclusiveReentrantLockManager.java new file mode 100644 index 00000000000..6ac7fa4df12 --- /dev/null +++ b/core/sail/api/src/main/java/org/eclipse/rdf4j/common/concurrent/locks/ExclusiveReentrantLockManager.java @@ -0,0 +1,174 @@ +/******************************************************************************* + * Copyright (c) 2024 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ +package org.eclipse.rdf4j.common.concurrent.locks; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.rdf4j.common.concurrent.locks.diagnostics.LockCleaner; +import org.eclipse.rdf4j.common.concurrent.locks.diagnostics.LockMonitoring; +import org.eclipse.rdf4j.common.concurrent.locks.diagnostics.LockTracking; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A simple reentrant lock that allows other threads to unlock the lock. + * + * @author Håvard M. Ottestad + */ +public class ExclusiveReentrantLockManager { + + private final static Logger logger = LoggerFactory.getLogger(ExclusiveReentrantLockManager.class); + + // the underlying lock object + final AtomicLong activeLocks = new AtomicLong(); + final AtomicReference owner = new AtomicReference<>(); + + private final int waitToCollect; + + LockMonitoring lockMonitoring; + + public ExclusiveReentrantLockManager() { + this(false); + } + + public ExclusiveReentrantLockManager(boolean trackLocks) { + this(trackLocks, LockMonitoring.INITIAL_WAIT_TO_COLLECT); + } + + public ExclusiveReentrantLockManager(boolean trackLocks, int collectionFrequency) { + + this.waitToCollect = collectionFrequency; + + if (trackLocks || Properties.lockTrackingEnabled()) { + + lockMonitoring = new LockTracking( + true, + "ExclusiveReentrantLockManager", + LoggerFactory.getLogger(this.getClass()), + waitToCollect, + Lock.ExtendedSupplier.wrap(this::getExclusiveLockInner, this::tryExclusiveLockInner) + ); + + } else { + lockMonitoring = new LockCleaner( + false, + "ExclusiveReentrantLockManager", + LoggerFactory.getLogger(this.getClass()), + Lock.ExtendedSupplier.wrap(this::getExclusiveLockInner, this::tryExclusiveLockInner) + ); + } + + } + + private Lock tryExclusiveLockInner() { + + synchronized (owner) { + if (owner.get() == Thread.currentThread()) { + activeLocks.incrementAndGet(); + return new ExclusiveReentrantLock(owner, activeLocks); + } + + if (owner.compareAndSet(null, Thread.currentThread())) { + activeLocks.incrementAndGet(); + return new ExclusiveReentrantLock(owner, activeLocks); + } + } + + return null; + + } + + private Lock getExclusiveLockInner() throws InterruptedException { + + synchronized (owner) { + + if (lockMonitoring.requiresManualCleanup()) { + do { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + Lock lock = tryExclusiveLockInner(); + if (lock != null) { + return lock; + } else { + lockMonitoring.runCleanup(); + owner.wait(waitToCollect); + } + } while (true); + } else { + while (true) { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + Lock lock = tryExclusiveLockInner(); + if (lock != null) { + return lock; + } else { + owner.wait(waitToCollect); + } + } + } + } + } + + public Lock tryExclusiveLock() { + return lockMonitoring.tryLock(); + } + + public Lock getExclusiveLock() throws InterruptedException { + return lockMonitoring.getLock(); + } + + public boolean isActiveLock() { + return owner.get() != null; + } + + static class ExclusiveReentrantLock implements Lock { + + final AtomicLong activeLocks; + final AtomicReference owner; + private boolean released = false; + + public ExclusiveReentrantLock(AtomicReference owner, AtomicLong activeLocks) { + this.owner = owner; + this.activeLocks = activeLocks; + } + + @Override + public boolean isActive() { + return !released; + } + + @Override + public void release() { + if (released) { + throw new IllegalStateException("Lock already released"); + } + + synchronized (owner) { + if (owner.get() != Thread.currentThread()) { + logger.warn("Releasing lock from different thread, owner: " + owner.get() + ", current: " + + Thread.currentThread()); + } + + if (activeLocks.decrementAndGet() == 0) { + owner.set(null); + owner.notifyAll(); + } + } + + released = true; + + } + } + +} diff --git a/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java b/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java index 589d3d8a8ef..ca9bb7b60fb 100644 --- a/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java +++ b/core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/AbstractSailConnection.java @@ -19,6 +19,7 @@ import java.util.LinkedList; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.ReentrantLock; @@ -94,6 +95,7 @@ public abstract class AbstractSailConnection implements SailConnection { */ private final LongAdder blockClose = new LongAdder(); private final LongAdder unblockClose = new LongAdder(); + private final AtomicReference activeThread = new AtomicReference<>(); @SuppressWarnings("FieldMayBeFinal") private boolean isOpen = true; @@ -107,7 +109,6 @@ public abstract class AbstractSailConnection implements SailConnection { * */ private final ReentrantLock updateLock = new ReentrantLock(); - private final LongAdder iterationsOpened = new LongAdder(); private final LongAdder iterationsClosed = new LongAdder(); @@ -196,6 +197,8 @@ public void begin(IsolationLevel isolationLevel) throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -210,7 +213,12 @@ public void begin(IsolationLevel isolationLevel) throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } + } startUpdate(null); } @@ -231,7 +239,6 @@ public boolean isActive() throws UnknownSailTransactionStateException { @Override public final void close() throws SailException { - Thread deadlockPreventionThread = startDeadlockPreventionThread(); // obtain an exclusive lock so that any further operations on this // connection (including those from any concurrent threads) are blocked. @@ -240,96 +247,75 @@ public final void close() throws SailException { } try { - while (true) { - long sumDone = unblockClose.sum(); - long sumBlocking = blockClose.sum(); - if (sumDone == sumBlocking) { - break; - } else { - if (Thread.currentThread().isInterrupted()) { - throw new SailException( - "Connection was interrupted while waiting on active operations before it could be closed."); - } else { - LockSupport.parkNanos(Duration.ofMillis(10).toNanos()); - } - } - } + waitForOtherOperations(true); try { - forceCloseActiveOperations(); - - if (txnActive) { - logger.warn("Rolling back transaction due to connection close", - debugEnabled ? new Throwable() : null); - try { - // Use internal method to avoid deadlock: the public - // rollback method will try to obtain a connection lock - rollbackInternal(); - } finally { - txnActive = false; - txnPrepared = false; + try { + forceCloseActiveOperations(); + } finally { + if (txnActive) { + logger.warn("Rolling back transaction due to connection close", + debugEnabled ? new Throwable() : null); + try { + // Use internal method to avoid deadlock: the public + // rollback method will try to obtain a connection lock + rollbackInternal(); + } finally { + txnActive = false; + txnPrepared = false; + } } - } - closeInternal(); + closeInternal(); - if (isActiveOperation()) { - throw new SailException("Connection closed before all iterations were closed."); + if (isActiveOperation()) { + throw new SailException("Connection closed before all iterations were closed."); + } } + } finally { sailBase.connectionClosed(this); } } finally { - if (deadlockPreventionThread != null) { - deadlockPreventionThread.interrupt(); - } } } - /** - * If the current thread is not the owner, starts a thread to handle potential deadlocks by interrupting the owner. - * - * @return The started deadlock prevention thread or null if the current thread is the owner. - */ - private Thread startDeadlockPreventionThread() { - Thread deadlockPreventionThread = null; - - if (Thread.currentThread() != owner) { - - if (logger.isInfoEnabled()) { - // use info level for this because FedX prevalently closes connections from different threads - logger.info( - "Closing connection from a different thread than the one that opened it. Connections should not be shared between threads. Opened by {} closed by {}", - owner, Thread.currentThread(), new Throwable("Throwable used for stacktrace")); - } - - deadlockPreventionThread = new Thread(() -> { - try { - // This thread should sleep for a while so that the callee has a chance to finish. - // The callee will interrupt this thread when it is finished, which means that there were no - // deadlocks and we can exit. - Thread.sleep(sailBase.connectionTimeOut / 2); - - owner.interrupt(); - // wait for up to 1 second for the owner thread to die - owner.join(1000); - if (owner.isAlive()) { - logger.error("Interrupted thread {} but thread is still alive after 1000 ms!", owner); + @InternalUseOnly + public void waitForOtherOperations(boolean interrupt) { + int i = 0; + boolean interrupted = false; + while (true) { + long sumDone = unblockClose.sum(); + long sumBlocking = blockClose.sum(); + if (sumDone == sumBlocking) { + if (interrupted) { + logger.warn( + "Connection is no longer blocked by concurrent operation after interrupting the active thread"); + } + break; + } else { + if (Thread.currentThread().isInterrupted()) { + throw new SailException( + "Connection was interrupted while waiting on concurrent operations before it could be closed."); + } else { + LockSupport.parkNanos(Duration.ofMillis(10).toNanos()); + if (++i % 500 == 0) { // wait for 5 seconds before logging and interrupting + Thread acquire = activeThread.getAcquire(); + if (acquire != null) { + logger.warn("Connection is blocked by concurrent operation in thread: {}", acquire); + if (interrupt) { + acquire.interrupt(); + interrupted = true; + logger.error( + "Connection is blocked by concurrent operation in thread {} which was interrupted to attempt to forcefully abort the concurrent operation.", + acquire); + } + } } - - } catch (InterruptedException ignored) { - // this thread is interrupted as a signal that there were no deadlocks, so the exception can be - // ignored and we can simply exit } - - }); - - deadlockPreventionThread.setDaemon(true); - deadlockPreventionThread.start(); - + } } - return deadlockPreventionThread; } @Override @@ -339,6 +325,8 @@ public final CloseableIteration evaluate(TupleExpr tupleEx blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); CloseableIteration iteration = null; try { @@ -354,7 +342,11 @@ public final CloseableIteration evaluate(TupleExpr tupleEx throw t; } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -364,10 +356,16 @@ public final CloseableIteration getContextIDs() throws SailE blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); return registerIteration(getContextIDsInternal()); } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -378,6 +376,7 @@ public final CloseableIteration getStatements(Resource subj blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); verifyIsOpen(); CloseableIteration iteration = null; try { @@ -390,7 +389,11 @@ public final CloseableIteration getStatements(Resource subj throw t; } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -424,10 +427,15 @@ public final boolean hasStatement(Resource subj, IRI pred, Value obj, boolean in blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); verifyIsOpen(); return hasStatementInternal(subj, pred, obj, includeInferred, contexts); } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -445,10 +453,15 @@ public final long size(Resource... contexts) throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); verifyIsOpen(); return sizeInternal(contexts); } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -489,6 +502,7 @@ public final void prepare() throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); verifyIsOpen(); updateLock.lock(); @@ -501,7 +515,11 @@ public final void prepare() throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -513,6 +531,8 @@ public final void commit() throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -529,7 +549,11 @@ public final void commit() throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -544,6 +568,8 @@ public final void rollback() throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -563,7 +589,11 @@ public final void rollback() throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -660,6 +690,8 @@ public final void endUpdate(UpdateContext op) throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -670,8 +702,11 @@ public final void endUpdate(UpdateContext op) throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); - + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } if (op != null) { flush(); } @@ -711,6 +746,8 @@ public final void clear(Resource... contexts) throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -722,19 +759,27 @@ public final void clear(Resource... contexts) throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @Override public final CloseableIteration getNamespaces() throws SailException { - blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); verifyIsOpen(); return registerIteration(getNamespacesInternal()); } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -745,11 +790,18 @@ public final String getNamespace(String prefix) throws SailException { } blockClose.increment(); + try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); return getNamespaceInternal(prefix); } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -764,6 +816,8 @@ public final void setNamespace(String prefix, String name) throws SailException blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -774,7 +828,11 @@ public final void setNamespace(String prefix, String name) throws SailException updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -786,6 +844,8 @@ public final void removeNamespace(String prefix) throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -796,7 +856,11 @@ public final void removeNamespace(String prefix) throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -805,6 +869,8 @@ public final void clearNamespaces() throws SailException { blockClose.increment(); try { + activeThread.setRelease(Thread.currentThread()); + verifyIsOpen(); updateLock.lock(); @@ -815,7 +881,11 @@ public final void clearNamespaces() throws SailException { updateLock.unlock(); } } finally { - unblockClose.increment(); + try { + activeThread.setRelease(null); + } finally { + unblockClose.increment(); + } } } @@ -930,47 +1000,40 @@ protected AbstractSail getSailBase() { } private void forceCloseActiveOperations() throws SailException { - Thread deadlockPreventionThread = startDeadlockPreventionThread(); - try { - for (int i = 0; i < 10 && isActiveOperation() && !debugEnabled; i++) { - System.gc(); - try { - Thread.sleep(1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SailException(e); - } + for (int i = 0; i < 10 && isActiveOperation() && !debugEnabled; i++) { + System.gc(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SailException(e); } + } - if (debugEnabled) { + if (debugEnabled) { - var activeIterationsCopy = new IdentityHashMap<>(activeIterationsDebug); - activeIterationsDebug.clear(); + var activeIterationsCopy = new IdentityHashMap<>(activeIterationsDebug); + activeIterationsDebug.clear(); - if (!activeIterationsCopy.isEmpty()) { - for (var entry : activeIterationsCopy.entrySet()) { - try { - logger.warn("Unclosed iteration", entry.getValue()); - entry.getKey().close(); - } catch (Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - throw new SailException(e); - } - logger.warn("Exception occurred while closing unclosed iterations.", e); + if (!activeIterationsCopy.isEmpty()) { + for (var entry : activeIterationsCopy.entrySet()) { + try { + logger.warn("Unclosed iteration", entry.getValue()); + entry.getKey().close(); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + throw new SailException(e); } + logger.warn("Exception occurred while closing unclosed iterations.", e); } + } - var entry = activeIterationsCopy.entrySet().stream().findAny().orElseThrow(); + var entry = activeIterationsCopy.entrySet().stream().findAny().orElseThrow(); - throw new SailException( - "Connection closed before all iterations were closed: " + entry.getKey().toString(), - entry.getValue()); - } - } - } finally { - if (deadlockPreventionThread != null) { - deadlockPreventionThread.interrupt(); + throw new SailException( + "Connection closed before all iterations were closed: " + entry.getKey().toString(), + entry.getValue()); } } diff --git a/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/ExclusiveReentrantLockManagerTest.java b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/ExclusiveReentrantLockManagerTest.java new file mode 100644 index 00000000000..83fe59d93ea --- /dev/null +++ b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/ExclusiveReentrantLockManagerTest.java @@ -0,0 +1,150 @@ +/******************************************************************************* + * Copyright (c) 2024 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.common.concurrent.locks; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; + +class ExclusiveReentrantLockManagerTest { + + private ExclusiveReentrantLockManager lockManager; + private ExclusiveReentrantLockManager lockManagerTracking; + private MemoryAppender memoryAppender; + + @BeforeEach + void beforeEach() { + Properties.setLockTrackingEnabled(false); + lockManager = new ExclusiveReentrantLockManager(false, 1); + lockManagerTracking = new ExclusiveReentrantLockManager(true, 1); + + Logger logger = (Logger) LoggerFactory.getLogger(ExclusiveReentrantLockManager.class.getName()); + memoryAppender = new MemoryAppender(); + memoryAppender.setContext((LoggerContext) LoggerFactory.getILoggerFactory()); + logger.detachAndStopAllAppenders(); + logger.setLevel(Level.INFO); + logger.addAppender(memoryAppender); + memoryAppender.start(); + + } + + @Test + void createLock() throws InterruptedException { + Lock lock = lockManager.getExclusiveLock(); + assertTrue(lock.isActive()); + lock.release(); + assertFalse(lock.isActive()); + } + + @Test + @Timeout(2) + void cleanupUnreleasedLocks() throws InterruptedException { + + lock(lockManager); + + TestHelper.callGC(lockManager); + + Lock exclusiveLock = lockManager.getExclusiveLock(); + exclusiveLock.release(); + + } + + @Test + @Timeout(2) + void cleanupUnreleasedLocksWithTracking() throws InterruptedException { + + lock(lockManagerTracking); + + Lock exclusiveLock = lockManagerTracking.getExclusiveLock(); + exclusiveLock.release(); + + memoryAppender.waitForEvents(2); + + assertThat(memoryAppender.countEventsForLogger(ExclusiveReentrantLockManager.class.getName())).isEqualTo(2); + memoryAppender.assertContains( + "at org.eclipse.rdf4j.common.concurrent.locks.ExclusiveReentrantLockManagerTest.lambda$lock$2", + Level.WARN); + + } + + @Test + @Timeout(2) + void stalledTest() throws InterruptedException { + + AtomicReference exclusiveLock1 = new AtomicReference<>(); + Thread thread = new Thread(() -> { + try { + exclusiveLock1.set(lockManagerTracking.getExclusiveLock()); + } catch (InterruptedException ignored) { + } + }); + thread.start(); + thread.join(); + + try { + thread = new Thread(() -> { + try { + Lock exclusiveLock2 = lockManagerTracking.getExclusiveLock(); + exclusiveLock2.release(); + } catch (InterruptedException ignored) { + } + }); + + thread.setDaemon(true); + thread.start(); + + memoryAppender.waitForEvents(); + + } finally { + TestHelper.interruptAndJoin(thread); + } + + assertNull(lockManagerTracking.tryExclusiveLock()); + assertTrue(exclusiveLock1.get().isActive()); + exclusiveLock1.get().release(); + assertFalse(exclusiveLock1.get().isActive()); + + memoryAppender.waitForEvents(2); + + assertThat(memoryAppender.countEventsForLogger(ExclusiveReentrantLockManager.class.getName())) + .isGreaterThanOrEqualTo(1); + memoryAppender.assertContains("is waiting on a possibly stalled lock \"ExclusiveReentrantLockManager\" with id", + Level.INFO); + memoryAppender.assertContains( + "at org.eclipse.rdf4j.common.concurrent.locks.ExclusiveReentrantLockManagerTest.lambda$stalledTest$0(ExclusiveReentrantLockManagerTest.java:", + Level.INFO); + + } + + private void lock(ExclusiveReentrantLockManager lockManager) throws InterruptedException { + Thread thread = new Thread(() -> { + try { + lockManager.getExclusiveLock(); + } catch (InterruptedException ignored) { + } + }); + thread.start(); + thread.join(2000); + assertThat(thread.isAlive()).isFalse(); + } +} diff --git a/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/MemoryAppender.java b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/MemoryAppender.java index e1fc0fc3b19..1e3d9b9e171 100644 --- a/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/MemoryAppender.java +++ b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/MemoryAppender.java @@ -99,6 +99,18 @@ public String toString() { public void waitForEvents() { while (list.isEmpty()) { try { + System.gc(); + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + public void waitForEvents(int size) { + while (list.size() < size) { + try { + System.gc(); Thread.sleep(1); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/TestHelper.java b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/TestHelper.java index cee18c9e1a8..1c57b527465 100644 --- a/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/TestHelper.java +++ b/core/sail/api/src/test/java/org/eclipse/rdf4j/common/concurrent/locks/TestHelper.java @@ -42,6 +42,13 @@ public static void callGC(LockManager lockManager) throws InterruptedException { } } + public static void callGC(ExclusiveReentrantLockManager lockManager) throws InterruptedException { + while (lockManager.isActiveLock()) { + System.gc(); + Thread.sleep(1); + } + } + public static void interruptAndJoin(Thread thread) throws InterruptedException { assertNotNull(thread); thread.interrupt(); diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbContextIdIterator.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbContextIdIterator.java index 5dd7c82a954..1e81df6fd98 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbContextIdIterator.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbContextIdIterator.java @@ -104,12 +104,12 @@ public long[] next() { Varint.writeUnsigned(minKeyBuf, record[0]); minKeyBuf.flip(); keyData.mv_data(minKeyBuf); - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET)); - if (lastResult != 0) { + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET); + if (lastResult != MDB_SUCCESS) { // use MDB_SET_RANGE if key was deleted - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); } - if (lastResult != 0) { + if (lastResult != MDB_SUCCESS) { closeInternal(false); return null; } @@ -119,16 +119,16 @@ public long[] next() { } if (fetchNext) { - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); fetchNext = false; } else { if (minKeyBuf != null) { // set cursor to min key keyData.mv_data(minKeyBuf); - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); } else { // set cursor to first item - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); } } diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java index 1bcd6938d3c..dbdb69479f7 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java @@ -138,12 +138,12 @@ public long[] next() { index.toKey(minKeyBuf, quad[0], quad[1], quad[2], quad[3]); minKeyBuf.flip(); keyData.mv_data(minKeyBuf); - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET)); - if (lastResult != 0) { + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET); + if (lastResult != MDB_SUCCESS) { // use MDB_SET_RANGE if key was deleted - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); } - if (lastResult != 0) { + if (lastResult != MDB_SUCCESS) { closeInternal(false); return null; } @@ -153,16 +153,16 @@ public long[] next() { } if (fetchNext) { - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); fetchNext = false; } else { if (minKeyBuf != null) { // set cursor to min key keyData.mv_data(minKeyBuf); - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); } else { // set cursor to first item - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); } } @@ -172,7 +172,7 @@ public long[] next() { lastResult = MDB_NOTFOUND; } else if (groupMatcher != null && !groupMatcher.matches(keyData.mv_data())) { // value doesn't match search key/mask, fetch next value - lastResult = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); + lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); } else { // Matching value found index.keyToQuad(keyData.mv_data(), quad); @@ -183,8 +183,6 @@ public long[] next() { } closeInternal(false); return null; - } catch (IOException e) { - throw new SailException(e); } finally { txnLock.unlockRead(stamp); } diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java index 76f72f415e2..02e7d71bf5d 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java @@ -572,12 +572,14 @@ public void approve(Resource subj, IRI pred, Value obj, Resource ctx) throws Sai @Override public void approveAll(Set approved, Set approvedContexts) { + Statement last = null; sinkStoreAccessLock.lock(); try { startTransaction(true); for (Statement statement : approved) { + last = statement; Resource subj = statement.getSubject(); IRI pred = statement.getPredicate(); Value obj = statement.getObject(); @@ -604,13 +606,20 @@ public void approveAll(Set approved, Set approvedContexts) } } - } catch (IOException e) { + } catch (IOException | RuntimeException e) { rollback(); + if (multiThreadingActive) { + logger.error("Encountered an unexpected problem while trying to add a statement.", e); + } else { + logger.error( + "Encountered an unexpected problem while trying to add a statement. Last statement that was attempted to be added: [ {} ]", + last, e); + } + + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } throw new SailException(e); - } catch (RuntimeException e) { - rollback(); - logger.error("Encountered an unexpected problem while trying to add a statement", e); - throw e; } finally { sinkStoreAccessLock.unlock(); } diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbUtil.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbUtil.java index 89e1650686a..9a2c68f7083 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbUtil.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbUtil.java @@ -16,6 +16,7 @@ import static org.lwjgl.system.MemoryStack.stackPush; import static org.lwjgl.system.MemoryUtil.NULL; +import static org.lwjgl.util.lmdb.LMDB.MDB_DBS_FULL; import static org.lwjgl.util.lmdb.LMDB.MDB_KEYEXIST; import static org.lwjgl.util.lmdb.LMDB.MDB_NOTFOUND; import static org.lwjgl.util.lmdb.LMDB.MDB_RDONLY; @@ -39,12 +40,16 @@ import org.lwjgl.system.Pointer; import org.lwjgl.util.lmdb.MDBCmpFuncI; import org.lwjgl.util.lmdb.MDBVal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Utility class for working with LMDB. */ final class LmdbUtil { + private static final Logger logger = LoggerFactory.getLogger(LmdbUtil.class); + /** * Minimum free space in an LMDB db before automatically resizing the map. */ @@ -61,7 +66,9 @@ private LmdbUtil() { static int E(int rc) throws IOException { if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND && rc != MDB_KEYEXIST) { - throw new IOException(mdb_strerror(rc)); + IOException ioException = new IOException(mdb_strerror(rc)); + logger.info("Possible LMDB error: {}", mdb_strerror(rc), ioException); + throw ioException; } return rc; } @@ -105,7 +112,7 @@ static T transaction(long env, Transaction transaction) throws IOExceptio int err; try { ret = transaction.exec(stack, txn); - err = E(mdb_txn_commit(txn)); + err = mdb_txn_commit(txn); } catch (Throwable t) { mdb_txn_abort(txn); throw t; diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/PersistentSet.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/PersistentSet.java index b0bbd20aa38..79d27cf4b7a 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/PersistentSet.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/PersistentSet.java @@ -12,6 +12,7 @@ import static org.eclipse.rdf4j.sail.lmdb.LmdbUtil.E; import static org.lwjgl.system.MemoryUtil.NULL; +import static org.lwjgl.util.lmdb.LMDB.MDB_MAP_FULL; import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT; import static org.lwjgl.util.lmdb.LMDB.MDB_NOOVERWRITE; import static org.lwjgl.util.lmdb.LMDB.MDB_SET; @@ -24,6 +25,7 @@ import static org.lwjgl.util.lmdb.LMDB.mdb_del; import static org.lwjgl.util.lmdb.LMDB.mdb_drop; import static org.lwjgl.util.lmdb.LMDB.mdb_put; +import static org.lwjgl.util.lmdb.LMDB.mdb_strerror; import static org.lwjgl.util.lmdb.LMDB.mdb_txn_abort; import static org.lwjgl.util.lmdb.LMDB.mdb_txn_begin; @@ -43,12 +45,16 @@ import org.lwjgl.PointerBuffer; import org.lwjgl.system.MemoryStack; import org.lwjgl.util.lmdb.MDBVal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A LMDB-based persistent set. */ class PersistentSet extends AbstractSet { + private static final Logger logger = LoggerFactory.getLogger(PersistentSet.class); + private PersistentSetFactory factory; private final int dbi; private int size; @@ -126,15 +132,35 @@ private synchronized boolean update(Object element, boolean add) throws IOExcept keyVal.mv_data(keyBuf); if (add) { - if (E(mdb_put(factory.writeTxn, dbi, keyVal, dataVal, MDB_NOOVERWRITE)) == MDB_SUCCESS) { + int rc = mdb_put(factory.writeTxn, dbi, keyVal, dataVal, MDB_NOOVERWRITE); + if (rc == MDB_SUCCESS) { size++; return true; + } else if (rc == MDB_MAP_FULL) { + factory.ensureResize(); + if (mdb_put(factory.writeTxn, dbi, keyVal, dataVal, MDB_NOOVERWRITE) == MDB_SUCCESS) { + size++; + return true; + } + return false; + } else { + logger.debug("Failed to add element due to error {}: {}", mdb_strerror(rc), element); } } else { // delete element - if (mdb_del(factory.writeTxn, dbi, keyVal, dataVal) == MDB_SUCCESS) { + int rc = mdb_del(factory.writeTxn, dbi, keyVal, dataVal); + if (rc == MDB_SUCCESS) { size--; return true; + } else if (rc == MDB_MAP_FULL) { + factory.ensureResize(); + if (mdb_del(factory.writeTxn, dbi, keyVal, dataVal) == MDB_SUCCESS) { + size--; + return true; + } + return false; + } else { + logger.debug("Failed to remove element due to error {}: {}", mdb_strerror(rc), element); } } return false; diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TripleStore.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TripleStore.java index 7c0c79c7448..d848d30dd5b 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TripleStore.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TripleStore.java @@ -561,7 +561,7 @@ protected void filterUsedIds(Collection ids) throws IOException { keyBuf.clear(); Varint.writeUnsigned(keyBuf, id); keyData.mv_data(keyBuf.flip()); - if (E(mdb_get(txn, contextsDbi, keyData, valueData)) == MDB_SUCCESS) { + if (mdb_get(txn, contextsDbi, keyData, valueData) == MDB_SUCCESS) { it.remove(); } } @@ -587,7 +587,7 @@ protected void filterUsedIds(Collection ids) throws IOException { if (fullScan) { long[] quad = new long[4]; - int rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_FIRST)); + int rc = mdb_cursor_get(cursor, keyData, valueData, MDB_FIRST); while (rc == MDB_SUCCESS && !ids.isEmpty()) { index.keyToQuad(keyData.mv_data(), quad); ids.remove(quad[0]); @@ -595,7 +595,7 @@ protected void filterUsedIds(Collection ids) throws IOException { ids.remove(quad[2]); ids.remove(quad[3]); - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); } } else { for (Iterator it = ids.iterator(); it.hasNext();) { @@ -625,15 +625,15 @@ protected void filterUsedIds(Collection ids) throws IOException { // set cursor to min key keyData.mv_data(keyBuf); - int rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); + int rc = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); boolean exists = false; - while (!exists && rc == 0) { + while (!exists && rc == MDB_SUCCESS) { if (mdb_cmp(txn, dbi, keyData, maxKey) > 0) { // id was not found break; } else if (!matcher.matches(keyData.mv_data())) { // value doesn't match search key/mask, fetch next value - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); } else { exists = true; } @@ -708,8 +708,8 @@ protected double cardinality(long subj, long pred, long obj, long context) throw // set cursor to min key keyData.mv_data(keyBuf); - int rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); - if (rc != 0 || mdb_cmp(txn, dbi, keyData, maxKey) >= 0) { + int rc = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); + if (rc != MDB_SUCCESS || mdb_cmp(txn, dbi, keyData, maxKey) >= 0) { break; } else { Varint.readListUnsigned(keyData.mv_data(), s.minValues); @@ -717,15 +717,15 @@ protected double cardinality(long subj, long pred, long obj, long context) throw // set cursor to max key keyData.mv_data(maxKeyBuf); - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); - if (rc != 0) { + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); + if (rc != MDB_SUCCESS) { // directly go to last value - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_LAST)); + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_LAST); } else { // go to previous value of selected key - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_PREV)); + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_PREV); } - if (rc == 0) { + if (rc == MDB_SUCCESS) { Varint.readListUnsigned(keyData.mv_data(), s.maxValues); // this is required to correctly estimate the range size at a later point s.startValues[s.MAX_BUCKETS] = s.maxValues; @@ -747,7 +747,7 @@ protected double cardinality(long subj, long pred, long obj, long context) throw keyData.mv_data(keyBuf); int currentSamplesCount = 0; - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); while (rc == MDB_SUCCESS && currentSamplesCount < s.MAX_SAMPLES_PER_BUCKET) { if (mdb_cmp(txn, dbi, keyData, maxKey) >= 0) { endOfRange = true; @@ -776,8 +776,8 @@ protected double cardinality(long subj, long pred, long obj, long context) throw } } } - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)); - if (rc != 0) { + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT); + if (rc != MDB_SUCCESS) { // no more elements are available endOfRange = true; } @@ -873,14 +873,14 @@ public boolean storeTriple(long subj, long pred, long obj, long context, boolean return recordCache.storeRecord(quad, explicit); } - int rc = E(mdb_put(writeTxn, mainIndex.getDB(explicit), keyVal, dataVal, MDB_NOOVERWRITE)); + int rc = mdb_put(writeTxn, mainIndex.getDB(explicit), keyVal, dataVal, MDB_NOOVERWRITE); if (rc != MDB_SUCCESS && rc != MDB_KEYEXIST) { throw new IOException(mdb_strerror(rc)); } stAdded = rc == MDB_SUCCESS; boolean foundImplicit = false; if (explicit && stAdded) { - foundImplicit = E(mdb_del(writeTxn, mainIndex.getDB(false), keyVal, dataVal)) == MDB_SUCCESS; + foundImplicit = mdb_del(writeTxn, mainIndex.getDB(false), keyVal, dataVal) == MDB_SUCCESS; } if (stAdded) { @@ -920,7 +920,7 @@ private void incrementContext(MemoryStack stack, long context) throws IOExceptio idVal.mv_data(bb); MDBVal dataVal = MDBVal.calloc(stack); long newCount = 1; - if (E(mdb_get(writeTxn, contextsDbi, idVal, dataVal)) == MDB_SUCCESS) { + if (mdb_get(writeTxn, contextsDbi, idVal, dataVal) == MDB_SUCCESS) { // update count newCount = Varint.readUnsigned(dataVal.mv_data()) + 1; } @@ -944,7 +944,7 @@ private boolean decrementContext(MemoryStack stack, long context) throws IOExcep bb.flip(); idVal.mv_data(bb); MDBVal dataVal = MDBVal.calloc(stack); - if (E(mdb_get(writeTxn, contextsDbi, idVal, dataVal)) == MDB_SUCCESS) { + if (mdb_get(writeTxn, contextsDbi, idVal, dataVal) == MDB_SUCCESS) { // update count long newCount = Varint.readUnsigned(dataVal.mv_data()) - 1; if (newCount <= 0) { diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TxnRecordCache.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TxnRecordCache.java index 27aa38d67a8..73dff3fb1a8 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TxnRecordCache.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/TxnRecordCache.java @@ -135,9 +135,9 @@ protected boolean update(long[] quad, boolean explicit, boolean add) throws IOEx keyBuf.flip(); keyVal.mv_data(keyBuf); - boolean foundExplicit = E(mdb_get(writeTxn, dbiExplicit, keyVal, dataVal)) == MDB_SUCCESS && + boolean foundExplicit = mdb_get(writeTxn, dbiExplicit, keyVal, dataVal) == MDB_SUCCESS && (dataVal.mv_data().get(0) & 0b1) != 0; - boolean foundImplicit = !foundExplicit && E(mdb_get(writeTxn, dbiInferred, keyVal, dataVal)) == MDB_SUCCESS + boolean foundImplicit = !foundExplicit && mdb_get(writeTxn, dbiInferred, keyVal, dataVal) == MDB_SUCCESS && (dataVal.mv_data().get(0) & 0b1) != 0; @@ -197,17 +197,13 @@ protected RecordCacheIterator(int dbi) throws IOException { } public Record next() { - try { - if (E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)) == MDB_SUCCESS) { - Varint.readListUnsigned(keyData.mv_data(), quad); - byte op = valueData.mv_data().get(0); - Record r = new Record(); - r.quad = quad; - r.add = op == 1; - return r; - } - } catch (IOException e) { - throw new SailException(e); + if (mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT) == MDB_SUCCESS) { + Varint.readListUnsigned(keyData.mv_data(), quad); + byte op = valueData.mv_data().get(0); + Record r = new Record(); + r.quad = quad; + r.add = op == 1; + return r; } close(); return null; diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java index 9242309f420..a0ef94ce123 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java @@ -214,13 +214,13 @@ class ValueStore extends AbstractValueFactory { // set cursor after max ID keyData.mv_data(stack.bytes(new byte[] { ID_KEY, (byte) 0xFF })); MDBVal valueData = MDBVal.calloc(stack); - int rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); - if (rc != 0) { + int rc = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); + if (rc != MDB_SUCCESS) { // directly go to last value - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_LAST)); + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_LAST); } else { // go to previous value of selected key - rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_PREV)); + rc = mdb_cursor_get(cursor, keyData, valueData, MDB_PREV); } if (rc == MDB_SUCCESS && keyData.mv_data().get(0) == ID_KEY) { // remove lower 2 type bits @@ -257,7 +257,7 @@ private void logValues() throws IOException { // set cursor to min key keyData.mv_data(stack.bytes(new byte[] { ID_KEY })); MDBVal valueData = MDBVal.calloc(stack); - int rc = E(mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE)); + int rc = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE); while (rc == MDB_SUCCESS && keyData.mv_data().get(0) == ID_KEY) { long id = data2id(keyData.mv_data()); try { @@ -373,7 +373,7 @@ private long nextId(byte type) throws IOException { E(mdb_cursor_del(cursor, 0)); return value; } - freeIdsAvailable = E(mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT)) == MDB_SUCCESS; + freeIdsAvailable = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT) == MDB_SUCCESS; return null; } finally { if (cursor != 0) { @@ -429,7 +429,7 @@ protected byte[] getData(long id) throws IOException { MDBVal keyData = MDBVal.calloc(stack); keyData.mv_data(id2data(idBuffer(stack), id).flip()); MDBVal valueData = MDBVal.calloc(stack); - if (E(mdb_get(txn, dbi, keyData, valueData)) == MDB_SUCCESS) { + if (mdb_get(txn, dbi, keyData, valueData) == MDB_SUCCESS) { byte[] valueBytes = new byte[valueData.mv_data().remaining()]; valueData.mv_data().get(valueBytes); return valueBytes; @@ -616,7 +616,7 @@ private void incrementRefCount(MemoryStack stack, long writeTxn, byte[] data) th MDBVal dataVal = MDBVal.calloc(stack); idVal.mv_data(idBuffer(stack).put(ID_KEY).put(data, 1, idLength).flip()); long newCount = 1; - if (E(mdb_get(writeTxn, refCountsDbi, idVal, dataVal)) == MDB_SUCCESS) { + if (mdb_get(writeTxn, refCountsDbi, idVal, dataVal) == MDB_SUCCESS) { // update count newCount = Varint.readUnsigned(dataVal.mv_data()) + 1; } @@ -638,7 +638,7 @@ private boolean decrementRefCount(MemoryStack stack, long writeTxn, ByteBuffer i MDBVal idVal = MDBVal.calloc(stack); idVal.mv_data(idBb); MDBVal dataVal = MDBVal.calloc(stack); - if (E(mdb_get(writeTxn, refCountsDbi, idVal, dataVal)) == MDB_SUCCESS) { + if (mdb_get(writeTxn, refCountsDbi, idVal, dataVal) == MDB_SUCCESS) { // update count long newCount = Varint.readUnsigned(dataVal.mv_data()) - 1; if (newCount <= 0) { @@ -664,7 +664,7 @@ private long findId(byte[] data, boolean create) throws IOException { MDBVal dataVal = MDBVal.calloc(stack); dataVal.mv_data(stack.bytes(data)); MDBVal idVal = MDBVal.calloc(stack); - if (E(mdb_get(txn, dbi, dataVal, idVal)) == MDB_SUCCESS) { + if (mdb_get(txn, dbi, dataVal, idVal) == MDB_SUCCESS) { return data2id(idVal.mv_data()); } if (!create) { @@ -702,10 +702,9 @@ private long findId(byte[] data, boolean create) throws IOException { MDBVal dataVal = MDBVal.calloc(stack); // ID of first value is directly stored with hash as key - if (E(mdb_get(txn, dbi, hashVal, dataVal)) == MDB_SUCCESS) { + if (mdb_get(txn, dbi, hashVal, dataVal) == MDB_SUCCESS) { idVal.mv_data(dataVal.mv_data()); - if (E(mdb_get(txn, dbi, idVal, dataVal)) == MDB_SUCCESS && - dataVal.mv_data().compareTo(dataBb) == 0) { + if (mdb_get(txn, dbi, idVal, dataVal) == MDB_SUCCESS && dataVal.mv_data().compareTo(dataBb) == 0) { return data2id(idVal.mv_data()); } } else { @@ -745,7 +744,7 @@ private long findId(byte[] data, boolean create) throws IOException { cursor = pp.get(0); // iterate all entries for hash value - if (E(mdb_cursor_get(cursor, hashVal, dataVal, MDB_SET_RANGE)) == MDB_SUCCESS) { + if (mdb_cursor_get(cursor, hashVal, dataVal, MDB_SET_RANGE) == MDB_SUCCESS) { do { if (compareRegion(hashVal.mv_data(), 0, hashBb, 0, hashLength) != 0) { break; @@ -755,12 +754,12 @@ private long findId(byte[] data, boolean create) throws IOException { ByteBuffer hashIdBb = hashVal.mv_data(); hashIdBb.position(hashLength); idVal.mv_data(hashIdBb); - if (E(mdb_get(txn, dbi, idVal, dataVal)) == MDB_SUCCESS + if (mdb_get(txn, dbi, idVal, dataVal) == MDB_SUCCESS && dataVal.mv_data().compareTo(dataBb) == 0) { // id was found if stored value is equal to requested value return data2id(hashIdBb); } - } while (E(mdb_cursor_get(cursor, hashVal, dataVal, MDB_NEXT)) == MDB_SUCCESS); + } while (mdb_cursor_get(cursor, hashVal, dataVal, MDB_NEXT) == MDB_SUCCESS); } } finally { if (cursor != 0) { @@ -959,7 +958,7 @@ public void gcIds(Collection ids, Collection nextIds) throws IOExcep revIdVal.mv_data(id2data(revIdBb, id).flip()); // check if id has internal references and therefore cannot be deleted idVal.mv_data(revIdBb.slice().position(revLength)); - if (E(mdb_get(writeTxn, refCountsDbi, idVal, dataVal)) == MDB_SUCCESS) { + if (mdb_get(writeTxn, refCountsDbi, idVal, dataVal) == MDB_SUCCESS) { continue; } // mark id as unused @@ -998,10 +997,9 @@ protected void deleteValueToIdMappings(MemoryStack stack, long txn, Collection observations; - - private volatile boolean txnLock; + private volatile Lock txnLock; private boolean requireCleanup; @@ -583,6 +583,7 @@ public MemorySailSink(boolean explicit, boolean serializable) throws SailExcepti this.serializable = Integer.MAX_VALUE; this.reservedSnapshot = null; } + } @Override @@ -593,7 +594,7 @@ public String toString() { } else { sb.append("inferred "); } - if (txnLock) { + if (txnLock != null) { sb.append("snapshot ").append(nextSnapshot); } else { sb.append(super.toString()); @@ -632,18 +633,17 @@ public synchronized void prepare() throws SailException { } } } - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override public synchronized void flush() throws SailException { - if (txnLock) { + if (txnLock != null && txnLock.isActive()) { invalidateCache(); currentSnapshot = Math.max(currentSnapshot, nextSnapshot); if (requireCleanup) { scheduleSnapshotCleanup(); } - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; } } @@ -668,16 +668,10 @@ public void close() { } synchronized private void releaseLock() { - if (txnLock) { - try { - txnLock = false; - txnLockManager.unlock(); - } catch (IllegalMonitorStateException t) { - txnLock = true; - throw new SailException("Failed to release lock from thread " + Thread.currentThread() - + " because it was locked by another thread.", t); - } - + if (txnLock != null) { + assert txnLock.isActive(); + txnLock.release(); + txnLock = null; } } @@ -685,21 +679,21 @@ synchronized private void releaseLock() { public synchronized void setNamespace(String prefix, String name) { acquireExclusiveTransactionLock(); namespaceStore.setNamespace(prefix, name); - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override public synchronized void removeNamespace(String prefix) { acquireExclusiveTransactionLock(); namespaceStore.removeNamespace(prefix); - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override public synchronized void clearNamespaces() { acquireExclusiveTransactionLock(); namespaceStore.clear(); - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override @@ -735,7 +729,7 @@ public synchronized void clear(Resource... contexts) { } catch (InterruptedException e) { throw convertToSailException(e); } - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override @@ -747,7 +741,7 @@ public synchronized void approve(Resource subj, IRI pred, Value obj, Resource ct } catch (InterruptedException e) { throw convertToSailException(e); } - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override @@ -762,7 +756,7 @@ public synchronized void approveAll(Set approved, Set appro } catch (InterruptedException e) { throw convertToSailException(e); } - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override @@ -774,7 +768,7 @@ public synchronized void deprecateAll(Set deprecated) { for (Statement statement : deprecated) { innerDeprecate(statement, nextSnapshot); } - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } @Override @@ -783,7 +777,7 @@ public synchronized void deprecate(Statement statement) throws SailException { invalidateCache(); requireCleanup = true; innerDeprecate(statement, nextSnapshot); - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; + } private void innerDeprecate(Statement statement, int nextSnapshot) { @@ -817,16 +811,15 @@ private void innerDeprecate(Statement statement, int nextSnapshot) { } private void acquireExclusiveTransactionLock() throws SailException { - if (!txnLock) { + if (txnLock == null) { synchronized (this) { - if (!txnLock) { + if (txnLock == null) { try { - txnLockManager.lockInterruptibly(); + txnLock = txnLockManager.getExclusiveLock(); + nextSnapshot = currentSnapshot + 1; } catch (InterruptedException e) { throw convertToSailException(e); } - nextSnapshot = currentSnapshot + 1; - txnLock = true; } } @@ -929,7 +922,6 @@ public boolean deprecateByQuery(Resource subj, IRI pred, Value obj, Resource[] c throw convertToSailException(e); } invalidateCache(); - assert txnLock && txnLockManager.isHeldByCurrentThread() : "Should still be holding lock"; return deprecated; } diff --git a/core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ShaclSailConnection.java b/core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ShaclSailConnection.java index 83e2da56ad7..1e0416e76ad 100644 --- a/core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ShaclSailConnection.java +++ b/core/sail/shacl/src/main/java/org/eclipse/rdf4j/sail/shacl/ShaclSailConnection.java @@ -738,24 +738,7 @@ synchronized public void close() throws SailException { if (getWrappedConnection() instanceof AbstractSailConnection) { AbstractSailConnection abstractSailConnection = (AbstractSailConnection) getWrappedConnection(); - if (Thread.currentThread() != abstractSailConnection.getOwner()) { - Thread owner = abstractSailConnection.getOwner(); - logger.error( - "Closing connection from a different thread than the one that opened it. Connections should not be shared between threads. Opened by " - + owner + " closed by " + Thread.currentThread(), - new Throwable("Throwable used for stacktrace")); - owner.interrupt(); - try { - owner.join(1000); - if (owner.isAlive()) { - logger.error("Interrupted thread {} but thread is still alive after 1000 ms!", owner); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SailException(e); - } - } - + abstractSailConnection.waitForOtherOperations(true); } try { diff --git a/core/sail/solr/pom.xml b/core/sail/solr/pom.xml index 831a688cf4a..c79bfa393e7 100644 --- a/core/sail/solr/pom.xml +++ b/core/sail/solr/pom.xml @@ -12,8 +12,29 @@ false + + 3.7.2 + + 1.1.10.5 + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + org.apache.zookeeper + zookeeper-jute + ${zookeeper.version} + + + + org.xerial.snappy + snappy-java + ${snappy.version} + ${project.groupId} rdf4j-sail-lucene-api diff --git a/pom.xml b/pom.xml index d905efa8377..8338a334348 100644 --- a/pom.xml +++ b/pom.xml @@ -371,12 +371,13 @@ 8.9.0 8.9.0 7.15.2 - 5.3.30 + 5.3.37 32.1.3-jre 1.37 3.1.0 5.9.3 9.4.54.v20240208 + 4.1.111.Final @@ -396,6 +397,13 @@ pom import + + io.netty + netty-bom + ${netty.version} + pom + import + diff --git a/scripts/milestone-release.sh b/scripts/milestone-release.sh index 54a0c4c6bec..28830823be2 100755 --- a/scripts/milestone-release.sh +++ b/scripts/milestone-release.sh @@ -208,11 +208,11 @@ mvn clean -Dmaven.clean.failOnError=false # temporarily disable exiting on error set +e mvn clean -mvn install -DskipTests; -mvn package -Passembly -DskipTests +mvn install -DskipTests -Djapicmp.skip +mvn package -Passembly -DskipTests -Djapicmp.skip set -e -mvn package -Passembly -DskipTests +mvn package -Passembly -DskipTests -Djapicmp.skip git checkout main RELEASE_NOTES_BRANCH="${MVN_VERSION_RELEASE}-release-notes" diff --git a/scripts/release.sh b/scripts/release.sh index c7155b8e4df..bc634902b55 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -139,12 +139,12 @@ git checkout main; cd scripts rm -rf temp mkdir temp -echo "MVN_CURRENT_SNAPSHOT_VERSION=\"${MVN_CURRENT_SNAPSHOT_VERSION}\"" > temp/constants.txt -echo "MVN_VERSION_RELEASE=\"${MVN_VERSION_RELEASE}\"" > temp/constants.txt -echo "MVN_NEXT_SNAPSHOT_VERSION=\"${MVN_NEXT_SNAPSHOT_VERSION}\"" > temp/constants.txt -echo "BRANCH=\"${BRANCH}\"" > temp/constants.txt -echo "RELEASE_NOTES_BRANCH=\"${RELEASE_NOTES_BRANCH}\"" > temp/constants.txt -echo "MVN_VERSION_DEVELOP=\"${MVN_VERSION_DEVELOP}\"" > temp/constants.txt +echo "MVN_CURRENT_SNAPSHOT_VERSION=\"${MVN_CURRENT_SNAPSHOT_VERSION}\"" >> temp/constants.txt +echo "MVN_VERSION_RELEASE=\"${MVN_VERSION_RELEASE}\"" >> temp/constants.txt +echo "MVN_NEXT_SNAPSHOT_VERSION=\"${MVN_NEXT_SNAPSHOT_VERSION}\"" >> temp/constants.txt +echo "BRANCH=\"${BRANCH}\"" >> temp/constants.txt +echo "RELEASE_NOTES_BRANCH=\"${RELEASE_NOTES_BRANCH}\"" >> temp/constants.txt +echo "MVN_VERSION_DEVELOP=\"${MVN_VERSION_DEVELOP}\"" >> temp/constants.txt cd .. echo "Running maven clean and install -DskipTests"; @@ -262,11 +262,11 @@ git checkout "${MVN_VERSION_RELEASE}" # temporarily disable exiting on error set +e mvn clean -mvn install -DskipTests -mvn package -Passembly -DskipTests +mvn install -DskipTests -Djapicmp.skip +mvn package -Passembly -DskipTests -Djapicmp.skip set -e -mvn package -Passembly -DskipTests +mvn package -Passembly -DskipTests -Djapicmp.skip git checkout main diff --git a/site/content/download.md b/site/content/download.md index f6d81bad5fe..9e5587f1cd3 100644 --- a/site/content/download.md +++ b/site/content/download.md @@ -5,15 +5,15 @@ toc: true You can either retrieve RDF4J via Apache Maven, or download the SDK or onejar directly. -## RDF4J 4.3.12 (latest) +## RDF4J 5.0.0 (latest) -RDF4J 4.3.12 is our latest stable release. It requires Java 11 minimally. -For details on what’s new and how to upgrade, see the [release and upgrade notes](/release-notes/4.3.12). +RDF4J 5.0.0 is our latest stable release. It requires Java 11 minimally. +For details on what’s new and how to upgrade, see the [release and upgrade notes](/release-notes/5.0.0). -- [RDF4J 4.3.12 SDK (zip)](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-4.3.12-sdk.zip)
+- [RDF4J 5.0.0 SDK (zip)](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-5.0.0-sdk.zip)
Full Eclipse RDF4J SDK, containing all libraries, RDF4J Server, Workbench, and Console applications, and Javadoc API. -- [RDF4J 4.3.12 onejar](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-4.3.12-onejar.jar)
+- [RDF4J 5.0.0 onejar](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-5.0.0-onejar.jar)
Single jar file for easy inclusion of the full RDF4J toolkit in your Java project. - [RDF4J artifacts](https://search.maven.org/search?q=org.eclipse.rdf4j) on the [Maven Central Repository](http://search.maven.org/) @@ -28,7 +28,7 @@ You can include RDF4J as a Maven dependency in your Java project by including th org.eclipse.rdf4j rdf4j-bom - 4.3.12 + 5.0.0 pom import @@ -50,31 +50,21 @@ See the [Setup instructions](/documentation/programming/setup) in the [Programmer’s documentation](/documentation/) for more details on Maven and which artifacts RDF4J provides. +## Older releases -## RDF4J 5.0.0-M3 - -RDF4J 5.0.0-M3 is our latest milestone build of the upcoming 5.0.0 release. It requires Java 11 minimally. -For details on what’s new and how to upgrade, see the [release and upgrade notes](/news/rdf4j-500-M2.md). - -- [RDF4J 5.0.0-M3 SDK (zip)](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-5.0.0-M3-sdk.zip)
- Full Eclipse RDF4J SDK, containing all libraries, RDF4J Server, Workbench, and Console applications, and Javadoc API. - -- [RDF4J 5.0.0-M3 onejar](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-5.0.0-M3-onejar.jar)
- Single jar file for easy inclusion of the full RDF4J toolkit in your Java project. - -- [RDF4J artifacts](https://search.maven.org/search?q=org.eclipse.rdf4j) on the [Maven Central Repository](http://search.maven.org/) +### RDF4J 4.3 -RDF4J 5.0.0-M3 is also available through maven. +- [RDF4J 4.3.12 SDK (zip)](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-4.3.12-sdk.zip) +- [RDF4J 4.3.12 onejar](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-4.3.12-onejar.jar) -## Older releases - ### RDF4J 4.2 - [RDF4J 4.2.4 SDK (zip)](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-4.2.4-sdk.zip) - [RDF4J 4.2.4 onejar](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-4.2.4-onejar.jar) + ### RDF4J 4.1 - [RDF4J 4.1.3 SDK (zip)](http://www.eclipse.org/downloads/download.php?file=/rdf4j/eclipse-rdf4j-4.1.3-sdk.zip) diff --git a/site/content/news/rdf4j-500.md b/site/content/news/rdf4j-500.md new file mode 100644 index 00000000000..9311e6ee9f0 --- /dev/null +++ b/site/content/news/rdf4j-500.md @@ -0,0 +1,25 @@ +--- +title: "RDF4J 5.0.0 released" +date: 2024-06-21T10:01:02+0200 +layout: "single" +categories: ["news"] +--- +balloons +We are very excited to announce the release of RDF4J 5.0.0! + +RDF4J 5.0.0 is a major release of the RDF4J framework with many new features and improvements. + +Highlights include: +- JSON-LD 1.1 support +- Many improvements to FedX +- Improved SHACL validation with support for [sh:closed](https://www.w3.org/TR/shacl/#ClosedConstraintComponent) and [pairwise](https://www.w3.org/TR/shacl/#core-components-property-pairs) validation +- Stability and performance improvements to the LmdbStore +- Upgrade of MapDB + - More queries with intermediary results are no longer limited by RAM/java heap but disk space available + +For more details, including instruction on how to upgrade, see the [release notes](/release-notes/5.0.0). + +### Links + +- [Download RDF4J](/download/) +- [release notes](/release-notes/5.0.0) diff --git a/site/content/release-notes/5.0.0.md b/site/content/release-notes/5.0.0.md index 199d8dc735a..d495494f5ed 100644 --- a/site/content/release-notes/5.0.0.md +++ b/site/content/release-notes/5.0.0.md @@ -5,15 +5,21 @@ toc: true RDF4J 5.0.0 is a major release of the Eclipse RDF4J framework. Some highlights: - Replacement of the custom Iteration interface with Java Iterable/Iterator -- Upgrade of MapDB - Replacement of old openrdf.org config vocabulary IRIS with new rdf4j.org vocabulary +- Improved SHACL validation with support for [sh:closed](https://www.w3.org/TR/shacl/#ClosedConstraintComponent) and [pairwise](https://www.w3.org/TR/shacl/#core-components-property-pairs) validation +- Upgrade of MapDB + - More queries with intermediary results are no longer limited by RAM/java heap but disk space available +- Improve performance, query throughput and correctness in a transparent federation by refining various evaluation strategies (bind joins, property paths, limit pushing, ...) +- JSON-LD 1.1 support +- Implementation of merge join for future use in query evaluation +- Stability and performance improvements to the LmdbStore +- Improved spilling to disk for large transactions For a complete overview, see [all issues fixed in 5.0.0](https://github.com/eclipse/rdf4j/milestone/80?closed=1). ## Upgrade notes -RDF4J 5.0.0 contains several [backward incompatible -changes](https://github.com/eclipse/rdf4j/issues?q=is%3Aclosed+is%3Aissue+label%3A%22%E2%9B%94+Not+backwards+compatible%22+milestone%3A%225.0.0%22), including removal of several deprecated modules and classes. +RDF4J 5.0.0 contains several [backward incompatible changes](https://github.com/eclipse/rdf4j/issues?q=is%3Aclosed+is%3Aissue+label%3A%22%E2%9B%94+Not+backwards+compatible%22+milestone%3A%225.0.0%22), including removal of several deprecated modules and classes. ### Configuration vocabulary upgrade diff --git a/site/static/javadoc/5.0.0.tgz b/site/static/javadoc/5.0.0.tgz new file mode 100644 index 00000000000..5374065d234 Binary files /dev/null and b/site/static/javadoc/5.0.0.tgz differ diff --git a/site/static/javadoc/latest.tgz b/site/static/javadoc/latest.tgz index 657de822519..5374065d234 100644 Binary files a/site/static/javadoc/latest.tgz and b/site/static/javadoc/latest.tgz differ diff --git a/spring-components/rdf4j-spring/src/main/java/org/eclipse/rdf4j/spring/support/RDF4JTemplate.java b/spring-components/rdf4j-spring/src/main/java/org/eclipse/rdf4j/spring/support/RDF4JTemplate.java index 0dbac366732..86cf1d27143 100644 --- a/spring-components/rdf4j-spring/src/main/java/org/eclipse/rdf4j/spring/support/RDF4JTemplate.java +++ b/spring-components/rdf4j-spring/src/main/java/org/eclipse/rdf4j/spring/support/RDF4JTemplate.java @@ -315,7 +315,9 @@ public void delete(IRI start, List propertyPaths) { Variable p2 = SparqlBuilder.var("p2_" + i); Variable s2 = SparqlBuilder.var("s2_" + i); q.delete(target.has(p1, o1), s2.has(p2, target)) - .where(toIri(start).has(p, target).optional(), target.has(p1, o1).optional(), + .where( + toIri(start).has(p, target), + target.has(p1, o1).optional(), s2.has(p2, target).optional()); } update(q.getQueryString()).execute(); diff --git a/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/domain/model/EX.java b/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/domain/model/EX.java index d68ef9b6d9d..361b38061e7 100644 --- a/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/domain/model/EX.java +++ b/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/domain/model/EX.java @@ -34,6 +34,7 @@ public class EX { public static final IRI sunflowers = SimpleValueFactory.getInstance().createIRI(base, "sunflowers"); public static final IRI potatoEaters = SimpleValueFactory.getInstance().createIRI(base, "potatoEaters"); public static final IRI guernica = SimpleValueFactory.getInstance().createIRI(base, "guernica"); + public static final IRI homeAddress = SimpleValueFactory.getInstance().createIRI(base, "homeAddress"); public static IRI of(String localName) { return SimpleValueFactory.getInstance().createIRI(base, localName); diff --git a/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/support/RDF4JTemplateTests.java b/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/support/RDF4JTemplateTests.java index 99270244368..514b96b8804 100644 --- a/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/support/RDF4JTemplateTests.java +++ b/spring-components/rdf4j-spring/src/test/java/org/eclipse/rdf4j/spring/support/RDF4JTemplateTests.java @@ -7,10 +7,14 @@ * http://www.eclipse.org/org/documents/edl-v10.php. * * SPDX-License-Identifier: BSD-3-Clause - *******************************************************************************/ + ******************************************************************************/ package org.eclipse.rdf4j.spring.support; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.List; import java.util.Set; @@ -20,13 +24,20 @@ import org.eclipse.rdf4j.model.impl.SimpleValueFactory; import org.eclipse.rdf4j.model.vocabulary.FOAF; import org.eclipse.rdf4j.model.vocabulary.RDF; +import org.eclipse.rdf4j.query.BindingSet; import org.eclipse.rdf4j.sparqlbuilder.constraint.propertypath.builder.PropertyPathBuilder; import org.eclipse.rdf4j.sparqlbuilder.rdf.Rdf; import org.eclipse.rdf4j.spring.RDF4JSpringTestBase; +import org.eclipse.rdf4j.spring.dao.exception.IncorrectResultSetSizeException; import org.eclipse.rdf4j.spring.dao.support.opbuilder.UpdateExecutionBuilder; import org.eclipse.rdf4j.spring.dao.support.sparql.NamedSparqlSupplier; +import org.eclipse.rdf4j.spring.domain.dao.ArtistDao; +import org.eclipse.rdf4j.spring.domain.dao.PaintingDao; +import org.eclipse.rdf4j.spring.domain.model.Artist; import org.eclipse.rdf4j.spring.domain.model.EX; +import org.eclipse.rdf4j.spring.domain.model.Painting; import org.eclipse.rdf4j.spring.util.QueryResultUtils; +import org.eclipse.rdf4j.spring.util.TypeMappingUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -40,6 +51,13 @@ public class RDF4JTemplateTests extends RDF4JSpringTestBase { @Autowired private RDF4JTemplate rdf4JTemplate; + // used for checks + @Autowired + private ArtistDao artistDao; + + @Autowired + PaintingDao paintingDao; + @Test public void testUpdate1() { UpdateExecutionBuilder updateBuilder = rdf4JTemplate.update( @@ -427,6 +445,62 @@ public void testAssociate_deleteOutgoing() { } + @Test + public void testDeleteWithPropertyPaths() { + int triplesBeforeDelete = countTriples(); + Artist picasso = artistDao.getById(EX.Picasso); + assertNotNull(picasso); + Painting guernica = paintingDao.getById(EX.guernica); + assertNotNull(guernica); + rdf4JTemplate.delete(EX.Picasso, List.of(PropertyPathBuilder.of(EX.creatorOf).build())); + assertThrows(IncorrectResultSetSizeException.class, () -> artistDao.getById(EX.Picasso)); + assertThrows(IncorrectResultSetSizeException.class, () -> paintingDao.getById(EX.guernica)); + assertEquals(triplesBeforeDelete - 8, countTriples()); + } + + @Test + public void testDeleteWithDisjunctivePropertyPaths() { + int triplesBeforeDelete = countTriples(); + Artist picasso = artistDao.getById(EX.Picasso); + assertNotNull(picasso); + Painting guernica = paintingDao.getById(EX.guernica); + assertNotNull(guernica); + rdf4JTemplate.delete(EX.Picasso, List.of(PropertyPathBuilder.of(EX.creatorOf).or(EX.homeAddress).build())); + assertThrows(IncorrectResultSetSizeException.class, () -> artistDao.getById(EX.Picasso)); + assertThrows(IncorrectResultSetSizeException.class, () -> paintingDao.getById(EX.guernica)); + assertEquals(triplesBeforeDelete - 11, countTriples()); + } + + @Test + public void testDeleteWithMultiplePropertyPaths() { + int triplesBeforeDelete = countTriples(); + Artist picasso = artistDao.getById(EX.Picasso); + assertNotNull(picasso); + Painting guernica = paintingDao.getById(EX.guernica); + assertNotNull(guernica); + rdf4JTemplate.delete(EX.Picasso, + List.of(PropertyPathBuilder.of(EX.creatorOf).build(), PropertyPathBuilder.of(EX.homeAddress).build())); + assertThrows(IncorrectResultSetSizeException.class, () -> artistDao.getById(EX.Picasso)); + assertThrows(IncorrectResultSetSizeException.class, () -> paintingDao.getById(EX.guernica)); + assertEquals(triplesBeforeDelete - 11, countTriples()); + } + + @Test + public void testDeleteWithLongerPropertyPaths() { + int triplesBeforeDelete = countTriples(); + Artist picasso = artistDao.getById(EX.Picasso); + assertNotNull(picasso); + Painting guernica = paintingDao.getById(EX.guernica); + assertNotNull(guernica); + // deletes guernica and the home address, but not picasso + rdf4JTemplate.delete(EX.guernica, + List.of(PropertyPathBuilder.of(EX.creatorOf).inv().then(EX.homeAddress).build())); + picasso = artistDao.getById(EX.Picasso); + assertNotNull(picasso); + assertThrows(IncorrectResultSetSizeException.class, () -> paintingDao.getById(EX.guernica)); + assertEquals(triplesBeforeDelete - 8, countTriples()); + } + @Test public void testAssociate() { IRI me = EX.of("me"); @@ -464,4 +538,14 @@ public void testAssociate() { .size()); } + + private int countTriples() { + return this.rdf4JTemplate + .tupleQuery("SELECT (count(*) AS ?count) WHERE { ?a ?b ?c }") + .evaluateAndConvert() + .toSingletonOfWholeResult(result -> { + BindingSet bs = result.next(); + return TypeMappingUtils.toInt(QueryResultUtils.getValue(bs, "count")); + }); + } } diff --git a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/SailConcurrencyTest.java b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/SailConcurrencyTest.java index 0823aab17e3..d47793cc3ec 100644 --- a/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/SailConcurrencyTest.java +++ b/testsuites/sail/src/main/java/org/eclipse/rdf4j/testsuite/sail/SailConcurrencyTest.java @@ -356,6 +356,87 @@ public void testConcurrentConnectionsShutdown() throws InterruptedException { } +// @Disabled + @Test + public void testSerialThreads() throws InterruptedException { + if (store instanceof AbstractSail) { + ((AbstractSail) store).setConnectionTimeOut(200); + } else if (store instanceof SailWrapper) { + Sail baseSail = ((SailWrapper) store).getBaseSail(); + if (baseSail instanceof AbstractSail) { + ((AbstractSail) baseSail).setConnectionTimeOut(200); + } + } + + try (SailConnection connection = store.getConnection()) { + connection.begin(); + connection.addStatement(RDF.TYPE, RDF.TYPE, RDF.PROPERTY, RDF.TYPE); + connection.commit(); + } + + AtomicReference connection1 = new AtomicReference<>(); + + Thread thread1 = new Thread(() -> { + SailConnection connection = store.getConnection(); + connection1.setRelease(connection); + + }); + + thread1.start(); + thread1.join(); + + thread1 = new Thread(() -> { + SailConnection connection = connection1.getAcquire(); + connection.begin(IsolationLevels.NONE); + }); + + thread1.start(); + thread1.join(); + + thread1 = new Thread(() -> { + SailConnection connection = connection1.getAcquire(); + connection.addStatement(RDF.FIRST, RDF.TYPE, RDF.PROPERTY); + }); + + thread1.start(); + thread1.join(); + + thread1 = new Thread(() -> { + SailConnection connection = connection1.getAcquire(); + connection.clear(RDF.TYPE); + }); + + thread1.start(); + thread1.join(); + + thread1 = new Thread(() -> { + SailConnection connection = connection1.getAcquire(); + connection.commit(); + }); + + thread1.start(); + thread1.join(); + + thread1 = new Thread(() -> { + SailConnection connection = connection1.getAcquire(); + connection.close(); + }); + + thread1.start(); + thread1.join(); + + try (SailConnection connection = store.getConnection()) { + connection.begin(); + long size = connection.size(); + assertEquals(1, size); + connection.clear(); + connection.commit(); + } + + store.shutDown(); + + } + @Test public void testConcurrentConnectionsShutdownReadCommitted() throws InterruptedException { if (store instanceof AbstractSail) { @@ -472,6 +553,17 @@ public void testConcurrentConnectionsShutdownAndClose() throws InterruptedExcept assertThat(size).isLessThanOrEqualTo(1); } + try (SailConnection connection = store.getConnection()) { + connection.begin(); + connection.addStatement(RDF.TYPE, RDF.TYPE, RDF.PROPERTY); + connection.commit(); + } + try (SailConnection connection = store.getConnection()) { + connection.begin(); + connection.clear(); + connection.commit(); + } + store.shutDown(); } diff --git a/tools/federation/pom.xml b/tools/federation/pom.xml index 7a8daf55286..5430f89d375 100644 --- a/tools/federation/pom.xml +++ b/tools/federation/pom.xml @@ -92,7 +92,7 @@ ${project.groupId} - rdf4j-collection-factory-mapdb + rdf4j-collection-factory-mapdb3 ${project.version} diff --git a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedX.java b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedX.java index 4d3ec554b66..568ae96d56a 100644 --- a/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedX.java +++ b/tools/federation/src/main/java/org/eclipse/rdf4j/federated/FedX.java @@ -17,7 +17,7 @@ import java.util.function.Supplier; import org.eclipse.rdf4j.collection.factory.api.CollectionFactory; -import org.eclipse.rdf4j.collection.factory.mapdb.MapDbCollectionFactory; +import org.eclipse.rdf4j.collection.factory.mapdb.MapDb3CollectionFactory; import org.eclipse.rdf4j.common.transaction.IsolationLevels; import org.eclipse.rdf4j.federated.endpoint.Endpoint; import org.eclipse.rdf4j.federated.endpoint.ResolvableEndpoint; @@ -247,6 +247,6 @@ public void setRepositoryResolver(RepositoryResolver resolver) { @Override public Supplier getCollectionFactory() { - return () -> new MapDbCollectionFactory(getIterationCacheSyncThreshold()); + return () -> new MapDb3CollectionFactory(getIterationCacheSyncThreshold()); } }