From 567e6438d2b4c91b65d3a136dc0b0bff04fb3ef6 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 27 Nov 2019 00:26:46 +0100 Subject: [PATCH 01/38] Fix datbase tests and enable running using GitHub workflows - fix behavior when no entries have to be inserted during a synch - fix database tests - have dbms test test only one database - DBMS to test is passed via environment variable "DBMS", defaulting to PostgreSQL - add shared information on Canonical BibTeX entry - some code improvement - have checkstyle running on github workflows (not on Travis) --- .github/workflows/checkstyle.yml | 27 +++ .github/workflows/test-on-mysql.yml | 40 ++++ .github/workflows/test-on-postgres.yml | 40 ++++ .travis.yml | 12 -- docs/testing.md | 17 ++ .../shared/DBMSConnectionProperties.java | 8 +- .../jabref/logic/shared/DBMSProcessor.java | 9 +- .../jabref/logic/shared/DBMSSynchronizer.java | 52 +++-- .../model/database/shared/DBMSType.java | 12 +- .../jabref/model/entry/CanonicalBibEntry.java | 3 + .../model/entry/SharedBibEntryData.java | 2 +- .../shared/DBMSConnectionPropertiesTest.java | 16 ++ .../logic/shared/DBMSConnectionTest.java | 13 +- .../logic/shared/DBMSProcessorTest.java | 192 +++++++----------- .../logic/shared/DBMSSynchronizerTest.java | 177 ++++++++-------- .../shared/SynchronizationTestSimulator.java | 111 +++++----- .../jabref/logic/shared/TestConnector.java | 24 +-- .../org/jabref/logic/shared/TestManager.java | 35 +--- .../model/entry/CanonicalBibEntryTest.java | 7 +- 19 files changed, 436 insertions(+), 361 deletions(-) create mode 100644 .github/workflows/checkstyle.yml create mode 100644 .github/workflows/test-on-mysql.yml create mode 100644 .github/workflows/test-on-postgres.yml create mode 100644 docs/testing.md create mode 100644 src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml new file mode 100644 index 00000000000..eec5b0a3d7a --- /dev/null +++ b/.github/workflows/checkstyle.yml @@ -0,0 +1,27 @@ +name: checkstyle + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run checkstyle test + run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh diff --git a/.github/workflows/test-on-mysql.yml b/.github/workflows/test-on-mysql.yml new file mode 100644 index 00000000000..73afca981d0 --- /dev/null +++ b/.github/workflows/test-on-mysql.yml @@ -0,0 +1,40 @@ +name: Test on MySQL + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - uses: mirromutth/mysql-action@v1.1 + with: + host port: 3800 + container port: 3307 + character set server: 'utf8' + collation server: 'utf8_general_ci' + mysql version: '8.0' + mysql database: 'jabref' + mysql root password: 'root' + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "mysql" diff --git a/.github/workflows/test-on-postgres.yml b/.github/workflows/test-on-postgres.yml new file mode 100644 index 00000000000..f5b3db7d496 --- /dev/null +++ b/.github/workflows/test-on-postgres.yml @@ -0,0 +1,40 @@ +name: Test on PostgreSQL + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "postgresql" diff --git a/.travis.yml b/.travis.yml index dc29ce8976a..b3d2e28825a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,18 +11,12 @@ sudo: required git: depth: 1 -services: - - postgresql - - mysql - env: global: - GRADLE_OPTS=-Dorg.gradle.daemon=false matrix: - TEST_SUITE=check - - TEST_SUITE=checkstyle - TEST_SUITE=fetcherTest - - TEST_SUITE=databaseTest - TEST_SUITE=guiTest # codecov may fail by itself (see gradle build file) # The codecov test itself fails only in case there is an issue with the codecov update. @@ -34,7 +28,6 @@ matrix: fast_finish: true allow_failures: - env: TEST_SUITE=fetcherTest - - env: TEST_SUITE=databaseTest - env: TEST_SUITE=guiTest - env: DEPENDENCY_UPDATES=check @@ -46,13 +39,8 @@ before_install: install: true -before_script: - - psql -c 'create database jabref;' -U postgres - - mysql -u root -e 'create database jabref' - script: - if [ "$TEST_SUITE" != "guiTest" ] && [ "$TEST_SUITE" != "checkstyle" ] && [ "$TEST_SUITE" != "codecov" ]; then ./gradlew $TEST_SUITE $OPTIONS -x checkstyleJmh -x checkstyleMain -x checkstyleTest; fi - - if [ "$TEST_SUITE" == "checkstyle" ]; then ./gradlew checkstyleMain checkstyleTest checkstyleJmh; fi - if [ "$TEST_SUITE" == "guiTest" ]; then ./buildres/gui-tests.sh; fi - if [ "$TEST_SUITE" == "codecov" ]; then ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); fi - if [ "$DEPENDENCY_UPDATES" == "check" ]; then ./gradlew -q checkOutdatedDependencies; fi diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 00000000000..b7d7bb81bfe --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,17 @@ +# How to test + +## Database tests + +To quickly host a local PostgreSQL database, execute following statement: + +```shell command +docker run -d -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres -p 5432:5432 --name db postgres:10 postgres -c log_statement=all +``` + +Then, all DBMS Tests (annotated with `@org.jabref.testutils.category.DatabaseTest`) run properly. + +A MySQL can be started using following command: + +``´shell command +docker run -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=jabref -p 3800:3307 mysql:8.0 --port=3307 +``` diff --git a/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java b/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java index f39eb8a41a4..f7b7d9a8765 100644 --- a/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java +++ b/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java @@ -117,7 +117,13 @@ public void setUseSSL(boolean useSSL) { } public String getUrl() { - return type.getUrl(host, port, database); + String url = type.getUrl(host, port, database); + if (this.type == DBMSType.MYSQL) { + // quickfix - see https://mysqlconnector.net/troubleshooting/retrieval-public-key/ + url += "?allowPublicKeyRetrieval=true"; + url += "&useSSL=" + this.isUseSSL(); + } + return url; } @Override diff --git a/src/main/java/org/jabref/logic/shared/DBMSProcessor.java b/src/main/java/org/jabref/logic/shared/DBMSProcessor.java index b4651f96eaf..62d8a45a820 100644 --- a/src/main/java/org/jabref/logic/shared/DBMSProcessor.java +++ b/src/main/java/org/jabref/logic/shared/DBMSProcessor.java @@ -66,7 +66,7 @@ public boolean checkBaseIntegrity() throws SQLException { * * @return true if the structure is old, else false. */ - public boolean checkForPare3Dot6Integrity() throws SQLException { + public boolean databaseIsAtMostJabRef35() throws SQLException { return checkTableAvailability( "ENTRIES", "ENTRY_GROUP", @@ -439,7 +439,14 @@ public Optional getSharedEntry(int sharedID) { return Optional.empty(); } + /** + * Queries the database for shared entries. Optionally, they are filtered by the given list of sharedIds + * + * @param sharedIDs the list of Ids to filter. If list is empty, then no filter is applied + */ public List getSharedEntries(List sharedIDs) { + Objects.requireNonNull(sharedIDs); + List sharedEntries = new ArrayList<>(); StringBuilder query = new StringBuilder(); diff --git a/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java b/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java index 23753cdde40..9ddf90cea2c 100644 --- a/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java +++ b/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java @@ -8,6 +8,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import org.jabref.logic.exporter.BibDatabaseWriter; import org.jabref.logic.exporter.MetaDataSerializer; @@ -146,15 +147,17 @@ public void initializeDatabases() throws DatabaseNotSupportedException { try { if (!dbmsProcessor.checkBaseIntegrity()) { LOGGER.info("Integrity check failed. Fixing..."); - dbmsProcessor.setupSharedDatabase(); // This check should only be performed once on initial database setup. - // Calling dbmsProcessor.setupSharedDatabase() lets dbmsProcessor.checkBaseIntegrity() be true. - if (dbmsProcessor.checkForPare3Dot6Integrity()) { + if (dbmsProcessor.databaseIsAtMostJabRef35()) { throw new DatabaseNotSupportedException(); } + + // Calling dbmsProcessor.setupSharedDatabase() lets dbmsProcessor.checkBaseIntegrity() be true. + dbmsProcessor.setupSharedDatabase(); } } catch (SQLException e) { + LOGGER.error("Could not check intergrity", e); throw new IllegalStateException(e); } @@ -164,7 +167,7 @@ public void initializeDatabases() throws DatabaseNotSupportedException { } /** - * Synchronizes the local database with shared one. Possible update types are removal, update or insert of a {@link + * Synchronizes the local database with shared one. Possible update types are: removal, update, or insert of a {@link * BibEntry}. */ @Override @@ -178,13 +181,13 @@ public void synchronizeLocalDatabase() { // remove old entries locally removeNotSharedEntries(localEntries, idVersionMap.keySet()); - List entriesToDrag = new ArrayList<>(); + List entriesToInsertIntoLocalDatabase = new ArrayList<>(); // compare versions and update local entry if needed for (Map.Entry idVersionEntry : idVersionMap.entrySet()) { - boolean match = false; + boolean remoteEntryMatchingOneLocalEntryFound = false; for (BibEntry localEntry : localEntries) { - if (idVersionEntry.getKey() == localEntry.getSharedBibEntryData().getSharedID()) { - match = true; + if (idVersionEntry.getKey().equals(localEntry.getSharedBibEntryData().getSharedID())) { + remoteEntryMatchingOneLocalEntryFound = true; if (idVersionEntry.getValue() > localEntry.getSharedBibEntryData().getVersion()) { Optional sharedEntry = dbmsProcessor.getSharedEntry(idVersionEntry.getKey()); if (sharedEntry.isPresent()) { @@ -193,7 +196,7 @@ public void synchronizeLocalDatabase() { localEntry.getSharedBibEntryData() .setVersion(sharedEntry.get().getSharedBibEntryData().getVersion()); sharedEntry.get().getFieldMap().forEach( - // copy remote values to local entry + // copy remote values to local entry (field, value) -> localEntry.setField(field, value, EntriesEventSource.SHARED) ); @@ -207,13 +210,16 @@ public void synchronizeLocalDatabase() { } } } - if (!match) { - entriesToDrag.add(idVersionEntry.getKey()); + if (!remoteEntryMatchingOneLocalEntryFound) { + entriesToInsertIntoLocalDatabase.add(idVersionEntry.getKey()); } } - for (BibEntry bibEntry : dbmsProcessor.getSharedEntries(entriesToDrag)) { - bibDatabase.insertEntry(bibEntry, EntriesEventSource.SHARED); + if (!entriesToInsertIntoLocalDatabase.isEmpty()) { + // in case entries should be added into the local database, insert them + for (BibEntry bibEntry : dbmsProcessor.getSharedEntries(entriesToInsertIntoLocalDatabase)) { + bibDatabase.insertEntry(bibEntry, EntriesEventSource.SHARED); + } } } @@ -224,23 +230,15 @@ public void synchronizeLocalDatabase() { * @param sharedIDs Set of all IDs which are present on shared database */ private void removeNotSharedEntries(List localEntries, Set sharedIDs) { - List entriesToRemove = new ArrayList<>(); - for (BibEntry localEntry : localEntries) { - boolean match = false; - for (int sharedID : sharedIDs) { - if (localEntry.getSharedBibEntryData().getSharedID() == sharedID) { - match = true; - break; - } - } - if (!match) { - entriesToRemove.add(localEntry); - } - } + List entriesToRemove = + localEntries.stream() + .filter(localEntry -> !sharedIDs.contains(localEntry.getSharedBibEntryData().getSharedID())) + .collect(Collectors.toList()); if (!entriesToRemove.isEmpty()) { eventBus.post(new SharedEntriesNotPresentEvent(entriesToRemove)); + // remove all non-shared entries without triggering listeners + bibDatabase.removeEntries(entriesToRemove, EntriesEventSource.SHARED); } - bibDatabase.removeEntries(entriesToRemove, EntriesEventSource.SHARED); // Should not reach the listeners above. } /** diff --git a/src/main/java/org/jabref/model/database/shared/DBMSType.java b/src/main/java/org/jabref/model/database/shared/DBMSType.java index 90d940c2eb8..71b9e2538be 100644 --- a/src/main/java/org/jabref/model/database/shared/DBMSType.java +++ b/src/main/java/org/jabref/model/database/shared/DBMSType.java @@ -51,11 +51,15 @@ public int getDefaultPort() { } public static Optional fromString(String typeName) { - try { - return Optional.of(Enum.valueOf(DBMSType.class, typeName.toUpperCase(Locale.ENGLISH))); - } catch (IllegalArgumentException exception) { + if (typeName == null) { return Optional.empty(); + } else { + try { + return Optional.of(Enum.valueOf(DBMSType.class, typeName.toUpperCase(Locale.ENGLISH))); + } catch (IllegalArgumentException exception) { + // IllegalArgumentException is thrown if "typename" is not covered by the Enum + return Optional.empty(); + } } } - } diff --git a/src/main/java/org/jabref/model/entry/CanonicalBibEntry.java b/src/main/java/org/jabref/model/entry/CanonicalBibEntry.java index a854db41907..1136c253001 100644 --- a/src/main/java/org/jabref/model/entry/CanonicalBibEntry.java +++ b/src/main/java/org/jabref/model/entry/CanonicalBibEntry.java @@ -56,6 +56,9 @@ public static String getCanonicalRepresentation(BibEntry entry) { String line = String.format(" %s = {%s}", fieldName, String.valueOf(mapFieldToValue.get(fieldName)).replaceAll("\\r\\n", "\n")); sj.add(line); } + + sj.add(String.format(" _jabref_shared = {sharedId: %d, version: %d}", entry.getSharedBibEntryData().getSharedID(), entry.getSharedBibEntryData().getVersion())); + sb.append(sj); // append the closing entry bracket diff --git a/src/main/java/org/jabref/model/entry/SharedBibEntryData.java b/src/main/java/org/jabref/model/entry/SharedBibEntryData.java index 25672e829d8..79546fc202c 100644 --- a/src/main/java/org/jabref/model/entry/SharedBibEntryData.java +++ b/src/main/java/org/jabref/model/entry/SharedBibEntryData.java @@ -1,7 +1,7 @@ package org.jabref.model.entry; /** - * Stores all informations needed to manage entries on a shared (SQL) database. + * Stores all information needed to manage entries on a shared (SQL) database. */ public class SharedBibEntryData { diff --git a/src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java b/src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java new file mode 100644 index 00000000000..de9347e51af --- /dev/null +++ b/src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java @@ -0,0 +1,16 @@ +package org.jabref.logic.shared; + +import org.jabref.model.database.shared.DBMSType; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DBMSConnectionPropertiesTest { + + @Test + void urlForMySqlIncludesSslConfig() { + DBMSConnectionProperties connectionProperties = new DBMSConnectionProperties(DBMSType.MYSQL, "localhost", 3108, "jabref", "user", "password", false, ""); + assertEquals("jdbc:mariadb://localhost:3108/jabref?allowPublicKeyRetrieval=true&useSSL=false", connectionProperties.getUrl()); + } +} diff --git a/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java b/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java index e0cb37bae26..9a3a34d3549 100644 --- a/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java +++ b/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java @@ -2,14 +2,12 @@ import java.sql.SQLException; -import org.jabref.logic.shared.exception.InvalidDBMSConnectionPropertiesException; import org.jabref.model.database.shared.DBMSType; import org.jabref.testutils.category.DatabaseTest; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @DatabaseTest @@ -17,15 +15,8 @@ public class DBMSConnectionTest { @ParameterizedTest @EnumSource(DBMSType.class) - public void testGetConnection(DBMSType dbmsType) throws SQLException, InvalidDBMSConnectionPropertiesException { - DBMSConnectionProperties properties = TestConnector.getTestConnectionProperties(dbmsType); - assertNotNull(new DBMSConnection(properties).getConnection()); - } - - @ParameterizedTest - @EnumSource(DBMSType.class) - public void testGetConnectionFail(DBMSType dbmsType) throws SQLException, InvalidDBMSConnectionPropertiesException { + public void getConnectionFailsWhenconnectingToInvalidHost(DBMSType dbmsType) { assertThrows(SQLException.class, - () -> new DBMSConnection(new DBMSConnectionProperties(dbmsType, "XXXX", 0, "XXXX", "XXXX", "XXXX", false, "XXXX")).getConnection()); + () -> new DBMSConnection(new DBMSConnectionProperties(dbmsType, "XXXX", 33778, "XXXX", "XXXX", "XXXX", false, "XXXX")).getConnection()); } } diff --git a/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java b/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java index 401ba599020..c35d0e7c0eb 100644 --- a/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java +++ b/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java @@ -19,8 +19,11 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.DatabaseTest; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -29,33 +32,44 @@ import static org.junit.jupiter.api.Assertions.fail; @DatabaseTest +@Execution(ExecutionMode.SAME_THREAD) class DBMSProcessorTest { - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testCheckBaseIntegrity(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + private DBMSConnection dbmsConnection; + private DBMSProcessor dbmsProcessor; + private DBMSType dbmsType; + + @BeforeEach + public void setup() throws Exception { + this.dbmsType = TestManager.getDBMSTypeTestParameter(); + this.dbmsConnection = TestConnector.getTestDBMSConnection(dbmsType); + this.dbmsProcessor = DBMSProcessor.getProcessorInstance(TestConnector.getTestDBMSConnection(dbmsType)); + TestManager.clearTables(this.dbmsConnection); + } + + @AfterEach + public void clear() throws SQLException { + this.dbmsConnection.getConnection().close(); + } + + @Test + void testCheckBaseIntegrity() throws SQLException { dbmsProcessor.setupSharedDatabase(); assertTrue(dbmsProcessor.checkBaseIntegrity()); - clear(dbmsConnection); + TestManager.clearTables(this.dbmsConnection); assertFalse(dbmsProcessor.checkBaseIntegrity()); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testSetUpSharedDatabase(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testSetUpSharedDatabase() throws SQLException { dbmsProcessor.setupSharedDatabase(); - clear(dbmsConnection); + clear(); dbmsProcessor.setupSharedDatabase(); assertTrue(dbmsProcessor.checkBaseIntegrity()); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testInsertEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testInsertEntry() throws SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry expectedEntry = getBibEntryExample(); @@ -84,12 +98,10 @@ void testInsertEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProce Map expectedFieldMap = expectedEntry.getFieldMap().entrySet().stream().collect(Collectors.toMap((entry) -> entry.getKey().getName(), Map.Entry::getValue)); assertEquals(expectedFieldMap, actualFieldMap); - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testUpdateEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { + @Test + void testUpdateEntry() throws Exception { dbmsProcessor.setupSharedDatabase(); BibEntry expectedEntry = getBibEntryExample(); dbmsProcessor.insertEntry(expectedEntry); @@ -101,14 +113,11 @@ void testUpdateEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProce dbmsProcessor.updateEntry(expectedEntry); Optional actualEntry = dbmsProcessor.getSharedEntry(expectedEntry.getSharedBibEntryData().getSharedID()); - assertEquals(expectedEntry, actualEntry.get()); - - clear(dbmsConnection); + assertEquals(Optional.of(expectedEntry), actualEntry); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testGetEntriesByIdList(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { + @Test + void testGetEntriesByIdList() throws Exception { dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); firstEntry.setField(InternalField.INTERNAL_ID_FIELD, "00001"); @@ -120,16 +129,11 @@ void testGetEntriesByIdList(DBMSType dbmsType, DBMSConnection dbmsConnection, DB List sharedEntriesByIdList = dbmsProcessor.getSharedEntries(Arrays.asList(1, 2)); - clear(dbmsConnection); - - assertEquals(firstEntry, sharedEntriesByIdList.get(0)); - assertEquals(secondEntry, sharedEntriesByIdList.get(1)); - + assertEquals(List.of(firstEntry, secondEntry), sharedEntriesByIdList); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testUpdateNewerEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws OfflineLockException, SQLException { + @Test + void testUpdateNewerEntry() throws OfflineLockException, SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry bibEntry = getBibEntryExample(); @@ -140,13 +144,10 @@ void testUpdateNewerEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMS bibEntry.setField(StandardField.YEAR, "1993"); assertThrows(OfflineLockException.class, () -> dbmsProcessor.updateEntry(bibEntry)); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testUpdateEqualEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws OfflineLockException, SQLException { + @Test + void testUpdateEqualEntry() throws OfflineLockException, SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry expectedBibEntry = getBibEntryExample(); @@ -156,16 +157,13 @@ void testUpdateEqualEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMS dbmsProcessor.updateEntry(expectedBibEntry); Optional actualBibEntryOptional = dbmsProcessor - .getSharedEntry(expectedBibEntry.getSharedBibEntryData().getSharedID()); - - assertEquals(expectedBibEntry, actualBibEntryOptional.get()); + .getSharedEntry(expectedBibEntry.getSharedBibEntryData().getSharedID()); - clear(dbmsConnection); + assertEquals(Optional.of(expectedBibEntry), actualBibEntryOptional); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testRemoveAllEntries(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testRemoveAllEntries() throws SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); BibEntry secondEntry = getBibEntryExample(); @@ -177,12 +175,10 @@ void testRemoveAllEntries(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMS try (ResultSet resultSet = selectFrom("ENTRY", dbmsConnection, dbmsProcessor)) { assertFalse(resultSet.next()); } - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testRemoveSomeEntries(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testRemoveSomeEntries() throws SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); BibEntry secondEntry = getBibEntryExample(); @@ -201,13 +197,10 @@ void testRemoveSomeEntries(DBMSType dbmsType, DBMSConnection dbmsConnection, DBM assertEquals(2, entryResultSet.getInt("SHARED_ID")); assertFalse(entryResultSet.next()); } - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testRemoveSingleEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testRemoveSingleEntry() throws SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry entryToRemove = getBibEntryExample(); dbmsProcessor.insertEntry(entryToRemove); @@ -216,49 +209,38 @@ void testRemoveSingleEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBM try (ResultSet entryResultSet = selectFrom("ENTRY", dbmsConnection, dbmsProcessor)) { assertFalse(entryResultSet.next()); } - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testRemoveEntriesOnNullThrows(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testRemoveEntriesOnNullThrows() throws SQLException { dbmsProcessor.setupSharedDatabase(); assertThrows(NullPointerException.class, () -> dbmsProcessor.removeEntries(null)); - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testRemoveEmptyEntryList(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testRemoveEmptyEntryList() throws SQLException { dbmsProcessor.setupSharedDatabase(); dbmsProcessor.removeEntries(Collections.emptyList()); try (ResultSet entryResultSet = selectFrom("ENTRY", dbmsConnection, dbmsProcessor)) { assertFalse(entryResultSet.next()); } - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testGetSharedEntries(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testGetSharedEntries() throws SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry bibEntry = getBibEntryExampleWithEmptyFields(); dbmsProcessor.insertEntry(bibEntry); - List expectedEntries = Arrays.asList(bibEntry); List actualEntries = dbmsProcessor.getSharedEntries(); - assertEquals(expectedEntries, actualEntries); - clear(dbmsConnection); + assertEquals(List.of(bibEntry), actualEntries); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testGetSharedEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testGetSharedEntry() throws SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry expectedBibEntry = getBibEntryExampleWithEmptyFields(); @@ -266,23 +248,18 @@ void testGetSharedEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSPr Optional actualBibEntryOptional = dbmsProcessor.getSharedEntry(expectedBibEntry.getSharedBibEntryData().getSharedID()); - assertEquals(expectedBibEntry, actualBibEntryOptional.get()); - clear(dbmsConnection); + assertEquals(Optional.of(expectedBibEntry), actualBibEntryOptional); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testGetNotExistingSharedEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testGetNotExistingSharedEntry() throws SQLException { dbmsProcessor.setupSharedDatabase(); Optional actualBibEntryOptional = dbmsProcessor.getSharedEntry(1); assertFalse(actualBibEntryOptional.isPresent()); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testGetSharedIDVersionMapping(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws OfflineLockException, SQLException { + @Test + void testGetSharedIDVersionMapping() throws OfflineLockException, SQLException { dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); BibEntry secondEntry = getBibEntryExample(); @@ -298,13 +275,10 @@ void testGetSharedIDVersionMapping(DBMSType dbmsType, DBMSConnection dbmsConnect Map actualIDVersionMap = dbmsProcessor.getSharedIDVersionMapping(); assertEquals(expectedIDVersionMap, actualIDVersionMap); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - void testGetSharedMetaData(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testGetSharedMetaData() throws SQLException { dbmsProcessor.setupSharedDatabase(); insertMetaData("databaseType", "bibtex;", dbmsConnection, dbmsProcessor); insertMetaData("protectedFlag", "true;", dbmsConnection, dbmsProcessor); @@ -315,12 +289,10 @@ void testGetSharedMetaData(DBMSType dbmsType, DBMSConnection dbmsConnection, DBM Map actualMetaData = dbmsProcessor.getSharedMetaData(); assertEquals(expectedMetaData, actualMetaData); - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("getTestingDatabaseSystems") - void testSetSharedMetaData(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws SQLException { + @Test + void testSetSharedMetaData() throws SQLException { dbmsProcessor.setupSharedDatabase(); Map expectedMetaData = getMetaDataExample(); dbmsProcessor.setSharedMetaData(expectedMetaData); @@ -328,7 +300,6 @@ void testSetSharedMetaData(DBMSType dbmsType, DBMSConnection dbmsConnection, DBM Map actualMetaData = dbmsProcessor.getSharedMetaData(); assertEquals(expectedMetaData, actualMetaData); - clear(dbmsConnection); } private Map getMetaDataExample() { @@ -343,22 +314,21 @@ private Map getMetaDataExample() { } private BibEntry getBibEntryExampleWithEmptyFields() { - BibEntry bibEntry = new BibEntry(); - bibEntry.setField(StandardField.AUTHOR, "Author"); - bibEntry.setField(StandardField.TITLE, ""); - bibEntry.setField(StandardField.YEAR, ""); + BibEntry bibEntry = new BibEntry() + .withField(StandardField.AUTHOR, "Author") + .withField(StandardField.TITLE, "") + .withField(StandardField.YEAR, ""); bibEntry.getSharedBibEntryData().setSharedID(1); return bibEntry; } private BibEntry getBibEntryExample() { - BibEntry bibEntry = new BibEntry(StandardEntryType.InProceedings); - bibEntry.setField(StandardField.AUTHOR, "Wirthlin, Michael J and Hutchings, Brad L and Gilson, Kent L"); - bibEntry.setField(StandardField.TITLE, "The nano processor: a low resource reconfigurable processor"); - bibEntry.setField(StandardField.BOOKTITLE, "FPGAs for Custom Computing Machines, 1994. Proceedings. IEEE Workshop on"); - bibEntry.setField(StandardField.YEAR, "1994"); - bibEntry.setCiteKey("nanoproc1994"); - return bibEntry; + return new BibEntry(StandardEntryType.InProceedings) + .withField(StandardField.AUTHOR, "Wirthlin, Michael J and Hutchings, Brad L and Gilson, Kent L") + .withField(StandardField.TITLE, "The nano processor: a low resource reconfigurable processor") + .withField(StandardField.BOOKTITLE, "FPGAs for Custom Computing Machines, 1994. Proceedings. IEEE Workshop on") + .withField(StandardField.YEAR, "1994") + .withCiteKey("nanoproc1994"); } private ResultSet selectFrom(String table, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) { @@ -375,8 +345,8 @@ private ResultSet selectFrom(String table, DBMSConnection dbmsConnection, DBMSPr private void insertMetaData(String key, String value, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) { try { dbmsConnection.getConnection().createStatement().executeUpdate("INSERT INTO " + escape("METADATA", dbmsProcessor) + "(" - + escape("KEY", dbmsProcessor) + ", " + escape("VALUE", dbmsProcessor) + ") VALUES(" - + escapeValue(key) + ", " + escapeValue(value) + ")"); + + escape("KEY", dbmsProcessor) + ", " + escape("VALUE", dbmsProcessor) + ") VALUES(" + + escapeValue(key) + ", " + escapeValue(value) + ")"); } catch (SQLException e) { fail(e.getMessage()); } @@ -389,8 +359,4 @@ private String escape(String expression, DBMSProcessor dbmsProcessor) { private String escapeValue(String value) { return "'" + value + "'"; } - - void clear(DBMSConnection dbmsConnection) throws SQLException { - TestManager.clearTables(dbmsConnection); - } } diff --git a/src/test/java/org/jabref/logic/shared/DBMSSynchronizerTest.java b/src/test/java/org/jabref/logic/shared/DBMSSynchronizerTest.java index 4d41a5a15d9..7c0cbba838f 100644 --- a/src/test/java/org/jabref/logic/shared/DBMSSynchronizerTest.java +++ b/src/test/java/org/jabref/logic/shared/DBMSSynchronizerTest.java @@ -24,38 +24,59 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.testutils.category.DatabaseTest; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @DatabaseTest +@Execution(ExecutionMode.SAME_THREAD) public class DBMSSynchronizerTest { private DBMSSynchronizer dbmsSynchronizer; private BibDatabase bibDatabase; private final GlobalBibtexKeyPattern pattern = GlobalBibtexKeyPattern.fromPattern("[auth][year]"); + private DBMSConnection dbmsConnection; + private DBMSProcessor dbmsProcessor; + private DBMSType dbmsType; + + private BibEntry createExampleBibEntry(int index) { + BibEntry bibEntry = new BibEntry(StandardEntryType.Book) + .withField(StandardField.AUTHOR, "Wirthlin, Michael J" + index) + .withField(StandardField.TITLE, "The nano processor" + index); + bibEntry.getSharedBibEntryData().setSharedID(index); + return bibEntry; + } - public void setUp(DBMSConnection dbmsConnection) throws Exception { - clear(dbmsConnection); + @BeforeEach + public void setup() throws Exception { + this.dbmsType = TestManager.getDBMSTypeTestParameter(); + this.dbmsConnection = TestConnector.getTestDBMSConnection(dbmsType); + this.dbmsProcessor = DBMSProcessor.getProcessorInstance(this.dbmsConnection); + TestManager.clearTables(this.dbmsConnection); bibDatabase = new BibDatabase(); BibDatabaseContext context = new BibDatabaseContext(bibDatabase); - dbmsSynchronizer = new DBMSSynchronizer(context, ',', pattern, new DummyFileUpdateMonitor()); + dbmsSynchronizer = new DBMSSynchronizer(context, ',', pattern, new DummyFileUpdateMonitor()); bibDatabase.registerListener(dbmsSynchronizer); dbmsSynchronizer.openSharedDatabase(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testEntryAddedEventListener(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); + @AfterEach + public void clear() throws SQLException { + this.dbmsConnection.getConnection().close(); + } - BibEntry expectedEntry = getBibEntryExample(1); - BibEntry furtherEntry = getBibEntryExample(1); + @Test + public void testEntryAddedEventListener() throws Exception { + BibEntry expectedEntry = createExampleBibEntry(1); + BibEntry furtherEntry = createExampleBibEntry(1); bibDatabase.insertEntry(expectedEntry); // should not add into shared database. @@ -63,36 +84,43 @@ public void testEntryAddedEventListener(DBMSType dbmsType, DBMSConnection dbmsCo List actualEntries = dbmsProcessor.getSharedEntries(); - assertEquals(1, actualEntries.size()); - assertEquals(expectedEntry, actualEntries.get(0)); - - clear(dbmsConnection); + assertEquals(List.of(expectedEntry), actualEntries); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testFieldChangedEventListener(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); - - BibEntry expectedEntry = getBibEntryExample(1); + @Test + public void twoLocalFieldChangesAreSynchronizedCorrectly() throws Exception { + BibEntry expectedEntry = createExampleBibEntry(1); expectedEntry.registerListener(dbmsSynchronizer); bibDatabase.insertEntry(expectedEntry); expectedEntry.setField(StandardField.AUTHOR, "Brad L and Gilson"); - expectedEntry.setField(StandardField.TITLE, "The micro multiplexer", EntriesEventSource.SHARED); + expectedEntry.setField(StandardField.TITLE, "The micro multiplexer"); List actualEntries = dbmsProcessor.getSharedEntries(); assertEquals(Collections.singletonList(expectedEntry), actualEntries); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testEntriesRemovedEventListener(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); + @Test + public void oneLocalAndOneSharedFieldChangeIsSynchronizedCorrectly() throws Exception { + BibEntry exampleBibEntry = createExampleBibEntry(1); + exampleBibEntry.registerListener(dbmsSynchronizer); - BibEntry bibEntry = getBibEntryExample(1); + bibDatabase.insertEntry(exampleBibEntry); + exampleBibEntry.setField(StandardField.AUTHOR, "Brad L and Gilson"); + // shared updates are not synchronized back to the remote database + exampleBibEntry.setField(StandardField.TITLE, "The micro multiplexer", EntriesEventSource.SHARED); + + List actualEntries = dbmsProcessor.getSharedEntries(); + + BibEntry expectedBibEntry = createExampleBibEntry(1) + .withField(StandardField.AUTHOR, "Brad L and Gilson"); + + assertEquals(Collections.singletonList(expectedBibEntry), actualEntries); + } + + @Test + public void testEntriesRemovedEventListener() throws Exception { + BibEntry bibEntry = createExampleBibEntry(1); bibDatabase.insertEntry(bibEntry); List actualEntries = dbmsProcessor.getSharedEntries(); @@ -110,15 +138,10 @@ public void testEntriesRemovedEventListener(DBMSType dbmsType, DBMSConnection db actualEntries = dbmsProcessor.getSharedEntries(); assertEquals(1, actualEntries.size()); assertEquals(bibEntry, actualEntries.get(0)); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testMetaDataChangedEventListener(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); - + @Test + public void testMetaDataChangedEventListener() throws Exception { MetaData testMetaData = new MetaData(); testMetaData.registerListener(dbmsSynchronizer); dbmsSynchronizer.setMetaData(testMetaData); @@ -128,29 +151,19 @@ public void testMetaDataChangedEventListener(DBMSType dbmsType, DBMSConnection d Map actualMap = dbmsProcessor.getSharedMetaData(); assertEquals(expectedMap, actualMap); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testInitializeDatabases(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); - - clear(dbmsConnection); + @Test + public void testInitializeDatabases() throws Exception { dbmsSynchronizer.initializeDatabases(); assertTrue(dbmsProcessor.checkBaseIntegrity()); dbmsSynchronizer.initializeDatabases(); assertTrue(dbmsProcessor.checkBaseIntegrity()); - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testSynchronizeLocalDatabaseWithEntryRemoval(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); - - List expectedBibEntries = Arrays.asList(getBibEntryExample(1), getBibEntryExample(2)); + @Test + public void testSynchronizeLocalDatabaseWithEntryRemoval() throws Exception { + List expectedBibEntries = Arrays.asList(createExampleBibEntry(1), createExampleBibEntry(2)); dbmsProcessor.insertEntry(expectedBibEntries.get(0)); dbmsProcessor.insertEntry(expectedBibEntries.get(1)); @@ -168,38 +181,47 @@ public void testSynchronizeLocalDatabaseWithEntryRemoval(DBMSType dbmsType, DBMS dbmsSynchronizer.synchronizeLocalDatabase(); assertEquals(expectedBibEntries, bibDatabase.getEntries()); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testSynchronizeLocalDatabaseWithEntryUpdate(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); - - BibEntry bibEntry = getBibEntryExample(1); + @Test + public void testSynchronizeLocalDatabaseWithEntryUpdate() throws Exception { + BibEntry bibEntry = createExampleBibEntry(1); bibDatabase.insertEntry(bibEntry); - assertEquals(1, bibDatabase.getEntries().size()); + assertEquals(List.of(bibEntry), bibDatabase.getEntries()); - BibEntry modifiedBibEntry = getBibEntryExample(1); - modifiedBibEntry.setField(new UnknownField("custom"), "custom value"); + BibEntry modifiedBibEntry = createExampleBibEntry(1) + .withField(new UnknownField("custom"), "custom value"); modifiedBibEntry.clearField(StandardField.TITLE); modifiedBibEntry.setType(StandardEntryType.Article); dbmsProcessor.updateEntry(modifiedBibEntry); - //testing point + assertEquals(1, modifiedBibEntry.getSharedBibEntryData().getSharedID()); dbmsSynchronizer.synchronizeLocalDatabase(); - assertEquals(bibDatabase.getEntries(), dbmsProcessor.getSharedEntries()); - clear(dbmsConnection); + assertEquals(List.of(modifiedBibEntry), bibDatabase.getEntries()); + assertEquals(List.of(modifiedBibEntry), dbmsProcessor.getSharedEntries()); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void testApplyMetaData(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsConnection); + @Test + public void updateEntryDoesNotModifyLocalDatabase() throws Exception { + BibEntry bibEntry = createExampleBibEntry(1); + bibDatabase.insertEntry(bibEntry); + assertEquals(List.of(bibEntry), bibDatabase.getEntries()); + + BibEntry modifiedBibEntry = createExampleBibEntry(1) + .withField(new UnknownField("custom"), "custom value"); + modifiedBibEntry.clearField(StandardField.TITLE); + modifiedBibEntry.setType(StandardEntryType.Article); + + dbmsProcessor.updateEntry(modifiedBibEntry); - BibEntry bibEntry = getBibEntryExample(1); + assertEquals(List.of(bibEntry), bibDatabase.getEntries()); + assertEquals(List.of(modifiedBibEntry), dbmsProcessor.getSharedEntries()); + } + + @Test + public void testApplyMetaData() throws Exception { + BibEntry bibEntry = createExampleBibEntry(1); bibDatabase.insertEntry(bibEntry); MetaData testMetaData = new MetaData(); @@ -209,20 +231,5 @@ public void testApplyMetaData(DBMSType dbmsType, DBMSConnection dbmsConnection, dbmsSynchronizer.applyMetaData(); assertEquals("wirthlin, michael j1", bibEntry.getField(StandardField.AUTHOR).get()); - - clear(dbmsConnection); - } - - private BibEntry getBibEntryExample(int index) { - BibEntry bibEntry = new BibEntry(); - bibEntry.setType(StandardEntryType.Book); - bibEntry.setField(StandardField.AUTHOR, "Wirthlin, Michael J" + index); - bibEntry.setField(StandardField.TITLE, "The nano processor" + index); - bibEntry.getSharedBibEntryData().setSharedID(index); - return bibEntry; - } - - public void clear(DBMSConnection dbmsConnection) throws SQLException { - TestManager.clearTables(dbmsConnection); } } diff --git a/src/test/java/org/jabref/logic/shared/SynchronizationTestSimulator.java b/src/test/java/org/jabref/logic/shared/SynchronizationTestSimulator.java index fb8bf59b0d5..6bda1fcb941 100644 --- a/src/test/java/org/jabref/logic/shared/SynchronizationTestSimulator.java +++ b/src/test/java/org/jabref/logic/shared/SynchronizationTestSimulator.java @@ -1,12 +1,12 @@ package org.jabref.logic.shared; import java.sql.SQLException; +import java.util.List; import org.jabref.model.Defaults; import org.jabref.model.bibtexkeypattern.GlobalBibtexKeyPattern; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; -import org.jabref.model.database.shared.DBMSType; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; @@ -14,8 +14,11 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.testutils.category.DatabaseTest; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -24,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @DatabaseTest +@Execution(ExecutionMode.SAME_THREAD) public class SynchronizationTestSimulator { private BibDatabaseContext clientContextA; @@ -31,24 +35,42 @@ public class SynchronizationTestSimulator { private SynchronizationTestEventListener eventListenerB; // used to monitor occurring events private final GlobalBibtexKeyPattern pattern = GlobalBibtexKeyPattern.fromPattern("[auth][year]"); - public void setUp(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { + private BibEntry getBibEntryExample(int index) { + return new BibEntry(StandardEntryType.InProceedings) + .withField(StandardField.AUTHOR, "Wirthlin, Michael J and Hutchings, Brad L and Gilson, Kent L " + index) + .withField(StandardField.TITLE, "The nano processor: a low resource reconfigurable processor " + index) + .withField(StandardField.BOOKTITLE, "FPGAs for Custom Computing Machines, 1994. Proceedings. IEEE Workshop on " + index) + .withField(StandardField.YEAR, "199" + index) + .withCiteKey("nanoproc199" + index); + } + + @BeforeEach + public void setup() throws Exception { + DBMSConnection dbmsConnection = TestConnector.getTestDBMSConnection(TestManager.getDBMSTypeTestParameter()); + TestManager.clearTables(dbmsConnection); + clientContextA = new BibDatabaseContext(new Defaults(BibDatabaseMode.BIBTEX)); DBMSSynchronizer synchronizerA = new DBMSSynchronizer(clientContextA, ',', pattern, new DummyFileUpdateMonitor()); clientContextA.convertToSharedDatabase(synchronizerA); clientContextA.getDBMSSynchronizer().openSharedDatabase(dbmsConnection); clientContextB = new BibDatabaseContext(new Defaults(BibDatabaseMode.BIBTEX)); - DBMSSynchronizer synchronizerB = new DBMSSynchronizer(clientContextA, ',', pattern, new DummyFileUpdateMonitor()); + DBMSSynchronizer synchronizerB = new DBMSSynchronizer(clientContextB, ',', pattern, new DummyFileUpdateMonitor()); clientContextB.convertToSharedDatabase(synchronizerB); - clientContextB.getDBMSSynchronizer().openSharedDatabase(dbmsConnection); + // use a second connection, because this is another client (typically on another machine) + clientContextB.getDBMSSynchronizer().openSharedDatabase(TestConnector.getTestDBMSConnection(TestManager.getDBMSTypeTestParameter())); eventListenerB = new SynchronizationTestEventListener(); clientContextB.getDBMSSynchronizer().registerListener(eventListenerB); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void simulateEntryInsertionAndManualPull(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsType, dbmsConnection, dbmsProcessor); + @AfterEach + public void clear() throws SQLException { + clientContextA.getDBMSSynchronizer().closeSharedDatabase(); + clientContextB.getDBMSSynchronizer().closeSharedDatabase(); + } + + @Test + public void simulateEntryInsertionAndManualPull() throws Exception { //client A inserts an entry clientContextA.getDatabase().insertEntry(getBibEntryExample(1)); //client A inserts another entry @@ -57,35 +79,25 @@ public void simulateEntryInsertionAndManualPull(DBMSType dbmsType, DBMSConnectio clientContextB.getDBMSSynchronizer().pullChanges(); assertEquals(clientContextA.getDatabase().getEntries(), clientContextB.getDatabase().getEntries()); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void simulateEntryUpdateAndManualPull(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsType, dbmsConnection, dbmsProcessor); - + @Test + public void simulateEntryUpdateAndManualPull() throws Exception { BibEntry bibEntry = getBibEntryExample(1); - //client A inserts an entry + // client A inserts an entry clientContextA.getDatabase().insertEntry(bibEntry); - //client A changes the entry + // client A changes the entry bibEntry.setField(new UnknownField("custom"), "custom value"); - //client B pulls the changes + // client B pulls the changes bibEntry.clearField(StandardField.AUTHOR); clientContextB.getDBMSSynchronizer().pullChanges(); assertEquals(clientContextA.getDatabase().getEntries(), clientContextB.getDatabase().getEntries()); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void simulateEntryDelitionAndManualPull(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsType, dbmsConnection, dbmsProcessor); - + @Test + public void simulateEntryDelitionAndManualPull() throws Exception { BibEntry bibEntry = getBibEntryExample(1); //client A inserts an entry clientContextA.getDatabase().insertEntry(bibEntry); @@ -103,15 +115,10 @@ public void simulateEntryDelitionAndManualPull(DBMSType dbmsType, DBMSConnection assertTrue(clientContextA.getDatabase().getEntries().isEmpty()); assertTrue(clientContextB.getDatabase().getEntries().isEmpty()); - - clear(dbmsConnection); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void simulateUpdateOnNoLongerExistingEntry(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsType, dbmsConnection, dbmsProcessor); - + @Test + public void simulateUpdateOnNoLongerExistingEntry() throws Exception { BibEntry bibEntryOfClientA = getBibEntryExample(1); //client A inserts an entry clientContextA.getDatabase().insertEntry(bibEntryOfClientA); @@ -133,53 +140,31 @@ public void simulateUpdateOnNoLongerExistingEntry(DBMSType dbmsType, DBMSConnect // here a new SharedEntryNotPresentEvent has been thrown. In this case the user B would get an pop-up window. assertNotNull(eventListenerB.getSharedEntriesNotPresentEvent()); - assertEquals(bibEntryOfClientB, eventListenerB.getSharedEntriesNotPresentEvent().getBibEntries()); - - clear(dbmsConnection); + assertEquals(List.of(bibEntryOfClientB), eventListenerB.getSharedEntriesNotPresentEvent().getBibEntries()); } - @ParameterizedTest - @MethodSource("org.jabref.logic.shared.TestManager#getTestingDatabaseSystems") - public void simulateEntryChangeConflicts(DBMSType dbmsType, DBMSConnection dbmsConnection, DBMSProcessor dbmsProcessor) throws Exception { - setUp(dbmsType, dbmsConnection, dbmsProcessor); - + @Test + public void simulateEntryChangeConflicts() { BibEntry bibEntryOfClientA = getBibEntryExample(1); - //client A inserts an entry + // client A inserts an entry clientContextA.getDatabase().insertEntry(bibEntryOfClientA); - //client B pulls the entry + // client B pulls the entry clientContextB.getDBMSSynchronizer().pullChanges(); - //A now increases the version number + // A now increases the version number bibEntryOfClientA.setField(StandardField.YEAR, "2001"); // B does nothing here, so there is no event occurrence - // B now tries to update the entry assertFalse(clientContextB.getDatabase().getEntries().isEmpty()); - assertNull(eventListenerB.getUpdateRefusedEvent()); BibEntry bibEntryOfClientB = clientContextB.getDatabase().getEntries().get(0); - //B also tries to change something + // B also tries to change something bibEntryOfClientB.setField(StandardField.YEAR, "2016"); // B now cannot update the shared entry, due to optimistic offline lock. // In this case an BibEntry merge dialog pops up. assertNotNull(eventListenerB.getUpdateRefusedEvent()); - - clear(dbmsConnection); - } - - private BibEntry getBibEntryExample(int index) { - BibEntry bibEntry = new BibEntry(StandardEntryType.InProceedings); - bibEntry.setField(StandardField.AUTHOR, "Wirthlin, Michael J and Hutchings, Brad L and Gilson, Kent L " + index); - bibEntry.setField(StandardField.TITLE, "The nano processor: a low resource reconfigurable processor " + index); - bibEntry.setField(StandardField.BOOKTITLE, "FPGAs for Custom Computing Machines, 1994. Proceedings. IEEE Workshop on " + index); - bibEntry.setField(StandardField.YEAR, "199" + index); - bibEntry.setCiteKey("nanoproc199" + index); - return bibEntry; } - public void clear(DBMSConnection dbmsConnection) throws SQLException { - TestManager.clearTables(dbmsConnection); - } } diff --git a/src/test/java/org/jabref/logic/shared/TestConnector.java b/src/test/java/org/jabref/logic/shared/TestConnector.java index 78f078ce2f9..ff72457dd4d 100644 --- a/src/test/java/org/jabref/logic/shared/TestConnector.java +++ b/src/test/java/org/jabref/logic/shared/TestConnector.java @@ -6,6 +6,9 @@ import org.jabref.model.database.shared.DBMSType; import org.jabref.testutils.category.DatabaseTest; +/** + * Stores the credentials for the test systems + */ @DatabaseTest public class TestConnector { @@ -15,18 +18,15 @@ public static DBMSConnection getTestDBMSConnection(DBMSType dbmsType) throws SQL } public static DBMSConnectionProperties getTestConnectionProperties(DBMSType dbmsType) { - if (dbmsType == DBMSType.MYSQL) { - return new DBMSConnectionProperties(dbmsType, "localhost", dbmsType.getDefaultPort(), "jabref", "root", "", false, ""); + switch (dbmsType) { + case MYSQL: + return new DBMSConnectionProperties(dbmsType, "127.0.0.1", 3800, "jabref", "root", "root", false, ""); + case POSTGRESQL: + return new DBMSConnectionProperties(dbmsType, "localhost", dbmsType.getDefaultPort(), "postgres", "postgres", "postgres", false, ""); + case ORACLE: + return new DBMSConnectionProperties(dbmsType, "localhost", dbmsType.getDefaultPort(), "xe", "travis", "travis", false, ""); + default: + return new DBMSConnectionProperties(); } - - if (dbmsType == DBMSType.POSTGRESQL) { - return new DBMSConnectionProperties(dbmsType, "localhost", dbmsType.getDefaultPort(), "jabref", "postgres", "", false, ""); - } - - if (dbmsType == DBMSType.ORACLE) { - return new DBMSConnectionProperties(dbmsType, "localhost", dbmsType.getDefaultPort(), "xe", "travis", "travis", false, ""); - } - - return new DBMSConnectionProperties(); } } diff --git a/src/test/java/org/jabref/logic/shared/TestManager.java b/src/test/java/org/jabref/logic/shared/TestManager.java index 2b8f1f4d2f5..f49d226157c 100644 --- a/src/test/java/org/jabref/logic/shared/TestManager.java +++ b/src/test/java/org/jabref/logic/shared/TestManager.java @@ -1,13 +1,8 @@ package org.jabref.logic.shared; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Stream; +import java.util.Objects; -import org.jabref.logic.shared.exception.InvalidDBMSConnectionPropertiesException; import org.jabref.model.database.shared.DBMSType; /** @@ -16,21 +11,15 @@ */ public class TestManager { - public static Collection getDBMSTypeTestParameter() { - - Set dbmsTypes = new HashSet<>(); - for (DBMSType dbmsType : DBMSType.values()) { - try { - TestConnector.getTestDBMSConnection(dbmsType); - dbmsTypes.add(dbmsType); - } catch (SQLException | InvalidDBMSConnectionPropertiesException e) { - // skip parameter - } - } - return dbmsTypes; + /** + * Determine the DBMSType to test from the environment variable "DMBS". In case that variable is not set, use "PostgreSQL" as default + */ + public static DBMSType getDBMSTypeTestParameter() { + return DBMSType.fromString(System.getenv("DBMS")).orElse(DBMSType.POSTGRESQL); } public static void clearTables(DBMSConnection dbmsConnection) throws SQLException { + Objects.requireNonNull(dbmsConnection); DBMSType dbmsType = dbmsConnection.getProperties().getType(); if (dbmsType == DBMSType.MYSQL) { @@ -51,14 +40,4 @@ public static void clearTables(DBMSConnection dbmsConnection) throws SQLExceptio } } - static Stream getTestingDatabaseSystems() throws InvalidDBMSConnectionPropertiesException, SQLException { - Collection result = new ArrayList<>(); - for (DBMSType dbmsType : getDBMSTypeTestParameter()) { - result.add(new Object[]{ - dbmsType, - TestConnector.getTestDBMSConnection(dbmsType), - DBMSProcessor.getProcessorInstance(TestConnector.getTestDBMSConnection(dbmsType))}); - } - return result.stream(); - } } diff --git a/src/test/java/org/jabref/model/entry/CanonicalBibEntryTest.java b/src/test/java/org/jabref/model/entry/CanonicalBibEntryTest.java index 6d692112871..2bae2288167 100644 --- a/src/test/java/org/jabref/model/entry/CanonicalBibEntryTest.java +++ b/src/test/java/org/jabref/model/entry/CanonicalBibEntryTest.java @@ -14,7 +14,8 @@ void canonicalRepresentationIsCorrectForStringMonth() { BibEntry entry = new BibEntry(); entry.setMonth(Month.MAY); assertEquals("@misc{,\n" + - " month = {#may#}\n" + + " month = {#may#},\n" + + " _jabref_shared = {sharedId: -1, version: 1}\n" + "}", CanonicalBibEntry.getCanonicalRepresentation(entry)); } @@ -26,7 +27,7 @@ public void simpleCanonicalRepresentation() { e.setField(StandardField.TITLE, "def"); e.setField(StandardField.JOURNAL, "hij"); String canonicalRepresentation = CanonicalBibEntry.getCanonicalRepresentation(e); - assertEquals("@article{key,\n author = {abc},\n journal = {hij},\n title = {def}\n}", + assertEquals("@article{key,\n author = {abc},\n journal = {hij},\n title = {def},\n _jabref_shared = {sharedId: -1, version: 1}\n}", canonicalRepresentation); } @@ -36,6 +37,6 @@ public void canonicalRepresentationWithNewlines() { e.setCiteKey("key"); e.setField(StandardField.ABSTRACT, "line 1\nline 2"); String canonicalRepresentation = CanonicalBibEntry.getCanonicalRepresentation(e); - assertEquals("@article{key,\n abstract = {line 1\nline 2}\n}", canonicalRepresentation); + assertEquals("@article{key,\n abstract = {line 1\nline 2},\n _jabref_shared = {sharedId: -1, version: 1}\n}", canonicalRepresentation); } } From 82eca7d324497062dd0db39813bae9da640a90a5 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 28 Nov 2019 06:45:53 +0100 Subject: [PATCH 02/38] One big workflow for test and binaries --- .github/workflows/checkstyle.yml | 27 --- .github/workflows/deployment.yml | 110 ---------- .github/workflows/test-and-binaries.yml | 270 ++++++++++++++++++++++++ .github/workflows/test-on-mysql.yml | 40 ---- 4 files changed, 270 insertions(+), 177 deletions(-) delete mode 100644 .github/workflows/checkstyle.yml delete mode 100644 .github/workflows/deployment.yml create mode 100644 .github/workflows/test-and-binaries.yml delete mode 100644 .github/workflows/test-on-mysql.yml diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml deleted file mode 100644 index eec5b0a3d7a..00000000000 --- a/.github/workflows/checkstyle.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: checkstyle - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Run checkstyle test - run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml deleted file mode 100644 index 213d08d4d38..00000000000 --- a/.github/workflows/deployment.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: Deployment - -on: [push] - -jobs: - deploy: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - include: - - os: ubuntu-latest - displayName: linux - jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_linux-x64_bin.tar.gz - jdk14Path: /jdk-14 - archivePortable: tar -czf build/distribution/JabRef-portable_linux.tar.gz -C build/distribution JabRef && rm -R build/distribution/JabRef - - os: windows-latest - displayName: windows - jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_windows-x64_bin.zip - jdk14Path: /jdk-14 - archivePortable: 7z a -r build/distribution/JabRef-portable_windows.zip ./build/distribution/JabRef && rm -R build/distribution/JabRef - - os: macOS-latest - displayName: macOS - jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_osx-x64_bin.tar.gz - jdk14Path: /jdk-14.jdk/Contents/Home - archivePortable: tar -czf build/distribution/JabRef-portable_macos.tar.gz -C build/distribution JabRef.app && rm -R build/distribution/JabRef.app - - runs-on: ${{ matrix.os }} - name: Deploy on ${{ matrix.displayName }} - - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Extract branch name - shell: bash - run: | - ref=${GITHUB_REF#refs/heads/} - ref=${ref#refs/pull/} - ref=${ref%/merge} - echo "##[set-output name=branch;]${ref}" - id: extract_branch - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Download jpackage - # We need to download jpackage from https://jdk.java.net/jpackage/ - run: | - import tarfile - import zipfile - import sys - if sys.version_info[0] >= 3: - from urllib.request import urlretrieve - else: - from urllib import urlretrieve - - url = "${{ matrix.jpackageDownload }}" - tmpfile, headers = urlretrieve(url) - if (url.endswith("tar.gz")): - tar = tarfile.open(tmpfile) - tar.extractall() - tar.close() - elif (url.endswith("zip")): - zip = zipfile.ZipFile(tmpfile) - zip.extractall() - zip.close() - shell: python - - name: Build runtime image - run: ./gradlew -Pdev=true jlinkZip - - name: Build installer - run: | - export BADASS_JLINK_JPACKAGE_HOME="${GITHUB_WORKSPACE}${{ matrix.jdk14Path }}" - ./gradlew -Pdev=true jpackage - shell: bash - - name: Add installer as artifact - uses: actions/upload-artifact@master - with: - name: JabRef-${{ matrix.displayName }} - path: build/distribution - - name: Package application image - run: ${{ matrix.archivePortable }} - shell: bash - - name: Build and publish snap - if: matrix.os == 'ubuntu-latest' && steps.extract_branch.outputs.branch == 'master' - env: - SNAPCRAFT_LOGIN_FILE: ${{ secrets.SNAPCRAFT_LOGIN_FILE }} - run: | - mkdir .snapcraft && echo ${SNAPCRAFT_LOGIN_FILE} | base64 --decode --ignore-garbage > .snapcraft/snapcraft.cfg - docker run -v $(pwd):$(pwd) -t lyzardking/snapcraft-bionic sh -c "apt update -qq && cd $(pwd) && snapcraft && mv jabref*.snap build/distribution/ && snapcraft push build/distribution/jabref*.snap --release edge || true" - shell: bash - - name: Upload to builds.jabref.org - uses: garygrossgarten/github-action-scp@release - with: - local: build/distribution - remote: www/${{ steps.extract_branch.outputs.branch }} - host: builds.jabref.org - username: builds_jabref_org - privateKey: ${{ secrets.buildJabRefPrivateKey }} - port: 9922 diff --git a/.github/workflows/test-and-binaries.yml b/.github/workflows/test-and-binaries.yml new file mode 100644 index 00000000000..e51d714530b --- /dev/null +++ b/.github/workflows/test-and-binaries.yml @@ -0,0 +1,270 @@ +name: Deployment + +on: [push] + +jobs: + build: + name: build + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Build classes + run: ./gradlew -Pdev=true classes + - name: Upload build + uses: actions/upload-artifact@v1 + with: + name: build + path: build + check: + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Download build + uses: actions/download-artifact@v1 + with: + name: build + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Build runtime image + run: ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest + checkstyle: + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Download build + uses: actions/download-artifact@v1 + with: + name: build + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Build runtime image + run: ./gradlew checkstyleJmh checkstyleMain checkstyleTest + testmysql: + name: Test on MySQL + needs: build + runs-on: ubuntu-latest + steps: + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - uses: mirromutth/mysql-action@v1.1 + with: + host port: 3800 + container port: 3307 + character set server: 'utf8' + collation server: 'utf8_general_ci' + mysql version: '8.0' + mysql database: 'jabref' + mysql root password: 'root' + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Download build + uses: actions/download-artifact@v1 + with: + name: build + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "mysql" + testpostgres: + runs-on: ubuntu-latest + needs: build + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Download build + uses: actions/download-artifact@v1 + with: + name: build + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "postgresql" + deploy: + needs: build + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + include: + - os: ubuntu-latest + displayName: linux + jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_linux-x64_bin.tar.gz + jdk14Path: /jdk-14 + archivePortable: tar -czf build/distribution/JabRef-portable_linux.tar.gz -C build/distribution JabRef && rm -R build/distribution/JabRef + - os: windows-latest + displayName: windows + jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_windows-x64_bin.zip + jdk14Path: /jdk-14 + archivePortable: 7z a -r build/distribution/JabRef-portable_windows.zip ./build/distribution/JabRef && rm -R build/distribution/JabRef + - os: macOS-latest + displayName: macOS + jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_osx-x64_bin.tar.gz + jdk14Path: /jdk-14.jdk/Contents/Home + archivePortable: tar -czf build/distribution/JabRef-portable_macos.tar.gz -C build/distribution JabRef.app && rm -R build/distribution/JabRef.app + + runs-on: ${{ matrix.os }} + name: Deploy on ${{ matrix.displayName }} + + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Download build + uses: actions/download-artifact@v1 + with: + name: build + - name: Extract branch name + shell: bash + run: | + ref=${GITHUB_REF#refs/heads/} + ref=${ref#refs/pull/} + ref=${ref%/merge} + echo "##[set-output name=branch;]${ref}" + id: extract_branch + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Download jpackage + # We need to download jpackage from https://jdk.java.net/jpackage/ + run: | + import tarfile + import zipfile + import sys + if sys.version_info[0] >= 3: + from urllib.request import urlretrieve + else: + from urllib import urlretrieve + + url = "${{ matrix.jpackageDownload }}" + tmpfile, headers = urlretrieve(url) + if (url.endswith("tar.gz")): + tar = tarfile.open(tmpfile) + tar.extractall() + tar.close() + elif (url.endswith("zip")): + zip = zipfile.ZipFile(tmpfile) + zip.extractall() + zip.close() + shell: python + - name: Build runtime image + run: ./gradlew -Pdev=true jlinkZip + - name: Build installer + run: | + export BADASS_JLINK_JPACKAGE_HOME="${GITHUB_WORKSPACE}${{ matrix.jdk14Path }}" + ./gradlew -Pdev=true jpackage + shell: bash + - name: Add installer as artifact + uses: actions/upload-artifact@master + with: + name: JabRef-${{ matrix.displayName }} + path: build/distribution + - name: Package application image + run: ${{ matrix.archivePortable }} + shell: bash + - name: Build and publish snap + if: matrix.os == 'ubuntu-latest' && steps.extract_branch.outputs.branch == 'master' + env: + SNAPCRAFT_LOGIN_FILE: ${{ secrets.SNAPCRAFT_LOGIN_FILE }} + run: | + mkdir .snapcraft && echo ${SNAPCRAFT_LOGIN_FILE} | base64 --decode --ignore-garbage > .snapcraft/snapcraft.cfg + docker run -v $(pwd):$(pwd) -t lyzardking/snapcraft-bionic sh -c "apt update -qq && cd $(pwd) && snapcraft && mv jabref*.snap build/distribution/ && snapcraft push build/distribution/jabref*.snap --release edge || true" + shell: bash + - name: Upload to builds.jabref.org + uses: garygrossgarten/github-action-scp@release + with: + local: build/distribution + remote: www/${{ steps.extract_branch.outputs.branch }} + host: builds.jabref.org + username: builds_jabref_org + privateKey: ${{ secrets.buildJabRefPrivateKey }} + port: 9922 diff --git a/.github/workflows/test-on-mysql.yml b/.github/workflows/test-on-mysql.yml deleted file mode 100644 index 73afca981d0..00000000000 --- a/.github/workflows/test-on-mysql.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Test on MySQL - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Shutdown Ubuntu MySQL (SUDO) - run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - - uses: mirromutth/mysql-action@v1.1 - with: - host port: 3800 - container port: 3307 - character set server: 'utf8' - collation server: 'utf8_general_ci' - mysql version: '8.0' - mysql database: 'jabref' - mysql root password: 'root' - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Run database test - run: ./gradlew databaseTest - env: - DBMS: "mysql" From 98e68c6613cb2093803692103d79aca1fbe0268e Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 28 Nov 2019 07:29:02 +0100 Subject: [PATCH 03/38] Make allowPublicKeyRetrieval a property of DBMSConnectionProperties (and create a builder for the properties) --- docs/testing.md | 10 +- .../SharedDatabaseLoginDialogViewModel.java | 69 ++++---- .../shared/DBMSConnectionProperties.java | 163 +++++++----------- .../DBMSConnectionPropertiesBuilder.java | 73 ++++++++ .../shared/DatabaseConnectionProperties.java | 2 + .../shared/DBMSConnectionPropertiesTest.java | 4 +- .../logic/shared/DBMSConnectionTest.java | 2 +- .../jabref/logic/shared/TestConnector.java | 8 +- 8 files changed, 190 insertions(+), 141 deletions(-) create mode 100644 src/main/java/org/jabref/logic/shared/DBMSConnectionPropertiesBuilder.java diff --git a/docs/testing.md b/docs/testing.md index b7d7bb81bfe..8a1e08e5d7b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2,16 +2,24 @@ ## Database tests +### PostgreSQL + To quickly host a local PostgreSQL database, execute following statement: ```shell command docker run -d -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres -p 5432:5432 --name db postgres:10 postgres -c log_statement=all ``` +Set the environment variable `DBMS` to `postgres` (or leave it unset) + Then, all DBMS Tests (annotated with `@org.jabref.testutils.category.DatabaseTest`) run properly. -A MySQL can be started using following command: +### MySQL + +A MySQL DBMS can be started using following command: ``´shell command docker run -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=jabref -p 3800:3307 mysql:8.0 --port=3307 ``` + +Set the environment variable `DBMS` to `mysql`. diff --git a/src/main/java/org/jabref/gui/shared/SharedDatabaseLoginDialogViewModel.java b/src/main/java/org/jabref/gui/shared/SharedDatabaseLoginDialogViewModel.java index 78a6f139091..23e96e4d1c8 100644 --- a/src/main/java/org/jabref/gui/shared/SharedDatabaseLoginDialogViewModel.java +++ b/src/main/java/org/jabref/gui/shared/SharedDatabaseLoginDialogViewModel.java @@ -32,6 +32,7 @@ import org.jabref.logic.help.HelpFile; import org.jabref.logic.l10n.Localization; import org.jabref.logic.shared.DBMSConnectionProperties; +import org.jabref.logic.shared.DBMSConnectionPropertiesBuilder; import org.jabref.logic.shared.exception.InvalidDBMSConnectionPropertiesException; import org.jabref.logic.shared.prefs.SharedDatabasePreferences; import org.jabref.logic.shared.security.Password; @@ -109,17 +110,17 @@ public SharedDatabaseLoginDialogViewModel(JabRefFrame frame, DialogService dialo } public boolean openDatabase() { - - DBMSConnectionProperties connectionProperties = new DBMSConnectionProperties(); - connectionProperties.setType(selectedDBMSType.getValue()); - connectionProperties.setHost(host.getValue()); - connectionProperties.setPort(Integer.parseInt(port.getValue())); - connectionProperties.setDatabase(database.getValue()); - connectionProperties.setUser(user.getValue()); - connectionProperties.setPassword(password.getValue()); - connectionProperties.setUseSSL(useSSL.getValue()); - connectionProperties.setKeyStore(keystore.getValue()); - connectionProperties.setServerTimezone(serverTimezone.getValue()); + DBMSConnectionProperties connectionProperties = new DBMSConnectionPropertiesBuilder() + .setType(selectedDBMSType.getValue()) + .setHost(host.getValue()) + .setPort(Integer.parseInt(port.getValue())) + .setDatabase(database.getValue()) + .setUser(user.getValue()) + .setPassword(password.getValue()) + .setUseSSL(useSSL.getValue()) + .setKeyStore(keystore.getValue()) + .setServerTimezone(serverTimezone.getValue()) + .createDBMSConnectionProperties(); setupKeyStore(); boolean connected = openSharedDatabase(connectionProperties); @@ -136,7 +137,7 @@ private boolean openSharedDatabase(DBMSConnectionProperties connectionProperties if (isSharedDatabaseAlreadyPresent(connectionProperties)) { dialogService.showWarningDialogAndWait(Localization.lang("Shared database connection"), - Localization.lang("You are already connected to a database using entered connection details.")); + Localization.lang("You are already connected to a database using entered connection details.")); return true; } @@ -146,9 +147,9 @@ private boolean openSharedDatabase(DBMSConnectionProperties connectionProperties if (Files.exists(localFilePath) && !Files.isDirectory(localFilePath)) { boolean overwriteFilePressed = dialogService.showConfirmationDialogAndWait(Localization.lang("Existing file"), - Localization.lang("'%0' exists. Overwrite file?", localFilePath.getFileName().toString()), - Localization.lang("Overwrite file"), - Localization.lang("Cancel")); + Localization.lang("'%0' exists. Overwrite file?", localFilePath.getFileName().toString()), + Localization.lang("Overwrite file"), + Localization.lang("Cancel")); if (!overwriteFilePressed) { return true; } @@ -174,22 +175,20 @@ private boolean openSharedDatabase(DBMSConnectionProperties connectionProperties } catch (SQLException | InvalidDBMSConnectionPropertiesException exception) { frame.getDialogService().showErrorDialogAndWait(Localization.lang("Connection error"), exception); - } catch (DatabaseNotSupportedException exception) { ButtonType openHelp = new ButtonType("Open Help", ButtonData.OTHER); Optional result = dialogService.showCustomButtonDialogAndWait(AlertType.INFORMATION, - Localization.lang("Migration help information"), - Localization.lang("Entered database has obsolete structure and is no longer supported.") - + "\n" + - Localization.lang("Click help to learn about the migration of pre-3.6 databases.") - + "\n" + - Localization.lang("However, a new database was created alongside the pre-3.6 one."), - ButtonType.OK, openHelp); + Localization.lang("Migration help information"), + Localization.lang("Entered database has obsolete structure and is no longer supported.") + + "\n" + + Localization.lang("Click help to learn about the migration of pre-3.6 databases.") + + "\n" + + Localization.lang("However, a new database was created alongside the pre-3.6 one."), + ButtonType.OK, openHelp); result.filter(btn -> btn.equals(openHelp)).ifPresent(btn -> HelpAction.openHelpPage(HelpFile.SQL_DATABASE_MIGRATION)); result.filter(btn -> btn.equals(ButtonType.OK)).ifPresent(btn -> openSharedDatabase(connectionProperties)); - } loading.set(false); return false; @@ -266,10 +265,10 @@ private boolean isSharedDatabaseAlreadyPresent(DBMSConnectionProperties connecti public void showSaveDbToFileDialog() { FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() - .addExtensionFilter(StandardFileType.BIBTEX_DB) - .withDefaultExtension(StandardFileType.BIBTEX_DB) - .withInitialDirectory(Globals.prefs.get(JabRefPreferences.WORKING_DIRECTORY)) - .build(); + .addExtensionFilter(StandardFileType.BIBTEX_DB) + .withDefaultExtension(StandardFileType.BIBTEX_DB) + .withInitialDirectory(Globals.prefs.get(JabRefPreferences.WORKING_DIRECTORY)) + .build(); Optional exportPath = dialogService.showFileSaveDialog(fileDialogConfiguration); exportPath.ifPresent(path -> { folder.setValue(path.toString()); @@ -278,11 +277,11 @@ public void showSaveDbToFileDialog() { public void showOpenKeystoreFileDialog() { FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() - .addExtensionFilter(FileFilterConverter.ANY_FILE) - .addExtensionFilter(StandardFileType.JAVA_KEYSTORE) - .withDefaultExtension(StandardFileType.JAVA_KEYSTORE) - .withInitialDirectory(Globals.prefs.get(JabRefPreferences.WORKING_DIRECTORY)) - .build(); + .addExtensionFilter(FileFilterConverter.ANY_FILE) + .addExtensionFilter(StandardFileType.JAVA_KEYSTORE) + .withDefaultExtension(StandardFileType.JAVA_KEYSTORE) + .withInitialDirectory(Globals.prefs.get(JabRefPreferences.WORKING_DIRECTORY)) + .build(); Optional keystorePath = dialogService.showFileOpenDialog(fileDialogConfiguration); keystorePath.ifPresent(path -> { keystore.setValue(path.toString()); @@ -369,5 +368,7 @@ public ValidationStatus formValidation() { return formValidator.getValidationStatus(); } - public StringProperty serverTimezoneProperty() { return serverTimezone; } + public StringProperty serverTimezoneProperty() { + return serverTimezone; + } } diff --git a/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java b/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java index f7b7d9a8765..e6d6a68ff43 100644 --- a/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java +++ b/src/main/java/org/jabref/logic/shared/DBMSConnectionProperties.java @@ -27,22 +27,50 @@ public class DBMSConnectionProperties implements DatabaseConnectionProperties { private String database; private String user; private String password; + private boolean allowPublicKeyRetrieval; private boolean useSSL; - private String serverTimezone; + private String serverTimezone = ""; - //Not needed for connection, but stored for future login + // Not needed for connection, but stored for future login private String keyStore; - public DBMSConnectionProperties() { - // no data - } - + /** + * Gets all required data from {@link SharedDatabasePreferences} and sets them if present. + */ public DBMSConnectionProperties(SharedDatabasePreferences prefs) { - setFromPreferences(prefs); + if (prefs.getType().isPresent()) { + Optional dbmsType = DBMSType.fromString(prefs.getType().get()); + if (dbmsType.isPresent()) { + this.type = dbmsType.get(); + } + } + + prefs.getHost().ifPresent(theHost -> this.host = theHost); + prefs.getPort().ifPresent(thePort -> this.port = Integer.parseInt(thePort)); + prefs.getName().ifPresent(theDatabase -> this.database = theDatabase); + prefs.getKeyStoreFile().ifPresent(theKeystore -> this.keyStore = theKeystore); + prefs.getServerTimezone().ifPresent(theServerTimezone -> this.serverTimezone = theServerTimezone); + this.useSSL = prefs.isUseSSL(); + + if (prefs.getUser().isPresent()) { + this.user = prefs.getUser().get(); + if (prefs.getPassword().isPresent()) { + try { + this.password = new Password(prefs.getPassword().get().toCharArray(), prefs.getUser().get()).decrypt(); + } catch (UnsupportedEncodingException | GeneralSecurityException e) { + LOGGER.error("Could not decrypt password", e); + } + } + } + + if (!prefs.getPassword().isPresent()) { + // Some DBMS require a non-null value as a password (in case of using an empty string). + this.password = ""; + } } - public DBMSConnectionProperties(DBMSType type, String host, int port, String database, String user, - String password, boolean useSSL, String serverTimezone) { + DBMSConnectionProperties(DBMSType type, String host, int port, String database, String user, + String password, boolean useSSL, boolean allowPublicKeyRetrieval, String serverTimezone, String keyStore) { this.type = type; this.host = host; this.port = port; @@ -50,7 +78,9 @@ public DBMSConnectionProperties(DBMSType type, String host, int port, String dat this.user = user; this.password = password; this.useSSL = useSSL; + this.allowPublicKeyRetrieval = allowPublicKeyRetrieval; this.serverTimezone = serverTimezone; + this.keyStore = keyStore; } @Override @@ -58,81 +88,54 @@ public DBMSType getType() { return type; } - public void setType(DBMSType type) { - this.type = type; - } - @Override public String getHost() { return host; } - public void setHost(String host) { - this.host = host; - } - @Override public int getPort() { return port; } - public void setPort(int port) { - this.port = port; - } - @Override public String getDatabase() { return database; } - public void setDatabase(String database) { - this.database = database; - } - @Override public String getUser() { return user; } - public void setUser(String user) { - this.user = user; - } - @Override public String getPassword() { return password; } - public void setPassword(String password) { - this.password = password; - } - @Override public boolean isUseSSL() { return useSSL; } - public void setUseSSL(boolean useSSL) { - this.useSSL = useSSL; + @Override + public boolean isAllowPublicKeyRetrieval() { + return allowPublicKeyRetrieval; + } + + @Override + public String getServerTimezone() { + return serverTimezone; } public String getUrl() { String url = type.getUrl(host, port, database); - if (this.type == DBMSType.MYSQL) { - // quickfix - see https://mysqlconnector.net/troubleshooting/retrieval-public-key/ - url += "?allowPublicKeyRetrieval=true"; - url += "&useSSL=" + this.isUseSSL(); - } return url; } - @Override - public String getServerTimezone() { return serverTimezone; } - - public void setServerTimezone(String serverTimezone) { this.serverTimezone = serverTimezone; } - /** * Returns username, password and ssl as Properties Object + * * @return Properties with values for user, password and ssl */ public Properties asProperties() { @@ -140,11 +143,13 @@ public Properties asProperties() { props.setProperty("user", user); props.setProperty("password", password); props.setProperty("serverTimezone", serverTimezone); - if (useSSL) { props.setProperty("ssl", Boolean.toString(useSSL)); + props.setProperty("useSSL", Boolean.toString(useSSL)); + } + if (allowPublicKeyRetrieval) { + props.setProperty("allowPublicKeyRetrieval", Boolean.toString(allowPublicKeyRetrieval)); } - return props; } @@ -153,10 +158,6 @@ public String getKeyStore() { return keyStore; } - public void setKeyStore(String keyStore) { - this.keyStore = keyStore; - } - /** * Compares all properties except the password. */ @@ -171,13 +172,13 @@ public boolean equals(Object obj) { } DBMSConnectionProperties properties = (DBMSConnectionProperties) obj; return Objects.equals(type, properties.getType()) - && this.host.equalsIgnoreCase(properties.getHost()) - && Objects.equals(port, properties.getPort()) - && Objects.equals(database, properties.getDatabase()) - && Objects.equals(user, properties.getUser()) - && Objects.equals(useSSL, properties.isUseSSL()) - && Objects.equals(serverTimezone, properties.getServerTimezone()); - + && this.host.equalsIgnoreCase(properties.getHost()) + && Objects.equals(port, properties.getPort()) + && Objects.equals(database, properties.getDatabase()) + && Objects.equals(user, properties.getUser()) + && Objects.equals(useSSL, properties.isUseSSL()) + && Objects.equals(allowPublicKeyRetrieval, properties.isAllowPublicKeyRetrieval()) + && Objects.equals(serverTimezone, properties.getServerTimezone()); } @Override @@ -185,49 +186,13 @@ public int hashCode() { return Objects.hash(type, host, port, database, user, useSSL); } - /** - * Gets all required data from {@link SharedDatabasePreferences} and sets them if present. - */ - private void setFromPreferences(SharedDatabasePreferences prefs) { - if (prefs.getType().isPresent()) { - Optional dbmsType = DBMSType.fromString(prefs.getType().get()); - if (dbmsType.isPresent()) { - this.type = dbmsType.get(); - } - } - - prefs.getHost().ifPresent(theHost -> this.host = theHost); - prefs.getPort().ifPresent(thePort -> this.port = Integer.parseInt(thePort)); - prefs.getName().ifPresent(theDatabase -> this.database = theDatabase); - prefs.getKeyStoreFile().ifPresent(theKeystore -> this.keyStore = theKeystore); - prefs.getServerTimezone().ifPresent(theServerTimezone -> this.serverTimezone = theServerTimezone); - this.setUseSSL(prefs.isUseSSL()); - - if (prefs.getUser().isPresent()) { - this.user = prefs.getUser().get(); - if (prefs.getPassword().isPresent()) { - try { - this.password = new Password(prefs.getPassword().get().toCharArray(), prefs.getUser().get()).decrypt(); - } catch (UnsupportedEncodingException | GeneralSecurityException e) { - LOGGER.error("Could not decrypt password", e); - } - } - } - - if (!prefs.getPassword().isPresent()) { - // Some DBMS require a non-null value as a password (in case of using an empty string). - this.password = ""; - } - } - @Override public boolean isValid() { return Objects.nonNull(type) - && Objects.nonNull(host) - && Objects.nonNull(port) - && Objects.nonNull(database) - && Objects.nonNull(user) - && Objects.nonNull(password); + && Objects.nonNull(host) + && Objects.nonNull(port) + && Objects.nonNull(database) + && Objects.nonNull(user) + && Objects.nonNull(password); } - } diff --git a/src/main/java/org/jabref/logic/shared/DBMSConnectionPropertiesBuilder.java b/src/main/java/org/jabref/logic/shared/DBMSConnectionPropertiesBuilder.java new file mode 100644 index 00000000000..35e01f6e7f7 --- /dev/null +++ b/src/main/java/org/jabref/logic/shared/DBMSConnectionPropertiesBuilder.java @@ -0,0 +1,73 @@ +package org.jabref.logic.shared; + +import org.jabref.model.database.shared.DBMSType; + +public class DBMSConnectionPropertiesBuilder { + private DBMSType type; + private String host; + private int port = -1; + private String database; + private String user; + private String password; + private boolean useSSL; + private boolean allowPublicKeyRetrieval; + private String serverTimezone = ""; + private String keyStore; + + public DBMSConnectionPropertiesBuilder setType(DBMSType type) { + this.type = type; + return this; + } + + public DBMSConnectionPropertiesBuilder setHost(String host) { + this.host = host; + return this; + } + + public DBMSConnectionPropertiesBuilder setPort(int port) { + this.port = port; + return this; + } + + public DBMSConnectionPropertiesBuilder setDatabase(String database) { + this.database = database; + return this; + } + + public DBMSConnectionPropertiesBuilder setUser(String user) { + this.user = user; + return this; + } + + public DBMSConnectionPropertiesBuilder setPassword(String password) { + this.password = password; + return this; + } + + public DBMSConnectionPropertiesBuilder setUseSSL(boolean useSSL) { + this.useSSL = useSSL; + return this; + } + + public DBMSConnectionPropertiesBuilder setAllowPublicKeyRetrieval(boolean allowPublicKeyRetrieval) { + this.allowPublicKeyRetrieval = allowPublicKeyRetrieval; + return this; + } + + public DBMSConnectionPropertiesBuilder setServerTimezone(String serverTimezone) { + this.serverTimezone = serverTimezone; + return this; + } + + public DBMSConnectionPropertiesBuilder setKeyStore(String keyStore) { + this.keyStore = keyStore; + return this; + } + + public DBMSConnectionProperties createDBMSConnectionProperties() { + if (port == -1) { + port = type.getDefaultPort(); + } + return new DBMSConnectionProperties(type, host, port, database, user, password, useSSL, allowPublicKeyRetrieval, serverTimezone, keyStore); + } +} diff --git a/src/main/java/org/jabref/model/database/shared/DatabaseConnectionProperties.java b/src/main/java/org/jabref/model/database/shared/DatabaseConnectionProperties.java index d6966994bf0..e1861f3f6fe 100644 --- a/src/main/java/org/jabref/model/database/shared/DatabaseConnectionProperties.java +++ b/src/main/java/org/jabref/model/database/shared/DatabaseConnectionProperties.java @@ -20,6 +20,8 @@ public interface DatabaseConnectionProperties { boolean isUseSSL(); + boolean isAllowPublicKeyRetrieval(); + String getServerTimezone(); } diff --git a/src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java b/src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java index de9347e51af..8842284082d 100644 --- a/src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java +++ b/src/test/java/org/jabref/logic/shared/DBMSConnectionPropertiesTest.java @@ -10,7 +10,7 @@ class DBMSConnectionPropertiesTest { @Test void urlForMySqlIncludesSslConfig() { - DBMSConnectionProperties connectionProperties = new DBMSConnectionProperties(DBMSType.MYSQL, "localhost", 3108, "jabref", "user", "password", false, ""); - assertEquals("jdbc:mariadb://localhost:3108/jabref?allowPublicKeyRetrieval=true&useSSL=false", connectionProperties.getUrl()); + DBMSConnectionProperties connectionProperties = new DBMSConnectionPropertiesBuilder().setType(DBMSType.MYSQL).setHost("localhost").setPort(3108).setDatabase("jabref").setUser("user").setPassword("password").setUseSSL(false).setAllowPublicKeyRetrieval(true).setServerTimezone("").createDBMSConnectionProperties(); + assertEquals("jdbc:mariadb://localhost:3108/jabref", connectionProperties.getUrl()); } } diff --git a/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java b/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java index 9a3a34d3549..c272d604e7f 100644 --- a/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java +++ b/src/test/java/org/jabref/logic/shared/DBMSConnectionTest.java @@ -17,6 +17,6 @@ public class DBMSConnectionTest { @EnumSource(DBMSType.class) public void getConnectionFailsWhenconnectingToInvalidHost(DBMSType dbmsType) { assertThrows(SQLException.class, - () -> new DBMSConnection(new DBMSConnectionProperties(dbmsType, "XXXX", 33778, "XXXX", "XXXX", "XXXX", false, "XXXX")).getConnection()); + () -> new DBMSConnection(new DBMSConnectionPropertiesBuilder().setType(dbmsType).setHost("XXXX").setPort(33778).setDatabase("XXXX").setUser("XXXX").setPassword("XXXX").setUseSSL(false).setServerTimezone("XXXX").createDBMSConnectionProperties()).getConnection()); } } diff --git a/src/test/java/org/jabref/logic/shared/TestConnector.java b/src/test/java/org/jabref/logic/shared/TestConnector.java index ff72457dd4d..904dd336ff1 100644 --- a/src/test/java/org/jabref/logic/shared/TestConnector.java +++ b/src/test/java/org/jabref/logic/shared/TestConnector.java @@ -20,13 +20,13 @@ public static DBMSConnection getTestDBMSConnection(DBMSType dbmsType) throws SQL public static DBMSConnectionProperties getTestConnectionProperties(DBMSType dbmsType) { switch (dbmsType) { case MYSQL: - return new DBMSConnectionProperties(dbmsType, "127.0.0.1", 3800, "jabref", "root", "root", false, ""); + return new DBMSConnectionPropertiesBuilder().setType(dbmsType).setHost("127.0.0.1").setPort(3800).setDatabase("jabref").setUser("root").setPassword("root").setUseSSL(false).setAllowPublicKeyRetrieval(true).createDBMSConnectionProperties(); case POSTGRESQL: - return new DBMSConnectionProperties(dbmsType, "localhost", dbmsType.getDefaultPort(), "postgres", "postgres", "postgres", false, ""); + return new DBMSConnectionPropertiesBuilder().setType(dbmsType).setHost("localhost").setPort(dbmsType.getDefaultPort()).setDatabase("postgres").setUser("postgres").setPassword("postgres").setUseSSL(false).createDBMSConnectionProperties(); case ORACLE: - return new DBMSConnectionProperties(dbmsType, "localhost", dbmsType.getDefaultPort(), "xe", "travis", "travis", false, ""); + return new DBMSConnectionPropertiesBuilder().setType(dbmsType).setHost("localhost").setPort(dbmsType.getDefaultPort()).setDatabase("xe").setUser("travis").setPassword("travis").setUseSSL(false).createDBMSConnectionProperties(); default: - return new DBMSConnectionProperties(); + return new DBMSConnectionPropertiesBuilder().createDBMSConnectionProperties(); } } } From e49e694aa618f0431396ac45f3c5953a6ef8233d Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 28 Nov 2019 07:52:17 +0100 Subject: [PATCH 04/38] Final fixes --- .../jabref/logic/shared/DBMSSynchronizer.java | 10 ++-- .../model/database/shared/DBMSType.java | 12 +---- .../logic/shared/DBMSProcessorTest.java | 54 ++++++------------- .../org/jabref/logic/shared/DBMSTypeTest.java | 27 ++++++++++ 4 files changed, 50 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java b/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java index 9ddf90cea2c..8e253219eb3 100644 --- a/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java +++ b/src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java @@ -167,8 +167,8 @@ public void initializeDatabases() throws DatabaseNotSupportedException { } /** - * Synchronizes the local database with shared one. Possible update types are: removal, update, or insert of a {@link - * BibEntry}. + * Synchronizes the local database with shared one. Possible update types are: removal, update, or insert of a + * {@link BibEntry}. */ @Override public void synchronizeLocalDatabase() { @@ -217,9 +217,7 @@ public void synchronizeLocalDatabase() { if (!entriesToInsertIntoLocalDatabase.isEmpty()) { // in case entries should be added into the local database, insert them - for (BibEntry bibEntry : dbmsProcessor.getSharedEntries(entriesToInsertIntoLocalDatabase)) { - bibDatabase.insertEntry(bibEntry, EntriesEventSource.SHARED); - } + bibDatabase.insertEntries(dbmsProcessor.getSharedEntries(entriesToInsertIntoLocalDatabase), EntriesEventSource.SHARED); } } @@ -255,7 +253,7 @@ public void synchronizeSharedEntry(BibEntry bibEntry) { } catch (OfflineLockException exception) { eventBus.post(new UpdateRefusedEvent(bibDatabaseContext, exception.getLocalBibEntry(), exception.getSharedBibEntry())); } catch (SQLException e) { - LOGGER.error("SQL Error: ", e); + LOGGER.error("SQL Error", e); } } diff --git a/src/main/java/org/jabref/model/database/shared/DBMSType.java b/src/main/java/org/jabref/model/database/shared/DBMSType.java index 71b9e2538be..1d92d76491e 100644 --- a/src/main/java/org/jabref/model/database/shared/DBMSType.java +++ b/src/main/java/org/jabref/model/database/shared/DBMSType.java @@ -1,5 +1,6 @@ package org.jabref.model.database.shared; +import java.util.Arrays; import java.util.Locale; import java.util.Optional; @@ -51,15 +52,6 @@ public int getDefaultPort() { } public static Optional fromString(String typeName) { - if (typeName == null) { - return Optional.empty(); - } else { - try { - return Optional.of(Enum.valueOf(DBMSType.class, typeName.toUpperCase(Locale.ENGLISH))); - } catch (IllegalArgumentException exception) { - // IllegalArgumentException is thrown if "typename" is not covered by the Enum - return Optional.empty(); - } - } + return Arrays.stream(DBMSType.values()).filter(dbmsType -> dbmsType.type.equalsIgnoreCase(typeName)).findAny(); } } diff --git a/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java b/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java index c35d0e7c0eb..1aae2ccb1dc 100644 --- a/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java +++ b/src/test/java/org/jabref/logic/shared/DBMSProcessorTest.java @@ -45,32 +45,27 @@ public void setup() throws Exception { this.dbmsConnection = TestConnector.getTestDBMSConnection(dbmsType); this.dbmsProcessor = DBMSProcessor.getProcessorInstance(TestConnector.getTestDBMSConnection(dbmsType)); TestManager.clearTables(this.dbmsConnection); + dbmsProcessor.setupSharedDatabase(); } @AfterEach - public void clear() throws SQLException { + public void closeDbmsConnection() throws SQLException { this.dbmsConnection.getConnection().close(); } @Test - void testCheckBaseIntegrity() throws SQLException { - dbmsProcessor.setupSharedDatabase(); + void databaseIntegrityFullFiledAfterSetup() throws SQLException { assertTrue(dbmsProcessor.checkBaseIntegrity()); - TestManager.clearTables(this.dbmsConnection); - assertFalse(dbmsProcessor.checkBaseIntegrity()); } @Test - void testSetUpSharedDatabase() throws SQLException { - dbmsProcessor.setupSharedDatabase(); - clear(); - dbmsProcessor.setupSharedDatabase(); - assertTrue(dbmsProcessor.checkBaseIntegrity()); + void databaseIntegrityBrokenAfterClearedTables() throws SQLException { + TestManager.clearTables(this.dbmsConnection); + assertFalse(dbmsProcessor.checkBaseIntegrity()); } @Test void testInsertEntry() throws SQLException { - dbmsProcessor.setupSharedDatabase(); BibEntry expectedEntry = getBibEntryExample(); dbmsProcessor.insertEntry(expectedEntry); @@ -102,7 +97,6 @@ void testInsertEntry() throws SQLException { @Test void testUpdateEntry() throws Exception { - dbmsProcessor.setupSharedDatabase(); BibEntry expectedEntry = getBibEntryExample(); dbmsProcessor.insertEntry(expectedEntry); @@ -118,7 +112,6 @@ void testUpdateEntry() throws Exception { @Test void testGetEntriesByIdList() throws Exception { - dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); firstEntry.setField(InternalField.INTERNAL_ID_FIELD, "00001"); BibEntry secondEntry = getBibEntryExample(); @@ -133,8 +126,7 @@ void testGetEntriesByIdList() throws Exception { } @Test - void testUpdateNewerEntry() throws OfflineLockException, SQLException { - dbmsProcessor.setupSharedDatabase(); + void testUpdateNewerEntry() { BibEntry bibEntry = getBibEntryExample(); dbmsProcessor.insertEntry(bibEntry); @@ -148,7 +140,6 @@ void testUpdateNewerEntry() throws OfflineLockException, SQLException { @Test void testUpdateEqualEntry() throws OfflineLockException, SQLException { - dbmsProcessor.setupSharedDatabase(); BibEntry expectedBibEntry = getBibEntryExample(); dbmsProcessor.insertEntry(expectedBibEntry); @@ -164,7 +155,6 @@ void testUpdateEqualEntry() throws OfflineLockException, SQLException { @Test void testRemoveAllEntries() throws SQLException { - dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); BibEntry secondEntry = getBibEntryExample(); List entriesToRemove = Arrays.asList(firstEntry, secondEntry); @@ -179,7 +169,6 @@ void testRemoveAllEntries() throws SQLException { @Test void testRemoveSomeEntries() throws SQLException { - dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); BibEntry secondEntry = getBibEntryExample(); BibEntry thirdEntry = getBibEntryExample(); @@ -201,7 +190,6 @@ void testRemoveSomeEntries() throws SQLException { @Test void testRemoveSingleEntry() throws SQLException { - dbmsProcessor.setupSharedDatabase(); BibEntry entryToRemove = getBibEntryExample(); dbmsProcessor.insertEntry(entryToRemove); dbmsProcessor.removeEntries(Collections.singletonList(entryToRemove)); @@ -212,14 +200,12 @@ void testRemoveSingleEntry() throws SQLException { } @Test - void testRemoveEntriesOnNullThrows() throws SQLException { - dbmsProcessor.setupSharedDatabase(); + void testRemoveEntriesOnNullThrows() { assertThrows(NullPointerException.class, () -> dbmsProcessor.removeEntries(null)); } @Test void testRemoveEmptyEntryList() throws SQLException { - dbmsProcessor.setupSharedDatabase(); dbmsProcessor.removeEntries(Collections.emptyList()); try (ResultSet entryResultSet = selectFrom("ENTRY", dbmsConnection, dbmsProcessor)) { @@ -228,8 +214,7 @@ void testRemoveEmptyEntryList() throws SQLException { } @Test - void testGetSharedEntries() throws SQLException { - dbmsProcessor.setupSharedDatabase(); + void testGetSharedEntries() { BibEntry bibEntry = getBibEntryExampleWithEmptyFields(); dbmsProcessor.insertEntry(bibEntry); @@ -240,8 +225,7 @@ void testGetSharedEntries() throws SQLException { } @Test - void testGetSharedEntry() throws SQLException { - dbmsProcessor.setupSharedDatabase(); + void testGetSharedEntry() { BibEntry expectedBibEntry = getBibEntryExampleWithEmptyFields(); dbmsProcessor.insertEntry(expectedBibEntry); @@ -252,15 +236,13 @@ void testGetSharedEntry() throws SQLException { } @Test - void testGetNotExistingSharedEntry() throws SQLException { - dbmsProcessor.setupSharedDatabase(); + void testGetNotExistingSharedEntry() { Optional actualBibEntryOptional = dbmsProcessor.getSharedEntry(1); assertFalse(actualBibEntryOptional.isPresent()); } @Test void testGetSharedIDVersionMapping() throws OfflineLockException, SQLException { - dbmsProcessor.setupSharedDatabase(); BibEntry firstEntry = getBibEntryExample(); BibEntry secondEntry = getBibEntryExample(); @@ -278,8 +260,7 @@ void testGetSharedIDVersionMapping() throws OfflineLockException, SQLException { } @Test - void testGetSharedMetaData() throws SQLException { - dbmsProcessor.setupSharedDatabase(); + void testGetSharedMetaData() { insertMetaData("databaseType", "bibtex;", dbmsConnection, dbmsProcessor); insertMetaData("protectedFlag", "true;", dbmsConnection, dbmsProcessor); insertMetaData("saveActions", "enabled;\nauthor[capitalize,html_to_latex]\ntitle[title_case]\n;", dbmsConnection, dbmsProcessor); @@ -293,7 +274,6 @@ void testGetSharedMetaData() throws SQLException { @Test void testSetSharedMetaData() throws SQLException { - dbmsProcessor.setupSharedDatabase(); Map expectedMetaData = getMetaDataExample(); dbmsProcessor.setSharedMetaData(expectedMetaData); @@ -302,7 +282,7 @@ void testSetSharedMetaData() throws SQLException { assertEquals(expectedMetaData, actualMetaData); } - private Map getMetaDataExample() { + private static Map getMetaDataExample() { Map expectedMetaData = new HashMap<>(); expectedMetaData.put("databaseType", "bibtex;"); @@ -313,7 +293,7 @@ private Map getMetaDataExample() { return expectedMetaData; } - private BibEntry getBibEntryExampleWithEmptyFields() { + private static BibEntry getBibEntryExampleWithEmptyFields() { BibEntry bibEntry = new BibEntry() .withField(StandardField.AUTHOR, "Author") .withField(StandardField.TITLE, "") @@ -322,7 +302,7 @@ private BibEntry getBibEntryExampleWithEmptyFields() { return bibEntry; } - private BibEntry getBibEntryExample() { + private static BibEntry getBibEntryExample() { return new BibEntry(StandardEntryType.InProceedings) .withField(StandardField.AUTHOR, "Wirthlin, Michael J and Hutchings, Brad L and Gilson, Kent L") .withField(StandardField.TITLE, "The nano processor: a low resource reconfigurable processor") @@ -352,11 +332,11 @@ private void insertMetaData(String key, String value, DBMSConnection dbmsConnect } } - private String escape(String expression, DBMSProcessor dbmsProcessor) { + private static String escape(String expression, DBMSProcessor dbmsProcessor) { return dbmsProcessor.escape(expression); } - private String escapeValue(String value) { + private static String escapeValue(String value) { return "'" + value + "'"; } } diff --git a/src/test/java/org/jabref/logic/shared/DBMSTypeTest.java b/src/test/java/org/jabref/logic/shared/DBMSTypeTest.java index 87ba1169cc3..94dd5d67dbf 100644 --- a/src/test/java/org/jabref/logic/shared/DBMSTypeTest.java +++ b/src/test/java/org/jabref/logic/shared/DBMSTypeTest.java @@ -1,5 +1,7 @@ package org.jabref.logic.shared; +import java.util.Optional; + import org.jabref.model.database.shared.DBMSType; import org.jabref.testutils.category.DatabaseTest; @@ -18,6 +20,31 @@ public void testToString() { assertEquals("PostgreSQL", DBMSType.POSTGRESQL.toString()); } + @Test + public void fromStringWorksForMySQL() { + assertEquals(Optional.of(DBMSType.MYSQL), DBMSType.fromString("MySQL")); + } + + @Test + public void fromStringWorksForPostgreSQL() { + assertEquals(Optional.of(DBMSType.POSTGRESQL), DBMSType.fromString("PostgreSQL")); + } + + @Test + public void fromStringWorksForNullString() { + assertEquals(Optional.empty(), DBMSType.fromString(null)); + } + + @Test + public void fromStringWorksForEmptyString() { + assertEquals(Optional.empty(), DBMSType.fromString("")); + } + + @Test + public void fromStringWorksForUnkownString() { + assertEquals(Optional.empty(), DBMSType.fromString("unknown")); + } + @Test public void testGetDriverClassPath() { assertEquals("org.mariadb.jdbc.Driver", DBMSType.MYSQL.getDriverClassPath()); From 6d6ab5ed5c6e53d88a86ce8331054f8701eb5e41 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 28 Nov 2019 08:25:05 +0100 Subject: [PATCH 05/38] Setup JDK directly before gradle --- .github/workflows/test-and-binaries.yml | 40 +++++++++++++++---------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-and-binaries.yml b/.github/workflows/test-and-binaries.yml index e51d714530b..f3e3f82e2c6 100644 --- a/.github/workflows/test-and-binaries.yml +++ b/.github/workflows/test-and-binaries.yml @@ -12,10 +12,6 @@ jobs: with: depth: 1 submodules: false - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - uses: actions/cache@v1 with: path: ~/.gradle/caches @@ -24,6 +20,10 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 - name: Build classes run: ./gradlew -Pdev=true classes - name: Upload build @@ -44,6 +44,10 @@ jobs: uses: actions/download-artifact@v1 with: name: build + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 - uses: actions/cache@v1 with: path: ~/.gradle/caches @@ -75,6 +79,10 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 - name: Build runtime image run: ./gradlew checkstyleJmh checkstyleMain checkstyleTest testmysql: @@ -93,10 +101,6 @@ jobs: mysql version: '8.0' mysql database: 'jabref' mysql root password: 'root' - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - name: Checkout source uses: actions/checkout@v1 with: @@ -114,6 +118,10 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 - name: Run database test run: ./gradlew databaseTest env: @@ -142,10 +150,6 @@ jobs: uses: actions/download-artifact@v1 with: name: build - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - uses: actions/cache@v1 with: path: ~/.gradle/caches @@ -154,6 +158,10 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 - name: Run database test run: ./gradlew databaseTest env: @@ -202,10 +210,6 @@ jobs: ref=${ref%/merge} echo "##[set-output name=branch;]${ref}" id: extract_branch - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - uses: actions/cache@v1 with: path: ~/.gradle/caches @@ -236,6 +240,10 @@ jobs: zip.extractall() zip.close() shell: python + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 - name: Build runtime image run: ./gradlew -Pdev=true jlinkZip - name: Build installer From 570bc2881c9b718eacf8164814a830ce8a44445f Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 28 Nov 2019 08:33:36 +0100 Subject: [PATCH 06/38] Really remove separate test-on-posgres workflow --- .github/workflows/test-and-binaries.yml | 1 + .github/workflows/test-on-postgres.yml | 40 ------------------------- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 .github/workflows/test-on-postgres.yml diff --git a/.github/workflows/test-and-binaries.yml b/.github/workflows/test-and-binaries.yml index f3e3f82e2c6..66bb49ea18f 100644 --- a/.github/workflows/test-and-binaries.yml +++ b/.github/workflows/test-and-binaries.yml @@ -127,6 +127,7 @@ jobs: env: DBMS: "mysql" testpostgres: + name: Test on PostgreSQL runs-on: ubuntu-latest needs: build services: diff --git a/.github/workflows/test-on-postgres.yml b/.github/workflows/test-on-postgres.yml deleted file mode 100644 index f5b3db7d496..00000000000 --- a/.github/workflows/test-on-postgres.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Test on PostgreSQL - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - services: - postgres: - image: postgres:10.8 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Run database test - run: ./gradlew databaseTest - env: - DBMS: "postgresql" From 77ee50043cb849389f9a20d5ceb427dc35ceda2f Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 28 Nov 2019 20:42:18 +0100 Subject: [PATCH 07/38] Revert "Really remove separate test-on-posgres workflow" This reverts commit aa9b10844ba2e758daafe5a65bf0c3775740fc49. --- .github/workflows/checkstyle.yml | 27 +++ .github/workflows/deployment.yml | 110 ++++++++++ .github/workflows/test-and-binaries.yml | 279 ------------------------ .github/workflows/test-on-mysql.yml | 40 ++++ .github/workflows/test-on-postgres.yml | 40 ++++ 5 files changed, 217 insertions(+), 279 deletions(-) create mode 100644 .github/workflows/checkstyle.yml create mode 100644 .github/workflows/deployment.yml delete mode 100644 .github/workflows/test-and-binaries.yml create mode 100644 .github/workflows/test-on-mysql.yml create mode 100644 .github/workflows/test-on-postgres.yml diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml new file mode 100644 index 00000000000..eec5b0a3d7a --- /dev/null +++ b/.github/workflows/checkstyle.yml @@ -0,0 +1,27 @@ +name: checkstyle + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run checkstyle test + run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml new file mode 100644 index 00000000000..213d08d4d38 --- /dev/null +++ b/.github/workflows/deployment.yml @@ -0,0 +1,110 @@ +name: Deployment + +on: [push] + +jobs: + deploy: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + include: + - os: ubuntu-latest + displayName: linux + jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_linux-x64_bin.tar.gz + jdk14Path: /jdk-14 + archivePortable: tar -czf build/distribution/JabRef-portable_linux.tar.gz -C build/distribution JabRef && rm -R build/distribution/JabRef + - os: windows-latest + displayName: windows + jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_windows-x64_bin.zip + jdk14Path: /jdk-14 + archivePortable: 7z a -r build/distribution/JabRef-portable_windows.zip ./build/distribution/JabRef && rm -R build/distribution/JabRef + - os: macOS-latest + displayName: macOS + jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_osx-x64_bin.tar.gz + jdk14Path: /jdk-14.jdk/Contents/Home + archivePortable: tar -czf build/distribution/JabRef-portable_macos.tar.gz -C build/distribution JabRef.app && rm -R build/distribution/JabRef.app + + runs-on: ${{ matrix.os }} + name: Deploy on ${{ matrix.displayName }} + + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Extract branch name + shell: bash + run: | + ref=${GITHUB_REF#refs/heads/} + ref=${ref#refs/pull/} + ref=${ref%/merge} + echo "##[set-output name=branch;]${ref}" + id: extract_branch + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Download jpackage + # We need to download jpackage from https://jdk.java.net/jpackage/ + run: | + import tarfile + import zipfile + import sys + if sys.version_info[0] >= 3: + from urllib.request import urlretrieve + else: + from urllib import urlretrieve + + url = "${{ matrix.jpackageDownload }}" + tmpfile, headers = urlretrieve(url) + if (url.endswith("tar.gz")): + tar = tarfile.open(tmpfile) + tar.extractall() + tar.close() + elif (url.endswith("zip")): + zip = zipfile.ZipFile(tmpfile) + zip.extractall() + zip.close() + shell: python + - name: Build runtime image + run: ./gradlew -Pdev=true jlinkZip + - name: Build installer + run: | + export BADASS_JLINK_JPACKAGE_HOME="${GITHUB_WORKSPACE}${{ matrix.jdk14Path }}" + ./gradlew -Pdev=true jpackage + shell: bash + - name: Add installer as artifact + uses: actions/upload-artifact@master + with: + name: JabRef-${{ matrix.displayName }} + path: build/distribution + - name: Package application image + run: ${{ matrix.archivePortable }} + shell: bash + - name: Build and publish snap + if: matrix.os == 'ubuntu-latest' && steps.extract_branch.outputs.branch == 'master' + env: + SNAPCRAFT_LOGIN_FILE: ${{ secrets.SNAPCRAFT_LOGIN_FILE }} + run: | + mkdir .snapcraft && echo ${SNAPCRAFT_LOGIN_FILE} | base64 --decode --ignore-garbage > .snapcraft/snapcraft.cfg + docker run -v $(pwd):$(pwd) -t lyzardking/snapcraft-bionic sh -c "apt update -qq && cd $(pwd) && snapcraft && mv jabref*.snap build/distribution/ && snapcraft push build/distribution/jabref*.snap --release edge || true" + shell: bash + - name: Upload to builds.jabref.org + uses: garygrossgarten/github-action-scp@release + with: + local: build/distribution + remote: www/${{ steps.extract_branch.outputs.branch }} + host: builds.jabref.org + username: builds_jabref_org + privateKey: ${{ secrets.buildJabRefPrivateKey }} + port: 9922 diff --git a/.github/workflows/test-and-binaries.yml b/.github/workflows/test-and-binaries.yml deleted file mode 100644 index 66bb49ea18f..00000000000 --- a/.github/workflows/test-and-binaries.yml +++ /dev/null @@ -1,279 +0,0 @@ -name: Deployment - -on: [push] - -jobs: - build: - name: build - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - name: Build classes - run: ./gradlew -Pdev=true classes - - name: Upload build - uses: actions/upload-artifact@v1 - with: - name: build - path: build - check: - needs: build - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Download build - uses: actions/download-artifact@v1 - with: - name: build - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Build runtime image - run: ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest - checkstyle: - needs: build - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Download build - uses: actions/download-artifact@v1 - with: - name: build - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - name: Build runtime image - run: ./gradlew checkstyleJmh checkstyleMain checkstyleTest - testmysql: - name: Test on MySQL - needs: build - runs-on: ubuntu-latest - steps: - - name: Shutdown Ubuntu MySQL (SUDO) - run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - - uses: mirromutth/mysql-action@v1.1 - with: - host port: 3800 - container port: 3307 - character set server: 'utf8' - collation server: 'utf8_general_ci' - mysql version: '8.0' - mysql database: 'jabref' - mysql root password: 'root' - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Download build - uses: actions/download-artifact@v1 - with: - name: build - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - name: Run database test - run: ./gradlew databaseTest - env: - DBMS: "mysql" - testpostgres: - name: Test on PostgreSQL - runs-on: ubuntu-latest - needs: build - services: - postgres: - image: postgres:10.8 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Download build - uses: actions/download-artifact@v1 - with: - name: build - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - name: Run database test - run: ./gradlew databaseTest - env: - DBMS: "postgresql" - deploy: - needs: build - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - include: - - os: ubuntu-latest - displayName: linux - jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_linux-x64_bin.tar.gz - jdk14Path: /jdk-14 - archivePortable: tar -czf build/distribution/JabRef-portable_linux.tar.gz -C build/distribution JabRef && rm -R build/distribution/JabRef - - os: windows-latest - displayName: windows - jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_windows-x64_bin.zip - jdk14Path: /jdk-14 - archivePortable: 7z a -r build/distribution/JabRef-portable_windows.zip ./build/distribution/JabRef && rm -R build/distribution/JabRef - - os: macOS-latest - displayName: macOS - jpackageDownload: https://download.java.net/java/early_access/jpackage/1/openjdk-14-jpackage+1-70_osx-x64_bin.tar.gz - jdk14Path: /jdk-14.jdk/Contents/Home - archivePortable: tar -czf build/distribution/JabRef-portable_macos.tar.gz -C build/distribution JabRef.app && rm -R build/distribution/JabRef.app - - runs-on: ${{ matrix.os }} - name: Deploy on ${{ matrix.displayName }} - - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Download build - uses: actions/download-artifact@v1 - with: - name: build - - name: Extract branch name - shell: bash - run: | - ref=${GITHUB_REF#refs/heads/} - ref=${ref#refs/pull/} - ref=${ref%/merge} - echo "##[set-output name=branch;]${ref}" - id: extract_branch - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Download jpackage - # We need to download jpackage from https://jdk.java.net/jpackage/ - run: | - import tarfile - import zipfile - import sys - if sys.version_info[0] >= 3: - from urllib.request import urlretrieve - else: - from urllib import urlretrieve - - url = "${{ matrix.jpackageDownload }}" - tmpfile, headers = urlretrieve(url) - if (url.endswith("tar.gz")): - tar = tarfile.open(tmpfile) - tar.extractall() - tar.close() - elif (url.endswith("zip")): - zip = zipfile.ZipFile(tmpfile) - zip.extractall() - zip.close() - shell: python - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - name: Build runtime image - run: ./gradlew -Pdev=true jlinkZip - - name: Build installer - run: | - export BADASS_JLINK_JPACKAGE_HOME="${GITHUB_WORKSPACE}${{ matrix.jdk14Path }}" - ./gradlew -Pdev=true jpackage - shell: bash - - name: Add installer as artifact - uses: actions/upload-artifact@master - with: - name: JabRef-${{ matrix.displayName }} - path: build/distribution - - name: Package application image - run: ${{ matrix.archivePortable }} - shell: bash - - name: Build and publish snap - if: matrix.os == 'ubuntu-latest' && steps.extract_branch.outputs.branch == 'master' - env: - SNAPCRAFT_LOGIN_FILE: ${{ secrets.SNAPCRAFT_LOGIN_FILE }} - run: | - mkdir .snapcraft && echo ${SNAPCRAFT_LOGIN_FILE} | base64 --decode --ignore-garbage > .snapcraft/snapcraft.cfg - docker run -v $(pwd):$(pwd) -t lyzardking/snapcraft-bionic sh -c "apt update -qq && cd $(pwd) && snapcraft && mv jabref*.snap build/distribution/ && snapcraft push build/distribution/jabref*.snap --release edge || true" - shell: bash - - name: Upload to builds.jabref.org - uses: garygrossgarten/github-action-scp@release - with: - local: build/distribution - remote: www/${{ steps.extract_branch.outputs.branch }} - host: builds.jabref.org - username: builds_jabref_org - privateKey: ${{ secrets.buildJabRefPrivateKey }} - port: 9922 diff --git a/.github/workflows/test-on-mysql.yml b/.github/workflows/test-on-mysql.yml new file mode 100644 index 00000000000..73afca981d0 --- /dev/null +++ b/.github/workflows/test-on-mysql.yml @@ -0,0 +1,40 @@ +name: Test on MySQL + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - uses: mirromutth/mysql-action@v1.1 + with: + host port: 3800 + container port: 3307 + character set server: 'utf8' + collation server: 'utf8_general_ci' + mysql version: '8.0' + mysql database: 'jabref' + mysql root password: 'root' + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "mysql" diff --git a/.github/workflows/test-on-postgres.yml b/.github/workflows/test-on-postgres.yml new file mode 100644 index 00000000000..f5b3db7d496 --- /dev/null +++ b/.github/workflows/test-on-postgres.yml @@ -0,0 +1,40 @@ +name: Test on PostgreSQL + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "postgresql" From f8ebaf4ac44bc76ee96f7d584fa972b90c66af66 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 28 Nov 2019 21:14:08 +0100 Subject: [PATCH 08/38] Fix checkstyle --- src/main/java/org/jabref/model/database/shared/DBMSType.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/jabref/model/database/shared/DBMSType.java b/src/main/java/org/jabref/model/database/shared/DBMSType.java index 1d92d76491e..4a3052f80e1 100644 --- a/src/main/java/org/jabref/model/database/shared/DBMSType.java +++ b/src/main/java/org/jabref/model/database/shared/DBMSType.java @@ -1,7 +1,6 @@ package org.jabref.model.database.shared; import java.util.Arrays; -import java.util.Locale; import java.util.Optional; /** From cac6ac666fdf2a0eb3bd844632ee9715f95e5fac Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 07:09:10 +0100 Subject: [PATCH 09/38] Merge MySQL and Postgres test into one workflow --- .github/workflows/database-tests.yml | 77 ++++++++++++++++++++++++++ .github/workflows/test-on-mysql.yml | 40 ------------- .github/workflows/test-on-postgres.yml | 40 ------------- 3 files changed, 77 insertions(+), 80 deletions(-) create mode 100644 .github/workflows/database-tests.yml delete mode 100644 .github/workflows/test-on-mysql.yml delete mode 100644 .github/workflows/test-on-postgres.yml diff --git a/.github/workflows/database-tests.yml b/.github/workflows/database-tests.yml new file mode 100644 index 00000000000..80215242174 --- /dev/null +++ b/.github/workflows/database-tests.yml @@ -0,0 +1,77 @@ +name: database tests + +on: [push] + +jobs: + mysql: + name: "MySQL" + runs-on: ubuntu-latest + steps: + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - uses: mirromutth/mysql-action@v1.1 + with: + host port: 3800 + container port: 3307 + character set server: 'utf8' + collation server: 'utf8_general_ci' + mysql version: '8.0' + mysql database: 'jabref' + mysql root password: 'root' + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "mysql" + postgres: + name: "PostgreSQL" + runs-on: ubuntu-latest + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run database test + run: ./gradlew databaseTest + env: + DBMS: "postgresql" diff --git a/.github/workflows/test-on-mysql.yml b/.github/workflows/test-on-mysql.yml deleted file mode 100644 index 73afca981d0..00000000000 --- a/.github/workflows/test-on-mysql.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Test on MySQL - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Shutdown Ubuntu MySQL (SUDO) - run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - - uses: mirromutth/mysql-action@v1.1 - with: - host port: 3800 - container port: 3307 - character set server: 'utf8' - collation server: 'utf8_general_ci' - mysql version: '8.0' - mysql database: 'jabref' - mysql root password: 'root' - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Run database test - run: ./gradlew databaseTest - env: - DBMS: "mysql" diff --git a/.github/workflows/test-on-postgres.yml b/.github/workflows/test-on-postgres.yml deleted file mode 100644 index f5b3db7d496..00000000000 --- a/.github/workflows/test-on-postgres.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Test on PostgreSQL - -on: [push] - -jobs: - test: - runs-on: ubuntu-latest - services: - postgres: - image: postgres:10.8 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Run database test - run: ./gradlew databaseTest - env: - DBMS: "postgresql" From 936ea25cb4a39a16fe35af86837a6d9954449f2e Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 07:12:08 +0100 Subject: [PATCH 10/38] Have names lowercase also in deployment.yml --- .github/workflows/deployment.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 213d08d4d38..5f423809ba3 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -1,4 +1,4 @@ -name: Deployment +name: deployment on: [push] @@ -26,15 +26,15 @@ jobs: archivePortable: tar -czf build/distribution/JabRef-portable_macos.tar.gz -C build/distribution JabRef.app && rm -R build/distribution/JabRef.app runs-on: ${{ matrix.os }} - name: Deploy on ${{ matrix.displayName }} + name: deploy on ${{ matrix.displayName }} steps: - - name: Checkout source + - name: checkout source uses: actions/checkout@v1 with: depth: 1 submodules: false - - name: Extract branch name + - name: extract branch name shell: bash run: | ref=${GITHUB_REF#refs/heads/} @@ -54,7 +54,7 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- - - name: Download jpackage + - name: download jpackage # We need to download jpackage from https://jdk.java.net/jpackage/ run: | import tarfile @@ -76,22 +76,22 @@ jobs: zip.extractall() zip.close() shell: python - - name: Build runtime image + - name: build runtime image run: ./gradlew -Pdev=true jlinkZip - - name: Build installer + - name: build installer run: | export BADASS_JLINK_JPACKAGE_HOME="${GITHUB_WORKSPACE}${{ matrix.jdk14Path }}" ./gradlew -Pdev=true jpackage shell: bash - - name: Add installer as artifact + - name: add installer as artifact uses: actions/upload-artifact@master with: name: JabRef-${{ matrix.displayName }} path: build/distribution - - name: Package application image + - name: package application image run: ${{ matrix.archivePortable }} shell: bash - - name: Build and publish snap + - name: build and publish snap if: matrix.os == 'ubuntu-latest' && steps.extract_branch.outputs.branch == 'master' env: SNAPCRAFT_LOGIN_FILE: ${{ secrets.SNAPCRAFT_LOGIN_FILE }} @@ -99,7 +99,7 @@ jobs: mkdir .snapcraft && echo ${SNAPCRAFT_LOGIN_FILE} | base64 --decode --ignore-garbage > .snapcraft/snapcraft.cfg docker run -v $(pwd):$(pwd) -t lyzardking/snapcraft-bionic sh -c "apt update -qq && cd $(pwd) && snapcraft && mv jabref*.snap build/distribution/ && snapcraft push build/distribution/jabref*.snap --release edge || true" shell: bash - - name: Upload to builds.jabref.org + - name: upload to builds.jabref.org uses: garygrossgarten/github-action-scp@release with: local: build/distribution From 565f4e468eaa4577dbdd66a2359454568db1b360 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 08:47:11 +0100 Subject: [PATCH 11/38] Sequentially execute database tests - and integrate codecov --- .github/workflows/database-tests.yml | 58 ++++++++++------------------ .travis.yml | 5 --- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/.github/workflows/database-tests.yml b/.github/workflows/database-tests.yml index 80215242174..a00a3361546 100644 --- a/.github/workflows/database-tests.yml +++ b/.github/workflows/database-tests.yml @@ -3,11 +3,21 @@ name: database tests on: [push] jobs: - mysql: - name: "MySQL" + databasetest: runs-on: ubuntu-latest + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - name: Shutdown Ubuntu MySQL (SUDO) + - name: shutdown Ubuntu MySQL run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - uses: mirromutth/mysql-action@v1.1 with: @@ -22,7 +32,7 @@ jobs: uses: actions/setup-java@v1 with: java-version: 13 - - name: Checkout source + - name: checkout source uses: actions/checkout@v1 with: depth: 1 @@ -35,43 +45,15 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- - - name: Run database test + - name: test on MySQL run: ./gradlew databaseTest env: DBMS: "mysql" - postgres: - name: "PostgreSQL" - runs-on: ubuntu-latest - services: - postgres: - image: postgres:10.8 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - name: Run database test + - name: test on PostgreSQL run: ./gradlew databaseTest env: DBMS: "postgresql" + - name: update code coverage + run: ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.travis.yml b/.travis.yml index b3d2e28825a..1d233124393 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,10 +18,6 @@ env: - TEST_SUITE=check - TEST_SUITE=fetcherTest - TEST_SUITE=guiTest - # codecov may fail by itself (see gradle build file) - # The codecov test itself fails only in case there is an issue with the codecov update. - # Thus, a decrease in the testing coverage does not lead to a failure in Travis - - TEST_SUITE=codecov - DEPENDENCY_UPDATES=check matrix: @@ -42,7 +38,6 @@ install: true script: - if [ "$TEST_SUITE" != "guiTest" ] && [ "$TEST_SUITE" != "checkstyle" ] && [ "$TEST_SUITE" != "codecov" ]; then ./gradlew $TEST_SUITE $OPTIONS -x checkstyleJmh -x checkstyleMain -x checkstyleTest; fi - if [ "$TEST_SUITE" == "guiTest" ]; then ./buildres/gui-tests.sh; fi - - if [ "$TEST_SUITE" == "codecov" ]; then ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); fi - if [ "$DEPENDENCY_UPDATES" == "check" ]; then ./gradlew -q checkOutdatedDependencies; fi after_failure: From 9cd50a1a9c7907f2fbd0c12f3f11aad97e3ee30a Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 10:21:18 +0100 Subject: [PATCH 12/38] Enable GUI tests using GitHub workflows --- .github/workflows/database-tests.yml | 13 +++++-------- .travis.yml | 7 ------- buildres/gui-tests.sh | 16 ---------------- 3 files changed, 5 insertions(+), 31 deletions(-) delete mode 100755 buildres/gui-tests.sh diff --git a/.github/workflows/database-tests.yml b/.github/workflows/database-tests.yml index a00a3361546..edec47d2f42 100644 --- a/.github/workflows/database-tests.yml +++ b/.github/workflows/database-tests.yml @@ -45,15 +45,12 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - name: test (include PostgreSQL and update code coverage metrics) + run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + DBMS: "postgresql" - name: test on MySQL run: ./gradlew databaseTest env: DBMS: "mysql" - - name: test on PostgreSQL - run: ./gradlew databaseTest - env: - DBMS: "postgresql" - - name: update code coverage - run: ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.travis.yml b/.travis.yml index 1d233124393..293c7a1f8b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,16 +15,11 @@ env: global: - GRADLE_OPTS=-Dorg.gradle.daemon=false matrix: - - TEST_SUITE=check - - TEST_SUITE=fetcherTest - - TEST_SUITE=guiTest - DEPENDENCY_UPDATES=check matrix: fast_finish: true allow_failures: - - env: TEST_SUITE=fetcherTest - - env: TEST_SUITE=guiTest - env: DEPENDENCY_UPDATES=check # JavaFX localization tests need a running X environment @@ -36,8 +31,6 @@ before_install: install: true script: - - if [ "$TEST_SUITE" != "guiTest" ] && [ "$TEST_SUITE" != "checkstyle" ] && [ "$TEST_SUITE" != "codecov" ]; then ./gradlew $TEST_SUITE $OPTIONS -x checkstyleJmh -x checkstyleMain -x checkstyleTest; fi - - if [ "$TEST_SUITE" == "guiTest" ]; then ./buildres/gui-tests.sh; fi - if [ "$DEPENDENCY_UPDATES" == "check" ]; then ./gradlew -q checkOutdatedDependencies; fi after_failure: diff --git a/buildres/gui-tests.sh b/buildres/gui-tests.sh deleted file mode 100755 index 137818a1bb3..00000000000 --- a/buildres/gui-tests.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# no need for databases for the integrationTest -> save memory overflow -# currently does not work: "stop: Unknown instance:" - sudo service mysql stop -sudo service postgresql stop -# following services identified by "sudo service --status-all" do not need to run, too -# excluded: rsyslog (feels wrong), udev (feels wrong), friendly-recovery ("Unknown instance" error) -sudo service acpid stop -sudo service atd stop -sudo service cron stop -sudo service memcached stop -sudo service ntp stop -sudo service rabbitmq-server stop -sudo service resolvconf stop -sudo service sshguard stop -sudo service ssh stop -timeout 60 ./gradlew guiTest -Dscan --info From 251d029df4fa480f444bbfc003e06e8cb605de27 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 10:29:38 +0100 Subject: [PATCH 13/38] Revert "Have names lowercase also in deployment.yml" This reverts commit 936ea25cb4a39a16fe35af86837a6d9954449f2e. --- .github/workflows/deployment.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 5f423809ba3..213d08d4d38 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -1,4 +1,4 @@ -name: deployment +name: Deployment on: [push] @@ -26,15 +26,15 @@ jobs: archivePortable: tar -czf build/distribution/JabRef-portable_macos.tar.gz -C build/distribution JabRef.app && rm -R build/distribution/JabRef.app runs-on: ${{ matrix.os }} - name: deploy on ${{ matrix.displayName }} + name: Deploy on ${{ matrix.displayName }} steps: - - name: checkout source + - name: Checkout source uses: actions/checkout@v1 with: depth: 1 submodules: false - - name: extract branch name + - name: Extract branch name shell: bash run: | ref=${GITHUB_REF#refs/heads/} @@ -54,7 +54,7 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- - - name: download jpackage + - name: Download jpackage # We need to download jpackage from https://jdk.java.net/jpackage/ run: | import tarfile @@ -76,22 +76,22 @@ jobs: zip.extractall() zip.close() shell: python - - name: build runtime image + - name: Build runtime image run: ./gradlew -Pdev=true jlinkZip - - name: build installer + - name: Build installer run: | export BADASS_JLINK_JPACKAGE_HOME="${GITHUB_WORKSPACE}${{ matrix.jdk14Path }}" ./gradlew -Pdev=true jpackage shell: bash - - name: add installer as artifact + - name: Add installer as artifact uses: actions/upload-artifact@master with: name: JabRef-${{ matrix.displayName }} path: build/distribution - - name: package application image + - name: Package application image run: ${{ matrix.archivePortable }} shell: bash - - name: build and publish snap + - name: Build and publish snap if: matrix.os == 'ubuntu-latest' && steps.extract_branch.outputs.branch == 'master' env: SNAPCRAFT_LOGIN_FILE: ${{ secrets.SNAPCRAFT_LOGIN_FILE }} @@ -99,7 +99,7 @@ jobs: mkdir .snapcraft && echo ${SNAPCRAFT_LOGIN_FILE} | base64 --decode --ignore-garbage > .snapcraft/snapcraft.cfg docker run -v $(pwd):$(pwd) -t lyzardking/snapcraft-bionic sh -c "apt update -qq && cd $(pwd) && snapcraft && mv jabref*.snap build/distribution/ && snapcraft push build/distribution/jabref*.snap --release edge || true" shell: bash - - name: upload to builds.jabref.org + - name: Upload to builds.jabref.org uses: garygrossgarten/github-action-scp@release with: local: build/distribution From 2db3ffc0517e4cd2d1e2cd99123641bf76651777 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 10:42:55 +0100 Subject: [PATCH 14/38] Fix name of workflow for tests, separate tests into basic, fetchers, gui, dBs --- .../{database-tests.yml => tests.yml} | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) rename .github/workflows/{database-tests.yml => tests.yml} (74%) diff --git a/.github/workflows/database-tests.yml b/.github/workflows/tests.yml similarity index 74% rename from .github/workflows/database-tests.yml rename to .github/workflows/tests.yml index edec47d2f42..9765553b2bc 100644 --- a/.github/workflows/database-tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: database tests +name: Tests on: [push] @@ -17,7 +17,7 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - name: shutdown Ubuntu MySQL + - name: Shutdown Ubuntu MySQL run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - uses: mirromutth/mysql-action@v1.1 with: @@ -32,7 +32,7 @@ jobs: uses: actions/setup-java@v1 with: java-version: 13 - - name: checkout source + - name: Checkout source uses: actions/checkout@v1 with: depth: 1 @@ -45,12 +45,22 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- - - name: test (include PostgreSQL and update code coverage metrics) - run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); + - name: Run basic tests (without checkstyle) + run: xvfb-run --auto-servernum ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest + - name: Run fetcher tests + run: ./gradlew fetcherTest + - name: Run GUI tests + run: xvfb-run --auto-servernum ./gradlew guiTest + - name: Run tests on PostgreSQL + run: ./gradlew databaseTest env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} DBMS: "postgresql" - - name: test on MySQL + - name: Run tests on MySQL run: ./gradlew databaseTest env: DBMS: "mysql" + - name: Update test coverage metrics + run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + DBMS: "postgresql" From 187069610405814173e45915d3e6749ad0865855 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 10:45:23 +0100 Subject: [PATCH 15/38] Fix test names --- .github/workflows/checkstyle.yml | 3 ++- .github/workflows/tests.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml index eec5b0a3d7a..c2246392483 100644 --- a/.github/workflows/checkstyle.yml +++ b/.github/workflows/checkstyle.yml @@ -1,9 +1,10 @@ -name: checkstyle +name: Checkstyle on: [push] jobs: test: + name: Checkstyle runs-on: ubuntu-latest steps: - name: Checkout source diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9765553b2bc..3b6469ba749 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,7 +3,8 @@ name: Tests on: [push] jobs: - databasetest: + test: + name: Tests runs-on: ubuntu-latest services: postgres: From d3c78c5dbbbacbb90ab084fb0a226531c553ba7a Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:02:11 +0100 Subject: [PATCH 16/38] Finally remove usage of Travis - Add check-outdated-dependencies as workflow - Add formatting of test results (scripts/after-failure.sh) - Have fetcher tests and GUI tests run after the database tests (because they currently fail) --- .../workflows/check-outdated-dependencies.yml | 28 ++++++++++ .github/workflows/tests.yml | 25 +++++++-- .travis.yml | 54 ------------------- scripts/after-failure.sh | 6 +-- 4 files changed, 50 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/check-outdated-dependencies.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/check-outdated-dependencies.yml b/.github/workflows/check-outdated-dependencies.yml new file mode 100644 index 00000000000..816094eca07 --- /dev/null +++ b/.github/workflows/check-outdated-dependencies.yml @@ -0,0 +1,28 @@ +name: Check outdated Dependencies + +on: [push] + +jobs: + check: + name: Check outdated Dependencies + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - name: Run check + run: ./gradlew -q checkOutdatedDependencies diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b6469ba749..056fff48876 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,6 +33,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 13 + - name: Set up test result helpers + - run: sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc - name: Checkout source uses: actions/checkout@v1 with: @@ -48,20 +50,35 @@ jobs: ${{ runner.OS }}- - name: Run basic tests (without checkstyle) run: xvfb-run --auto-servernum ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest - - name: Run fetcher tests - run: ./gradlew fetcherTest - - name: Run GUI tests - run: xvfb-run --auto-servernum ./gradlew guiTest + - name: Format failed test results + if: failure() + run: scripts/after-failure.sh - name: Run tests on PostgreSQL run: ./gradlew databaseTest env: DBMS: "postgresql" + - name: Format failed test results + if: failure() + run: scripts/after-failure.sh - name: Run tests on MySQL run: ./gradlew databaseTest env: DBMS: "mysql" + - name: Format failed test results + if: failure() + run: scripts/after-failure.sh - name: Update test coverage metrics run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} DBMS: "postgresql" + - name: Run fetcher tests + run: ./gradlew fetcherTest + - name: Format failed test results + if: failure() + run: scripts/after-failure.sh + - name: Run GUI tests + run: xvfb-run --auto-servernum ./gradlew guiTest + - name: Format failed test results + if: failure() + run: scripts/after-failure.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 293c7a1f8b4..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,54 +0,0 @@ -language: java -jdk: - - openjdk13 - -# we test at Ubuntu Trusty (Ubuntu 14.04 LTS) -# see https://docs.travis-ci.com/user/trusty-ci-environment/ -# This environment is continuously updated as described in https://docs.travis-ci.com/user/build-environment-updates/ -dist: trusty -sudo: required - -git: - depth: 1 - -env: - global: - - GRADLE_OPTS=-Dorg.gradle.daemon=false - matrix: - - DEPENDENCY_UPDATES=check - -matrix: - fast_finish: true - allow_failures: - - env: DEPENDENCY_UPDATES=check - -# JavaFX localization tests need a running X environment -before_install: - - "export DISPLAY=:99.0" - - "sh -e /etc/init.d/xvfb start" - - sleep 3 # give xvfb some time to start - -install: true - -script: - - if [ "$DEPENDENCY_UPDATES" == "check" ]; then ./gradlew -q checkOutdatedDependencies; fi - -after_failure: - # show test results if build fails - - $TRAVIS_BUILD_DIR/scripts/after-failure.sh - -branches: - only: - - master - - maintable-beta - -# cache gradle dependencies -# https://docs.travis-ci.com/user/languages/java#Caching -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ -cache: - directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - diff --git a/scripts/after-failure.sh b/scripts/after-failure.sh index fa029d36898..4770f6c78de 100755 --- a/scripts/after-failure.sh +++ b/scripts/after-failure.sh @@ -1,12 +1,8 @@ #!/bin/bash -# taken from https://github.com/lhotari/travis-gradle-test-failures-to-console/blob/master/travis/junit-errors-to-stdout.sh +# based on https://github.com/lhotari/travis-gradle-test-failures-to-console/blob/master/travis/junit-errors-to-stdout.sh IFS=' ' DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -if [ "$TRAVIS" = "true" ]; then - #echo 'Installing xml-twig-tools and xsltproc....' - sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc > /dev/null -fi ROOTDIR="$1" if [ -z "$ROOTDIR" ]; then ROOTDIR="." From 3c859c6526c7e142380ffffd7ff20a5cbbeecdc2 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:04:29 +0100 Subject: [PATCH 17/38] Try to cache gradle wrapper --- .github/workflows/check-outdated-dependencies.yml | 5 +++++ .github/workflows/checkstyle.yml | 5 +++++ .github/workflows/deployment.yml | 5 +++++ .github/workflows/tests.yml | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/.github/workflows/check-outdated-dependencies.yml b/.github/workflows/check-outdated-dependencies.yml index 816094eca07..5ee796ed533 100644 --- a/.github/workflows/check-outdated-dependencies.yml +++ b/.github/workflows/check-outdated-dependencies.yml @@ -24,5 +24,10 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Cache gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run check run: ./gradlew -q checkOutdatedDependencies diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml index c2246392483..7a8ec970bea 100644 --- a/.github/workflows/checkstyle.yml +++ b/.github/workflows/checkstyle.yml @@ -24,5 +24,10 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Cache gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run checkstyle test run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 213d08d4d38..f24fac8c4a6 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -54,6 +54,11 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Cache gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Download jpackage # We need to download jpackage from https://jdk.java.net/jpackage/ run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 056fff48876..b5b50915a08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,6 +48,11 @@ jobs: ${{ runner.OS }}-gradle-${{ env.cache-name }}- ${{ runner.OS }}-gradle- ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Cache gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run basic tests (without checkstyle) run: xvfb-run --auto-servernum ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest - name: Format failed test results From f1b394a9232d5918f0eacea06ae5750d97567afa Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:05:23 +0100 Subject: [PATCH 18/38] Fix YML --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5b50915a08..6c8d6ebd4d5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: with: java-version: 13 - name: Set up test result helpers - - run: sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc + run: sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc - name: Checkout source uses: actions/checkout@v1 with: From 562e201dc97eb84ca6683481ffb50cee91ac9429 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:07:58 +0100 Subject: [PATCH 19/38] Run MySQL tests after code coverage (as it takes 15s to setup MySQL) --- .github/workflows/tests.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c8d6ebd4d5..92689540e73 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,17 +18,6 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - name: Shutdown Ubuntu MySQL - run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - - uses: mirromutth/mysql-action@v1.1 - with: - host port: 3800 - container port: 3307 - character set server: 'utf8' - collation server: 'utf8_general_ci' - mysql version: '8.0' - mysql database: 'jabref' - mysql root password: 'root' - name: Set up JDK uses: actions/setup-java@v1 with: @@ -65,6 +54,22 @@ jobs: - name: Format failed test results if: failure() run: scripts/after-failure.sh + - name: Update test coverage metrics + run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + DBMS: "postgresql" + - name: Shutdown Ubuntu MySQL + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - uses: mirromutth/mysql-action@v1.1 + with: + host port: 3800 + container port: 3307 + character set server: 'utf8' + collation server: 'utf8_general_ci' + mysql version: '8.0' + mysql database: 'jabref' + mysql root password: 'root' - name: Run tests on MySQL run: ./gradlew databaseTest env: @@ -72,11 +77,6 @@ jobs: - name: Format failed test results if: failure() run: scripts/after-failure.sh - - name: Update test coverage metrics - run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - DBMS: "postgresql" - name: Run fetcher tests run: ./gradlew fetcherTest - name: Format failed test results From 778a81825ce8319b56145c996b64dbc44abed21b Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:09:41 +0100 Subject: [PATCH 20/38] Update gradle from 6.0.0 to 6.0.1 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 6ce793f21e8..94920145f34 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.0.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 3fa119ff6620556dea468ce6c53c6196457b4f84 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:13:24 +0100 Subject: [PATCH 21/38] Add name to start MySQL task --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92689540e73..739950f5706 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,8 @@ jobs: DBMS: "postgresql" - name: Shutdown Ubuntu MySQL run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - - uses: mirromutth/mysql-action@v1.1 + - name: Start custom MySQL + uses: mirromutth/mysql-action@v1.1 with: host port: 3800 container port: 3307 From 931750523ae17bed2c5ca575ea0aff455f17e1ab Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:20:18 +0100 Subject: [PATCH 22/38] One "after failure" job is enough --- .github/workflows/tests.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 739950f5706..1b2e42fff09 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,8 +22,6 @@ jobs: uses: actions/setup-java@v1 with: java-version: 13 - - name: Set up test result helpers - run: sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc - name: Checkout source uses: actions/checkout@v1 with: @@ -44,16 +42,10 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run basic tests (without checkstyle) run: xvfb-run --auto-servernum ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest - - name: Format failed test results - if: failure() - run: scripts/after-failure.sh - name: Run tests on PostgreSQL run: ./gradlew databaseTest env: DBMS: "postgresql" - - name: Format failed test results - if: failure() - run: scripts/after-failure.sh - name: Update test coverage metrics run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); env: @@ -75,16 +67,12 @@ jobs: run: ./gradlew databaseTest env: DBMS: "mysql" - - name: Format failed test results - if: failure() - run: scripts/after-failure.sh - name: Run fetcher tests run: ./gradlew fetcherTest - - name: Format failed test results - if: failure() - run: scripts/after-failure.sh - name: Run GUI tests run: xvfb-run --auto-servernum ./gradlew guiTest - name: Format failed test results if: failure() - run: scripts/after-failure.sh + run: | + sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc + scripts/after-failure.sh From 72a1de0cc7066759d2a8f17bc3d009d977599425 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:41:58 +0100 Subject: [PATCH 23/38] Try to get better location of failed tests --- .github/workflows/tests.yml | 1 + build.gradle | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1b2e42fff09..91f279d109b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,3 +76,4 @@ jobs: run: | sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc scripts/after-failure.sh + ls -la reports/ diff --git a/build.gradle b/build.gradle index 37529b7ca9b..3928251269f 100644 --- a/build.gradle +++ b/build.gradle @@ -24,6 +24,9 @@ plugins { id 'org.javamodularity.moduleplugin' version '1.5.0' id 'org.openjfx.javafxplugin' version '0.0.8' id 'org.beryx.jlink' version '2.16.4' + + // nicer test outputs during running and completion + id 'com.adarshr.test-logger' version '2.0.0' } gradle.startParameter.showStacktrace = org.gradle.api.logging.configuration.ShowStacktrace.ALWAYS @@ -438,7 +441,6 @@ javadoc { localization.script = 'scripts/syncLang.py' -// Test tasks test { useJUnitPlatform { excludeTags 'DatabaseTest', 'FetcherTest', 'GUITest' @@ -496,6 +498,7 @@ task copyTestResources(type: Copy) { processTestResources.dependsOn copyTestResources tasks.withType(Test) { + reports.csv.destination = file("${reporting.baseDir}/${name}") reports.html.destination = file("${reporting.baseDir}/${name}") } @@ -524,8 +527,9 @@ jacocoTestReport { dependsOn jacocoMerge reports { - xml.enabled = true // coveralls plugin depends on xml format report + csv.enabled = true html.enabled = true + xml.enabled = true // coveralls plugin depends on xml format report } } From 9c858efb732afc9498a49b69551a2f570e5cc507 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:43:09 +0100 Subject: [PATCH 24/38] Update test coverage metrics in all cases (failed and non-failed) --- .github/workflows/tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 91f279d109b..5868af9d06c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,6 +51,12 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} DBMS: "postgresql" + - name: Update test coverage metrics (in case of failure) + run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); + if: failure() + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + DBMS: "postgresql" - name: Shutdown Ubuntu MySQL run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - name: Start custom MySQL From e091e71450ff604b250ca07270f1f9e1d7cabb04 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 11:48:12 +0100 Subject: [PATCH 25/38] only html available; try html2text --- .github/workflows/tests.yml | 2 ++ build.gradle | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5868af9d06c..14ab7ccd515 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -83,3 +83,5 @@ jobs: sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc scripts/after-failure.sh ls -la reports/ + sudo apt-get install -y html2text + html2text reports/*.html diff --git a/build.gradle b/build.gradle index 3928251269f..3c04a0e2688 100644 --- a/build.gradle +++ b/build.gradle @@ -498,7 +498,6 @@ task copyTestResources(type: Copy) { processTestResources.dependsOn copyTestResources tasks.withType(Test) { - reports.csv.destination = file("${reporting.baseDir}/${name}") reports.html.destination = file("${reporting.baseDir}/${name}") } From 88b5ab4eaa8833d071a4f336d00ee5653d49ed4b Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 12:09:08 +0100 Subject: [PATCH 26/38] html2text did not work out --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 14ab7ccd515..ce91888799b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -80,8 +80,5 @@ jobs: - name: Format failed test results if: failure() run: | - sudo apt-get install -qq -y --force-yes xml-twig-tools xsltproc + sudo apt-get install -qq -y xml-twig-tools xsltproc scripts/after-failure.sh - ls -la reports/ - sudo apt-get install -y html2text - html2text reports/*.html From b888f26d718c9eb2d1ca6988ec20d9a93d9a72f8 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 12:12:11 +0100 Subject: [PATCH 27/38] Switch test logger theme to a) mocha and b) mocha.parallel to enable (future) parallel test execution --- build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.gradle b/build.gradle index 3c04a0e2688..6a00eba1aaf 100644 --- a/build.gradle +++ b/build.gradle @@ -454,6 +454,10 @@ test { } } +testlogger { + theme 'mocha-parallel' +} + task databaseTest(type: Test) { useJUnitPlatform { includeTags 'DatabaseTest' From 0cbbaee176f117c418abdbe57990ee4a0ba2f686 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 12:16:56 +0100 Subject: [PATCH 28/38] Don't overoptimize (parallel) - just show skipped and failed tests --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6a00eba1aaf..194c1cd20a3 100644 --- a/build.gradle +++ b/build.gradle @@ -455,7 +455,7 @@ test { } testlogger { - theme 'mocha-parallel' + showPassed false } task databaseTest(type: Test) { From 382fd97c43194dbb68e8e598b8130b40e3e62f46 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 12:27:12 +0100 Subject: [PATCH 29/38] Run outdated dependency check weekly --- .../workflows/check-outdated-dependencies.yml | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/check-outdated-dependencies.yml b/.github/workflows/check-outdated-dependencies.yml index 5ee796ed533..1d3fe5a7750 100644 --- a/.github/workflows/check-outdated-dependencies.yml +++ b/.github/workflows/check-outdated-dependencies.yml @@ -1,10 +1,12 @@ -name: Check outdated Dependencies +name: Check dependencies -on: [push] +on: + schedule: + - cron: '0 0 * * 0' # Run every Sunday jobs: - check: - name: Check outdated Dependencies + checkDependencies: + name: Check dependencies runs-on: ubuntu-latest steps: - name: Checkout source @@ -16,18 +18,12 @@ jobs: uses: actions/setup-java@v1 with: java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - uses: actions/cache@v1 - name: Cache gradle wrapper - with: - path: ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - - name: Run check + - name: Look for outdated dependencies run: ./gradlew -q checkOutdatedDependencies + - name: Report issues + if: failure() + uses: JasonEtco/create-an-issue@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + filename: .github/outdatedDependencies.md From f30066e49b2c47fe7ab2f98e076e0b951ec95e75 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 12:31:34 +0100 Subject: [PATCH 30/38] Create outdatedDependencies.md --- .github/outdatedDependencies.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/outdatedDependencies.md diff --git a/.github/outdatedDependencies.md b/.github/outdatedDependencies.md new file mode 100644 index 00000000000..e08f45ef495 --- /dev/null +++ b/.github/outdatedDependencies.md @@ -0,0 +1,6 @@ +--- +title: Outdated dependencies +labels: code-quality, dependencies +--- +There are outdated dependencies! +See https://github.com/JabRef/jabref/actions?query=is%3Afailure for details. From 0446759896740bfec00f84a7f9cd5e0d4b94da2e Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 12:38:12 +0100 Subject: [PATCH 31/38] Introduce TravisCI environment variable CI --- .github/workflows/tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce91888799b..125647b4b78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,19 +42,24 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run basic tests (without checkstyle) run: xvfb-run --auto-servernum ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest + env: + CI: "true" - name: Run tests on PostgreSQL run: ./gradlew databaseTest env: + CI: "true" DBMS: "postgresql" - name: Update test coverage metrics run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); env: + CI: "true" CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} DBMS: "postgresql" - name: Update test coverage metrics (in case of failure) run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); if: failure() env: + CI: "false" # we pretend to run locally - even if tests fail on the CI, they count towards test coverage CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} DBMS: "postgresql" - name: Shutdown Ubuntu MySQL @@ -72,11 +77,16 @@ jobs: - name: Run tests on MySQL run: ./gradlew databaseTest env: + CI: "true" DBMS: "mysql" - name: Run fetcher tests run: ./gradlew fetcherTest + env: + CI: "true" - name: Run GUI tests run: xvfb-run --auto-servernum ./gradlew guiTest + env: + CI: "true" - name: Format failed test results if: failure() run: | From ae2a0abed40c391c68bc70a04df89e7485f5a017 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 12:43:30 +0100 Subject: [PATCH 32/38] Include checkstyle in main test workflow --- .github/workflows/tests.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 125647b4b78..da27870d789 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,6 +3,34 @@ name: Tests on: [push] jobs: + checkstyle: + name: Checkstyle + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Cache gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} + - name: Run checkstyle + run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh test: name: Tests runs-on: ubuntu-latest From 99de01d8f18afd3553453ab9ab732a6c881e528b Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 12:43:51 +0100 Subject: [PATCH 33/38] Delete checkstyle.yml --- .github/workflows/checkstyle.yml | 33 -------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .github/workflows/checkstyle.yml diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml deleted file mode 100644 index 7a8ec970bea..00000000000 --- a/.github/workflows/checkstyle.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Checkstyle - -on: [push] - -jobs: - test: - name: Checkstyle - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - uses: actions/cache@v1 - name: Cache gradle wrapper - with: - path: ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - - name: Run checkstyle test - run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh From 02fe4fc94ce7a1defc1e9ddb46a765655b0a0c2c Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 12:52:59 +0100 Subject: [PATCH 34/38] Fix ordering of jacoco call --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da27870d789..62f701d905d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,15 +77,15 @@ jobs: env: CI: "true" DBMS: "postgresql" - - name: Update test coverage metrics + - name: Update test coverage metrics (in case of failure) run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); + if: failure() env: - CI: "true" + CI: "false" # we pretend to run locally - even if tests fail on the CI, they count towards test coverage CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} DBMS: "postgresql" - - name: Update test coverage metrics (in case of failure) + - name: Update test coverage metrics (in case of success) run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); - if: failure() env: CI: "false" # we pretend to run locally - even if tests fail on the CI, they count towards test coverage CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From a0021edb1d998d268da1bdd93db1ca184ec840d3 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Fri, 29 Nov 2019 12:58:16 +0100 Subject: [PATCH 35/38] Try to find out why fetcher tests are not executed on GitHub workflows --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 62f701d905d..5ed0686022f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,6 +111,10 @@ jobs: run: ./gradlew fetcherTest env: CI: "true" + - name: Run fetcher tests + run: ./gradlew fetcherTest + env: + CI: "false" - name: Run GUI tests run: xvfb-run --auto-servernum ./gradlew guiTest env: From 11bb9b9669bc3de3ba4f494d1cd86d3485c3cc08 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 13:11:15 +0100 Subject: [PATCH 36/38] Split tests into seperate jobs to parallelize --- .github/workflows/tests.yml | 223 +++++++++++++++++++++++++++--------- 1 file changed, 167 insertions(+), 56 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ed0686022f..e5637aa4167 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,32 +7,69 @@ jobs: name: Checkstyle runs-on: ubuntu-latest steps: - - name: Checkout source - uses: actions/checkout@v1 - with: - depth: 1 - submodules: false - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.OS }}-gradle-${{ env.cache-name }}- - ${{ runner.OS }}-gradle- - ${{ runner.OS }}- - - uses: actions/cache@v1 - name: Cache gradle wrapper - with: - path: ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - - name: Run checkstyle - run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh - test: - name: Tests + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + name: Restore gradle chache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Restore gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} + - name: Run checkstyle + run: ./gradlew checkstyleMain checkstyleTest checkstyleJmh + tests: + name: Unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + name: Restore gradle chache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Restore gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} + - name: Run tests + run: xvfb-run --auto-servernum ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest + env: + CI: "true" + - name: Format failed test results + if: failure() + run: | + sudo apt-get install -qq -y xml-twig-tools xsltproc + scripts/after-failure.sh + databasetests: + name: Database tests runs-on: ubuntu-latest services: postgres: @@ -46,16 +83,17 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 13 - name: Checkout source uses: actions/checkout@v1 with: depth: 1 submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 - uses: actions/cache@v1 + name: Restore gradle chache with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} @@ -64,32 +102,15 @@ jobs: ${{ runner.OS }}-gradle- ${{ runner.OS }}- - uses: actions/cache@v1 - name: Cache gradle wrapper + name: Restore gradle wrapper with: path: ~/.gradle/wrapper key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - - name: Run basic tests (without checkstyle) - run: xvfb-run --auto-servernum ./gradlew check -x checkstyleJmh -x checkstyleMain -x checkstyleTest - env: - CI: "true" - name: Run tests on PostgreSQL run: ./gradlew databaseTest env: CI: "true" DBMS: "postgresql" - - name: Update test coverage metrics (in case of failure) - run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); - if: failure() - env: - CI: "false" # we pretend to run locally - even if tests fail on the CI, they count towards test coverage - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - DBMS: "postgresql" - - name: Update test coverage metrics (in case of success) - run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); - env: - CI: "false" # we pretend to run locally - even if tests fail on the CI, they count towards test coverage - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - DBMS: "postgresql" - name: Shutdown Ubuntu MySQL run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - name: Start custom MySQL @@ -107,20 +128,110 @@ jobs: env: CI: "true" DBMS: "mysql" + fetchertests: + name: Fetcher tests + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + name: Restore gradle chache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Restore gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run fetcher tests run: ./gradlew fetcherTest env: CI: "true" - - name: Run fetcher tests - run: ./gradlew fetcherTest - env: - CI: "false" + guitests: + name: GUI tests + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + name: Restore gradle chache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Restore gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run GUI tests run: xvfb-run --auto-servernum ./gradlew guiTest env: CI: "true" - - name: Format failed test results + codecoverage: + name: Code coverage + runs-on: ubuntu-latest + services: + postgres: + image: postgres:10.8 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + depth: 1 + submodules: false + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 13 + - uses: actions/cache@v1 + name: Restore gradle chache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.OS }}-gradle-${{ env.cache-name }}- + ${{ runner.OS }}-gradle- + ${{ runner.OS }}- + - uses: actions/cache@v1 + name: Restore gradle wrapper + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} + - name: Update test coverage metrics + run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); if: failure() - run: | - sudo apt-get install -qq -y xml-twig-tools xsltproc - scripts/after-failure.sh + env: + CI: "false" # we pretend to run locally - even if tests fail on the CI, they count towards test coverage + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + DBMS: "postgresql" From 26fe33eedd705caee89d668f316beced9a01797c Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 13:18:15 +0100 Subject: [PATCH 37/38] Make sure all tests are executed --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5637aa4167..43efe5bd0dc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -107,7 +107,7 @@ jobs: path: ~/.gradle/wrapper key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Run tests on PostgreSQL - run: ./gradlew databaseTest + run: ./gradlew databaseTest --rerun-tasks env: CI: "true" DBMS: "postgresql" @@ -230,7 +230,6 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - name: Update test coverage metrics run: xvfb-run --auto-servernum ./gradlew jacocoTestReport && bash <(curl -s https://codecov.io/bash); - if: failure() env: CI: "false" # we pretend to run locally - even if tests fail on the CI, they count towards test coverage CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From 324f822046cadb165358eab18792ffcb4e9f1b9e Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 13:24:29 +0100 Subject: [PATCH 38/38] Make mysql tests run --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 43efe5bd0dc..3d7fc778686 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -124,7 +124,7 @@ jobs: mysql database: 'jabref' mysql root password: 'root' - name: Run tests on MySQL - run: ./gradlew databaseTest + run: ./gradlew databaseTest --rerun-tasks env: CI: "true" DBMS: "mysql"