From b754d071507f8af11be06e92b83dbc40fedf7f1f Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Tue, 8 Sep 2020 13:44:39 +0200 Subject: [PATCH 01/29] Rework of keyfile download and caching. * Supports interop * More modular for better testing and build flavour based behavior adjustments * More resilient handling of failed downloads * Preperations for future hourly download and serverside checksums TODO: Finish unit tests, keycache migration and cache health check --- .../1.json | 76 +++++ .../1.json | 82 +++++ .../storage/keycache/KeyCacheDaoTest.kt | 2 + .../util/security/DBPasswordTest.kt | 2 +- .../RiskLevelAndKeyRetrievalBenchmark.kt | 7 +- .../TestForAPIFragment.kt | 8 +- .../diagnosiskeys/DiagnosisKeysModule.kt | 68 +++++ .../diagnosiskeys/download/CountryData.kt | 67 +++++ .../download/KeyFileDownloader.kt | 264 ++++++++++++++++ .../diagnosiskeys/server/DownloadApiV1.kt | 41 +++ .../server/DownloadHomeCountry.kt | 8 + .../server/DownloadHttpClient.kt | 8 + .../diagnosiskeys/server/DownloadServer.kt | 140 +++++++++ .../diagnosiskeys/server/DownloadServerUrl.kt | 8 + .../diagnosiskeys/server/LocationCode.kt | 5 + .../diagnosiskeys/storage/CachedKeyFile.kt | 86 ++++++ .../diagnosiskeys/storage/KeyCacheDatabase.kt | 64 ++++ .../storage/KeyCacheRepository.kt | 135 +++++++++ .../storage/legacy}/KeyCacheDao.kt | 2 +- .../storage/legacy}/KeyCacheEntity.kt | 2 +- .../coronawarnapp/http/HttpClientDefault.kt | 8 + .../de/rki/coronawarnapp/http/HttpModule.kt | 57 ++++ .../rki/coronawarnapp/http/ServiceFactory.kt | 214 ++++--------- .../coronawarnapp/http/WebRequestBuilder.kt | 108 +------ .../http/service/DistributionService.kt | 22 -- .../ApplicationConfigurationService.kt | 6 +- .../diagnosiskey/DiagnosisKeyConstants.kt | 44 --- .../rki/coronawarnapp/storage/AppDatabase.kt | 14 +- .../storage/keycache/KeyCacheRepository.kt | 95 ------ .../RetrieveDiagnosisKeysTransaction.kt | 20 +- .../coronawarnapp/util/CachedKeyFileHolder.kt | 283 ------------------ .../rki/coronawarnapp/util/HashExtensions.kt | 40 +++ .../de/rki/coronawarnapp/util/TimeStamper.kt | 13 + .../util/database/CommonConverters.kt | 81 +++++ .../util/debug/TimeMeasurement.kt | 7 + .../util/di/ApplicationComponent.kt | 16 +- .../util/security/VerificationKeys.kt | 5 +- .../diagnosiskeys/download/CountryDataTest.kt | 74 +++++ .../download/KeyFileDownloaderTest.kt | 162 ++++++++++ .../diagnosiskeys/server/CDNModuleTest.kt | 18 ++ .../diagnosiskeys/server/CDNServerTest.kt | 37 +++ .../storage/CachedKeyFileTest.kt | 27 ++ .../storage/KeyCacheRepositoryTest.kt | 142 +++++++++ .../http/WebRequestBuilderTest.kt | 35 ++- .../diagnosiskey/DiagnosisKeyConstantsTest.kt | 10 - .../keycache/KeyCacheRepositoryTest.kt | 88 ------ .../RetrieveDiagnosisKeysTransactionTest.kt | 5 - .../util/CachedKeyFileHolderTest.kt | 126 -------- .../coronawarnapp/util/HashExtensionsTest.kt | 60 ++++ .../coronawarnapp/util/MockWebServerUtil.kt | 7 +- .../util/database/CommonConvertersTest.kt} | 48 ++- 51 files changed, 1952 insertions(+), 995 deletions(-) create mode 100644 Corona-Warn-App/schemas/de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheDatabase/1.json create mode 100644 Corona-Warn-App/schemas/de.rki.coronawarnapp.storage.keycache.KeyCacheDatabase/1.json create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHomeCountry.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHttpClient.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerUrl.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFile.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt rename Corona-Warn-App/src/main/java/de/rki/coronawarnapp/{storage/keycache => diagnosiskeys/storage/legacy}/KeyCacheDao.kt (97%) rename Corona-Warn-App/src/main/java/de/rki/coronawarnapp/{storage/keycache => diagnosiskeys/storage/legacy}/KeyCacheEntity.kt (96%) create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpClientDefault.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt delete mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/service/DistributionService.kt delete mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt delete mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/database/CommonConverters.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/debug/TimeMeasurement.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt delete mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepositoryTest.kt delete mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/HashExtensionsTest.kt rename Corona-Warn-App/src/{main/java/de/rki/coronawarnapp/util/Converters.kt => test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt} (72%) diff --git a/Corona-Warn-App/schemas/de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheDatabase/1.json b/Corona-Warn-App/schemas/de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheDatabase/1.json new file mode 100644 index 00000000000..72e78dde4ed --- /dev/null +++ b/Corona-Warn-App/schemas/de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheDatabase/1.json @@ -0,0 +1,76 @@ +{ + "formatVersion": 1, + "database": { + "version": 1, + "identityHash": "c4ef5f7d4d9672d11c8eb97a63d4a3c5", + "entities": [ + { + "tableName": "keyfiles", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `type` TEXT NOT NULL, `location` TEXT NOT NULL, `day` TEXT NOT NULL, `hour` TEXT, `createdAt` TEXT NOT NULL, `checksumMD5` TEXT, `completed` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "location", + "columnName": "location", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "day", + "columnName": "day", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "hour", + "columnName": "hour", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "checksumMD5", + "columnName": "checksumMD5", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "isDownloadComplete", + "columnName": "completed", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'c4ef5f7d4d9672d11c8eb97a63d4a3c5')" + ] + } +} \ No newline at end of file diff --git a/Corona-Warn-App/schemas/de.rki.coronawarnapp.storage.keycache.KeyCacheDatabase/1.json b/Corona-Warn-App/schemas/de.rki.coronawarnapp.storage.keycache.KeyCacheDatabase/1.json new file mode 100644 index 00000000000..9d856a5a015 --- /dev/null +++ b/Corona-Warn-App/schemas/de.rki.coronawarnapp.storage.keycache.KeyCacheDatabase/1.json @@ -0,0 +1,82 @@ +{ + "formatVersion": 1, + "database": { + "version": 1, + "identityHash": "03f6e8cba631c8ef60e506006913a1ad", + "entities": [ + { + "tableName": "keyfiles", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `type` TEXT NOT NULL, `location` TEXT NOT NULL, `day` TEXT NOT NULL, `hour` TEXT, `sourceUrl` TEXT NOT NULL, `createdAt` TEXT NOT NULL, `checksumMD5` TEXT, `completed` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "location", + "columnName": "location", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "day", + "columnName": "day", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "hour", + "columnName": "hour", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "sourceUrl", + "columnName": "sourceUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "checksumMD5", + "columnName": "checksumMD5", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "isDownloadComplete", + "columnName": "completed", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '03f6e8cba631c8ef60e506006913a1ad')" + ] + } +} \ No newline at end of file diff --git a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt index 31f173f6c73..daffbb8e9fe 100644 --- a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt +++ b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt @@ -5,6 +5,8 @@ import androidx.room.Room import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import com.google.common.truth.Truth.assertThat +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheDao +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheEntity import de.rki.coronawarnapp.storage.AppDatabase import kotlinx.coroutines.runBlocking import org.junit.After diff --git a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt index 5f1267524ea..3b06993c9b5 100644 --- a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt +++ b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt @@ -21,8 +21,8 @@ package de.rki.coronawarnapp.util.security import android.content.Context import androidx.test.core.app.ApplicationProvider +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheEntity import de.rki.coronawarnapp.storage.AppDatabase -import de.rki.coronawarnapp.storage.keycache.KeyCacheEntity import kotlinx.coroutines.runBlocking import net.sqlcipher.database.SQLiteException import org.hamcrest.Matchers.equalTo diff --git a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/RiskLevelAndKeyRetrievalBenchmark.kt b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/RiskLevelAndKeyRetrievalBenchmark.kt index 85b980a36e0..e92c287573f 100644 --- a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/RiskLevelAndKeyRetrievalBenchmark.kt +++ b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/RiskLevelAndKeyRetrievalBenchmark.kt @@ -5,9 +5,9 @@ import android.text.format.Formatter import de.rki.coronawarnapp.exception.ExceptionCategory import de.rki.coronawarnapp.exception.TransactionException import de.rki.coronawarnapp.exception.reporting.report -import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction import de.rki.coronawarnapp.transaction.RiskLevelTransaction +import de.rki.coronawarnapp.util.di.AppInjector import timber.log.Timber import kotlin.system.measureTimeMillis @@ -19,8 +19,7 @@ class RiskLevelAndKeyRetrievalBenchmark( /** * the key cache instance used to store queried dates and hours */ - private val keyCache = - KeyCacheRepository.getDateRepository(context) + private val keyCache = AppInjector.component.keyCacheRepository /** * Calls the RetrieveDiagnosisKeysTransaction and RiskLevelTransaction and measures them. @@ -36,7 +35,7 @@ class RiskLevelAndKeyRetrievalBenchmark( var resultInfo = StringBuilder() .append( "MEASUREMENT Running for Countries:\n " + - "${countries?.joinToString(", ")}\n\n" + "${countries.joinToString(", ")}\n\n" ) .append("Result: \n\n") .append("#\t Combined \t Download \t Sub \t Risk \t File # \t F. size\n") diff --git a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt index ac579527077..ed9e923bafb 100644 --- a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt +++ b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt @@ -50,17 +50,17 @@ import de.rki.coronawarnapp.storage.LocalData import de.rki.coronawarnapp.storage.tracing.TracingIntervalRepository import de.rki.coronawarnapp.transaction.RiskLevelTransaction import de.rki.coronawarnapp.ui.viewmodel.TracingViewModel -import de.rki.coronawarnapp.util.CachedKeyFileHolder import de.rki.coronawarnapp.util.KeyFileHelper +import de.rki.coronawarnapp.util.di.AppInjector import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.joda.time.DateTime import org.joda.time.DateTimeZone +import org.joda.time.LocalDate import timber.log.Timber import java.io.File import java.lang.reflect.Type -import java.util.Date import java.util.UUID @SuppressWarnings("TooManyFunctions", "MagicNumber", "LongMethod") @@ -321,9 +321,9 @@ class TestForAPIFragment : Fragment(), InternalExposureNotificationPermissionHel lastSetCountries = countryCodes // Trigger asyncFetchFiles which will use all Countries passed as parameter - val currentDate = Date(System.currentTimeMillis()) + val currentDate = LocalDate.now() lifecycleScope.launch { - CachedKeyFileHolder.asyncFetchFiles(currentDate, countryCodes) + AppInjector.component.keyFileDownloader.asyncFetchFiles(currentDate, countryCodes) updateCountryStatusLabel() } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt new file mode 100644 index 00000000000..1e1ddd71329 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt @@ -0,0 +1,68 @@ +package de.rki.coronawarnapp.diagnosiskeys + +import android.webkit.URLUtil +import dagger.Module +import dagger.Provides +import dagger.Reusable +import de.rki.coronawarnapp.BuildConfig +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadApiV1 +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadHomeCountry +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadHttpClient +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServerUrl +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.http.HttpClientDefault +import okhttp3.ConnectionSpec +import okhttp3.OkHttpClient +import okhttp3.TlsVersion +import retrofit2.Retrofit +import retrofit2.converter.gson.GsonConverterFactory +import javax.inject.Singleton + +@Module +class DiagnosisKeysModule { + + @Singleton + @DownloadHomeCountry + @Provides + fun provideCDNHomeCountry(): LocationCode = LocationCode("DE") + + @Reusable + @DownloadHttpClient + @Provides + fun cdnHttpClient(@HttpClientDefault defaultHttpClient: OkHttpClient): OkHttpClient = + defaultHttpClient.newBuilder().connectionSpecs(CDN_CONNECTION_SPECS).build() + + @Singleton + @Provides + fun cdnApi( + @DownloadHttpClient cdnHttpClient: OkHttpClient, + @DownloadServerUrl cdnUrl: String, + gsonConverterFactory: GsonConverterFactory + ): DownloadApiV1 = Retrofit.Builder() + .client(cdnHttpClient) + .baseUrl(cdnUrl) + .addConverterFactory(gsonConverterFactory) + .build() + .create(DownloadApiV1::class.java) + + @Singleton + @DownloadServerUrl + @Provides + fun provideCDNUrl(): String = BuildConfig.DOWNLOAD_CDN_URL.also { + if (!URLUtil.isHttpsUrl(it)) throw IllegalArgumentException("the url is invalid") + } + + companion object { + private val CDN_CONNECTION_SPECS = listOf( + ConnectionSpec.Builder(ConnectionSpec.COMPATIBLE_TLS) + .tlsVersions( + TlsVersion.TLS_1_0, + TlsVersion.TLS_1_1, + TlsVersion.TLS_1_2, + TlsVersion.TLS_1_3 + ) + .allEnabledCipherSuites() + .build() + ) + } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt new file mode 100644 index 00000000000..551f3be985e --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt @@ -0,0 +1,67 @@ +package de.rki.coronawarnapp.diagnosiskeys.download + +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyFile +import org.joda.time.LocalDate +import org.joda.time.LocalTime + +sealed class CountryData { + + abstract val country: LocationCode + +} + +internal data class CountryDays( + override val country: LocationCode, + val dayData: Collection +) : CountryData() { + + /** + * Return a filtered list that contains all dates which are part of this wrapper, but not in the parameter. + */ + fun getMissingDays(cachedKeys: List): Collection? { + val cachedCountryDates = cachedKeys + .filter { it.location == country } + .map { it.day } + + return dayData.filter { date -> + !cachedCountryDates.contains(date) + } + } + + /** + * Create a new country object that only contains those elements, + * that are part of this wrapper, but not in the cache. + */ + fun toMissingDays(cachedKeys: List): CountryDays? { + val missingDays = this.getMissingDays(cachedKeys) + if (missingDays == null || missingDays.isEmpty()) return null + + return CountryDays(this.country, missingDays) + } +} + +internal data class CountryHours( + override val country: LocationCode, + val hourData: Map> +) : CountryData() { + + fun getMissingHours(cachedKeys: List): Map>? { + val cachedHours = cachedKeys + .filter { it.location == country } + + return hourData.mapNotNull { (day, dayHours) -> + val missingHours = dayHours.filter { hour -> + cachedHours.none { it.day == day && it.hour == hour } + } + if (missingHours.isEmpty()) null else day to missingHours + }.toMap() + } + + fun toMissingHours(cachedKeys: List): CountryHours? { + val missingHours = this.getMissingHours(cachedKeys) + if (missingHours == null || missingHours.isEmpty()) return null + + return CountryHours(this.country, missingHours) + } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt new file mode 100644 index 00000000000..24517d9394a --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -0,0 +1,264 @@ +/****************************************************************************** + * Corona-Warn-App * + * * + * SAP SE and all other contributors / * + * copyright owners license this file to you under the Apache * + * License, Version 2.0 (the "License"); you may not use this * + * file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ******************************************************************************/ + +package de.rki.coronawarnapp.diagnosiskeys.download + +import dagger.Reusable +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyFile +import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import de.rki.coronawarnapp.storage.FileStorageHelper +import de.rki.coronawarnapp.storage.LocalData +import de.rki.coronawarnapp.util.CWADebug +import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 +import de.rki.coronawarnapp.util.TimeAndDateExtensions +import de.rki.coronawarnapp.util.debug.measureTimeMillisWithResult +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.withContext +import org.joda.time.LocalDate +import timber.log.Timber +import java.io.File +import javax.inject.Inject + +/** + * Downloads new or missing key files from the CDN + */ +@Reusable +class KeyFileDownloader @Inject constructor( + private val downloadServer: DownloadServer, + private val keyCache: KeyCacheRepository +) { + + /** + * Fetches all necessary Files from the Cached KeyFile Entries out of the [KeyCacheRepository] and + * adds to that all open Files currently available from the Server. + * + * Assumptions made about the implementation: + * - the app initializes with an empty cache and draws in every available data set in the beginning + * - the difference can only work properly if the date from the device is synchronized through the net + * - the difference in timezone is taken into account by using UTC in the Conversion from the Date to Server format + * - the missing days and hours are stored in one table as the actual stored data amount is low + * - the underlying repository from the database has no error and is reliable as source of truth + * + * @param currentDate the current date - if this is adjusted by the calendar, the cache is affected. + * @return list of all files from both the cache and the diff query + */ + suspend fun asyncFetchFiles( + currentDate: LocalDate, + countries: List + ): List = withContext(Dispatchers.IO) { + // Initiate key-cache folder needed for saving downloaded key files + FileStorageHelper.initializeExportSubDirectory() // TODO replace + + checkForFreeSpace() // TODO replace + + val availableCountries = downloadServer.getCountryIndex(countries) + Timber.tag(TAG).v("Available server data: %s", availableCountries) + + if (CWADebug.isDebugBuildOrMode && LocalData.last3HoursMode()) { + asyncHandleLast3HoursFilesFetch(currentDate, availableCountries) + } else { + asyncHandleFilesFetch(availableCountries) + } + } + + // TODO replace + private fun checkForFreeSpace() = FileStorageHelper.checkFileStorageFreeSpace() + + /** + * Fetches files given by serverDates by respecting countries + * @param availableCountries pair of dates per country code + */ + private suspend fun asyncHandleFilesFetch( + availableCountries: List + ): List = withContext(Dispatchers.IO) { + val availableCountriesWithDays = availableCountries.map { + val days = downloadServer.getDayIndex(it) + CountryDays(it, days) + } + + val cachedDays = keyCache + .getEntriesForType(CachedKeyFile.Type.COUNTRY_DAY) + .filter { it.isDownloadComplete } // We overwrite not completed ones + + // All cached files that are no longer on the server are considered stale + val staleKeyFiles = cachedDays.filter { cachedKeyFile -> + val availableCountry = availableCountriesWithDays.singleOrNull { + it.country == cachedKeyFile.location + } + if (availableCountry == null) { + Timber.tag(TAG) + .w( + "Unknown location %s, assuming stale cache.", + cachedKeyFile.location + ) + return@filter true // It's stale + } + + availableCountry.dayData.none { date -> + cachedKeyFile.day == date + } + } + if (staleKeyFiles.isNotEmpty()) keyCache.delete(staleKeyFiles) + + val nonStaleDays = cachedDays.minus(staleKeyFiles) + val countriesWithMissingDays = availableCountriesWithDays.mapNotNull { + it.toMissingDays(nonStaleDays) + } + Timber.tag(TAG).d("Downloading missing days: %s", countriesWithMissingDays) + val batchDownloadStart = System.currentTimeMillis() + val dayDownloads = countriesWithMissingDays + .flatMap { country -> + country.dayData.map { dayDate -> country to dayDate } + } + .map { (countryWrapper, dayDate) -> + async { + val (keyInfo, path) = keyCache.createCacheEntry( + location = countryWrapper.country, + dayIdentifier = dayDate, + hourIdentifier = null, + type = CachedKeyFile.Type.COUNTRY_DAY + ) + + return@async try { + downloadKeyFile(keyInfo, path) + keyInfo to path + } catch (e: Exception) { + Timber.tag(TAG).e(e, "Download failed: %s", keyInfo) + null + } + } + } + + Timber.tag(TAG).d("Waiting for %d missing day downloads.", dayDownloads.size) + // execute the query plan + val downloadedDays = dayDownloads.awaitAll().filterNotNull() + + Timber.tag(TAG).d( + "Batch download (%d files) finished in %dms", + dayDownloads.size, + (System.currentTimeMillis() - batchDownloadStart) + ) + + return@withContext downloadedDays.map { (keyInfo, path) -> + Timber.tag(TAG).v("Downloaded keyfile: %s to %s", keyInfo, path) + path + } + } + + /** + * Fetches files given by serverDates by respecting countries + * @param currentDate base for where only dates within 3 hours before will be fetched + * @param availableCountries pair of dates per country code + */ + private suspend fun asyncHandleLast3HoursFilesFetch( + currentDate: LocalDate, + availableCountries: List + ): List = withContext(Dispatchers.IO) { + Timber.tag(TAG).v( + "asyncHandleLast3HoursFilesFetch(currentDate=%s, availableCountries=%s)", + currentDate, availableCountries + ) + + // This is currently used for debugging, so we only fetch 3 hours + val availableHours = availableCountries.map { + val hoursForDate = downloadServer.getHourIndex(it, currentDate).filter { availHour -> + TimeAndDateExtensions.getCurrentHourUTC() - 3 <= availHour.hourOfDay + } + CountryHours(it, mapOf(currentDate to hoursForDate)) + } + + val cachedHours = keyCache + .getEntriesForType(CachedKeyFile.Type.COUNTRY_HOUR) + .filter { it.isDownloadComplete } // We overwrite not completed ones + + // All cached files that are no longer on the server are considered stale + val staleHours = cachedHours.filter { cachedHour -> + val availCountry = availableHours.singleOrNull { + it.country == cachedHour.location + } + if (availCountry == null) { + Timber.w("Unknown location %s, assuming stale.", cachedHour.location) + return@filter true // It's stale + } + + val availableDay = availCountry.hourData.get(currentDate) + if (availableDay == null) { + Timber.d("Unknown day %s, assuming stale.", cachedHour.location) + return@filter true // It's stale + } + + availableDay.none { time -> + cachedHour.hour == time + } + } + if (staleHours.isNotEmpty()) keyCache.delete(staleHours) + + val nonStaleHours = cachedHours.minus(staleHours) + val missingHours = availableHours.mapNotNull { + it.toMissingHours(nonStaleHours) + } + Timber.tag(TAG).d("Downloading missing hours: %s", missingHours) + + val hourDownloads = missingHours.flatMap { country -> + country.hourData.flatMap { (day, missingHours) -> + missingHours.map { missingHour -> + async { + val (keyInfo, path) = keyCache.createCacheEntry( + location = country.country, + dayIdentifier = day, + hourIdentifier = missingHour, + type = CachedKeyFile.Type.COUNTRY_HOUR + ) + + downloadKeyFile(keyInfo, path) + + return@async keyInfo to path + } + } + } + } + + Timber.tag(TAG).d("Waiting for %d missing hour downloads.", hourDownloads.size) + val downloadedHours = hourDownloads.awaitAll() + + return@withContext downloadedHours.map { (keyInfo, path) -> + Timber.tag(TAG).d("Downloaded keyfile: %s to %s", keyInfo, path) + path + } + } + + private suspend fun downloadKeyFile(keyInfo: CachedKeyFile, path: File) { + downloadServer.downloadKeyFile(keyInfo.location, keyInfo.day, keyInfo.hour, path) + Timber.tag(TAG).v("Dowwnload finished: %s -> %s", keyInfo, path) + + val (downloadedMD5, duration) = measureTimeMillisWithResult { path.hashToMD5() } + Timber.tag(TAG).v("Hashed to MD5 in %dms: %s", duration, path) + + keyCache.markKeyComplete(keyInfo, downloadedMD5) + } + + companion object { + + private val TAG: String? = KeyFileDownloader::class.simpleName + } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt new file mode 100644 index 00000000000..7281c15455a --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt @@ -0,0 +1,41 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import okhttp3.ResponseBody +import retrofit2.http.GET +import retrofit2.http.Path +import retrofit2.http.Streaming + +interface DownloadApiV1 { + + @GET("/version/v1/configuration/country/{country}/app_config") + suspend fun getApplicationConfiguration(@Path("country") country: String): ResponseBody + + @GET("/version/v1/diagnosis-keys/country") + suspend fun getCountryIndex(): List // TODO Let retrofit format this to CountryCode + + @GET("/version/v1/diagnosis-keys/country/{country}/date") + suspend fun getDayIndex(@Path("country") country: String): List // TODO Let retrofit format this to LocalDate + + + @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}/hour") + suspend fun getHourIndex( + @Path("country") country: String, + @Path("day") day: String + ): List // TODO Let retrofit format this to LocalTime + + @Streaming + @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}") + suspend fun downloadKeyFileForDay( + @Path("country") country: String, + @Path("day") day: String + ): ResponseBody + + @Streaming + @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}/hour/{hour}") + suspend fun downloadKeyFileForHour( + @Path("country") country: String, + @Path("day") day: String, + @Path("hour") hour: String + ): ResponseBody + +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHomeCountry.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHomeCountry.kt new file mode 100644 index 00000000000..69aa0f792cb --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHomeCountry.kt @@ -0,0 +1,8 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import javax.inject.Qualifier + +@Qualifier +@MustBeDocumented +@Retention(AnnotationRetention.RUNTIME) +annotation class DownloadHomeCountry diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHttpClient.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHttpClient.kt new file mode 100644 index 00000000000..f9536135c97 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadHttpClient.kt @@ -0,0 +1,8 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import javax.inject.Qualifier + +@Qualifier +@MustBeDocumented +@Retention(AnnotationRetention.RUNTIME) +annotation class DownloadHttpClient diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt new file mode 100644 index 00000000000..59602d4fdc2 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt @@ -0,0 +1,140 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import com.google.protobuf.InvalidProtocolBufferException +import dagger.Lazy +import de.rki.coronawarnapp.exception.ApplicationConfigurationCorruptException +import de.rki.coronawarnapp.exception.ApplicationConfigurationInvalidException +import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass +import de.rki.coronawarnapp.util.ZipHelper.unzip +import de.rki.coronawarnapp.util.security.VerificationKeys +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.joda.time.LocalDate +import org.joda.time.LocalTime +import org.joda.time.format.DateTimeFormat +import timber.log.Timber +import java.io.File +import java.util.Locale +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class DownloadServer @Inject constructor( + private val DownloadAPI: Lazy, + private val verificationKeys: VerificationKeys, + @DownloadHomeCountry private val homeCountry: LocationCode +) { + + private val apiDownload: DownloadApiV1 + get() = DownloadAPI.get() + + suspend fun downloadAppConfig(): ApplicationConfigurationOuterClass.ApplicationConfiguration = + withContext(Dispatchers.IO) { + var exportBinary: ByteArray? = null + var exportSignature: ByteArray? = null + + apiDownload.getApplicationConfiguration(homeCountry.identifier).byteStream() + .unzip { entry, entryContent -> + if (entry.name == EXPORT_BINARY_FILE_NAME) exportBinary = + entryContent.copyOf() + if (entry.name == EXPORT_SIGNATURE_FILE_NAME) exportSignature = + entryContent.copyOf() + } + if (exportBinary == null || exportSignature == null) { + throw ApplicationConfigurationInvalidException() + } + + if (verificationKeys.hasInvalidSignature(exportBinary, exportSignature)) { + throw ApplicationConfigurationCorruptException() + } + + try { + return@withContext ApplicationConfigurationOuterClass.ApplicationConfiguration.parseFrom( + exportBinary + ) + } catch (e: InvalidProtocolBufferException) { + throw ApplicationConfigurationInvalidException() + } + } + + /** + * Gets the country index which is then filtered by given filter param or if param not set + * @param wantedCountries (array of country codes) used to filter + * only wanted countries of the country index (case insensitive) + */ + suspend fun getCountryIndex( + wantedCountries: List + ): List = withContext(Dispatchers.IO) { + apiDownload + .getCountryIndex().filter { + wantedCountries + .map { c -> c.toUpperCase(Locale.ROOT) } + .contains(it.toUpperCase(Locale.ROOT)) + } + .map { LocationCode(it) } + } + + suspend fun getDayIndex(location: LocationCode): List = withContext(Dispatchers.IO) { + apiDownload + .getDayIndex(location.identifier) + .map { dayString -> // 2020-08-19 + LocalDate.parse(dayString, DAY_FORMATTER) + } + } + + suspend fun getHourIndex(location: LocationCode, day: LocalDate): List = + withContext(Dispatchers.IO) { + apiDownload + .getHourIndex(location.identifier, day.toString(DAY_FORMATTER)) + .map { hourString -> LocalTime.parse(hourString, HOUR_FORMATTER) } + } + + /** + * Retrieves Key Files from the Server + * Leave **[hour]** null to download a day package + */ + suspend fun downloadKeyFile( + locationCode: LocationCode, + day: LocalDate, + hour: LocalTime? = null, + saveTo: File + ) = withContext(Dispatchers.IO) { + Timber.tag(TAG).v( + "Starting download: country=%s, day=%s, hour=%s -> %s.", + locationCode, day, hour, saveTo + ) + + if (saveTo.exists()) { + Timber.tag(TAG).w("File existed, overwriting: %s", saveTo) + saveTo.delete() + } + + saveTo.outputStream().use { + + val streamingBody = if (hour != null) { + apiDownload.downloadKeyFileForHour( + locationCode.identifier, + day.toString(DAY_FORMATTER), + hour.toString(HOUR_FORMATTER) + ) + } else { + apiDownload.downloadKeyFileForDay( + locationCode.identifier, + day.toString(DAY_FORMATTER) + ) + } + streamingBody.byteStream().copyTo(it, DEFAULT_BUFFER_SIZE) + } + Timber.tag(TAG).v("Key file download successful: %s", saveTo) + } + + companion object { + private val TAG = DownloadServer::class.java.simpleName + private val DAY_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd") + private val HOUR_FORMATTER = DateTimeFormat.forPattern("HH") + + private const val EXPORT_BINARY_FILE_NAME = "export.bin" + private const val EXPORT_SIGNATURE_FILE_NAME = "export.sig" + + } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerUrl.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerUrl.kt new file mode 100644 index 00000000000..f347e533316 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerUrl.kt @@ -0,0 +1,8 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import javax.inject.Qualifier + +@Qualifier +@MustBeDocumented +@Retention(AnnotationRetention.RUNTIME) +annotation class DownloadServerUrl diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt new file mode 100644 index 00000000000..62f7822c8a7 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt @@ -0,0 +1,5 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +data class LocationCode( + val identifier: String +) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFile.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFile.kt new file mode 100644 index 00000000000..fbc40571bde --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFile.kt @@ -0,0 +1,86 @@ +package de.rki.coronawarnapp.diagnosiskeys.storage + +import androidx.room.ColumnInfo +import androidx.room.Entity +import androidx.room.PrimaryKey +import androidx.room.TypeConverter +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.util.HashExtensions.toSHA1 +import org.joda.time.Instant +import org.joda.time.LocalDate +import org.joda.time.LocalTime + +@Entity(tableName = "keyfiles") +data class CachedKeyFile( + @PrimaryKey @ColumnInfo(name = "id") val id: String, + @ColumnInfo(name = "type") val type: Type, + @ColumnInfo(name = "location") val location: LocationCode, // i.e. "DE" + @ColumnInfo(name = "day") val day: LocalDate, // i.e. 2020-08-23 + @ColumnInfo(name = "hour") val hour: LocalTime?, // i.e. 23 + @ColumnInfo(name = "createdAt") val createdAt: Instant, + @ColumnInfo(name = "checksumMD5") val checksumMD5: String?, + @ColumnInfo(name = "completed") val isDownloadComplete: Boolean +) { + + constructor( + type: Type, + location: LocationCode, + day: LocalDate, + hour: LocalTime?, + createdAt: Instant + ) : this( + id = calcluateId(location, day, hour, type), + location = location, + day = day, + hour = hour, + type = type, + createdAt = createdAt, + checksumMD5 = null, + isDownloadComplete = false + ) + + @Transient + val fileName: String = "$id.zip" + + fun toDownloadCompleted(checksumMD5: String): DownloadUpdate = DownloadUpdate( + id = id, + checksumMD5 = checksumMD5, + isDownloadComplete = true + ) + + companion object { + fun calcluateId( + location: LocationCode, + day: LocalDate, + hour: LocalTime?, + type: Type + ): String { + var rawId = "${location.identifier}.${type.typeValue}.$day" + hour?.let { rawId += ".$hour" } + return rawId.toSHA1() + } + } + + enum class Type constructor(internal val typeValue: String) { + COUNTRY_DAY("country_day"), + COUNTRY_HOUR("country_hour"); + + class Converter { + @TypeConverter + fun toType(value: String?): Type? = + value?.let { values().single { it.typeValue == value } } + + @TypeConverter + fun fromType(type: Type?): String? = type?.typeValue + } + } + + @Entity + data class DownloadUpdate( + @PrimaryKey @ColumnInfo(name = "id") val id: String, + @ColumnInfo(name = "checksumMD5") val checksumMD5: String?, + @ColumnInfo(name = "completed") val isDownloadComplete: Boolean + ) +} + + diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt new file mode 100644 index 00000000000..f26fa21839a --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt @@ -0,0 +1,64 @@ +package de.rki.coronawarnapp.diagnosiskeys.storage + +import android.content.Context +import androidx.room.Dao +import androidx.room.Database +import androidx.room.Delete +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import androidx.room.Room +import androidx.room.RoomDatabase +import androidx.room.TypeConverters +import androidx.room.Update +import de.rki.coronawarnapp.util.database.CommonConverters +import javax.inject.Inject + +@Database( + entities = [CachedKeyFile::class], + version = 1, + exportSchema = true +) +@TypeConverters(CommonConverters::class, CachedKeyFile.Type.Converter::class) +abstract class KeyCacheDatabase : RoomDatabase() { + + abstract fun cachedKeyFiles(): CachedKeyFileDao + + @Dao + interface CachedKeyFileDao { + @Query("SELECT * FROM keyfiles") + suspend fun getAllEntries(): List + + @Query("SELECT * FROM keyfiles WHERE type = :type") + suspend fun getEntriesForType(type: String): List + + @Query("DELETE FROM keyfiles") + suspend fun clear() + + @Insert(onConflict = OnConflictStrategy.ABORT) + suspend fun insertEntry(cachedKeyFile: CachedKeyFile) + + @Delete + suspend fun deleteEntry(cachedKeyFile: CachedKeyFile) + + @Update(entity = CachedKeyFile::class) + suspend fun updateDownloadState(update: CachedKeyFile.DownloadUpdate) + } + + class Factory @Inject constructor(private val context: Context) { + /** + * The fallback behavior is to reset the app as we only store exposure summaries + * and cached references that are non-critical to app operation. + */ + fun create(): KeyCacheDatabase = Room + .databaseBuilder(context, KeyCacheDatabase::class.java, DATABASE_NAME) + .fallbackToDestructiveMigrationFrom() + .build() + + } + + companion object { + private const val DATABASE_NAME = "keycache.db" + } + +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt new file mode 100644 index 00000000000..bdda1b53c0c --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt @@ -0,0 +1,135 @@ +/****************************************************************************** + * Corona-Warn-App * + * * + * SAP SE and all other contributors / * + * copyright owners license this file to you under the Apache * + * License, Version 2.0 (the "License"); you may not use this * + * file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ******************************************************************************/ + +package de.rki.coronawarnapp.diagnosiskeys.storage + +import android.content.Context +import android.database.sqlite.SQLiteConstraintException +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.util.TimeStamper +import org.joda.time.LocalDate +import org.joda.time.LocalTime +import timber.log.Timber +import java.io.File +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class KeyCacheRepository @Inject constructor( + private val context: Context, + private val databaseFactory: KeyCacheDatabase.Factory, + private val timeStamper: TimeStamper +) { + + private val storageDir by lazy { + File(context.cacheDir, "diagnosis_keys").apply { + if (!exists()) { + if (mkdirs()) { + Timber.d("KeyCache directory created: %s", this) + } else { + Timber.w("KeyCache directory creation failed: %s", this) + } + } + } + } + + private val database by lazy { databaseFactory.create() } + + private val cacheKeyFilesDao: KeyCacheDatabase.CachedKeyFileDao + get() = database.cachedKeyFiles() + + private fun tryMigration() { + // TODO() from key-export + } + + private fun checkCacheHealth() { +// TODO() + } + + fun getPathForKey(cachedKeyFile: CachedKeyFile): File { + return File(storageDir, cachedKeyFile.fileName) + } + + suspend fun getAllCachedKeys(): List { + return cacheKeyFilesDao.getAllEntries() + } + + suspend fun getEntriesForType(type: CachedKeyFile.Type): List { + return cacheKeyFilesDao.getEntriesForType(type.typeValue) + } + + suspend fun createCacheEntry( + type: CachedKeyFile.Type, + location: LocationCode, + dayIdentifier: LocalDate, + hourIdentifier: LocalTime? + ): Pair { + val newKeyFile = CachedKeyFile( + type = type, + location = location, + day = dayIdentifier, + hour = hourIdentifier, + createdAt = timeStamper.nowUTC + ) + + val targetFile = getPathForKey(newKeyFile) + + try { + cacheKeyFilesDao.insertEntry(newKeyFile) + if (targetFile.exists()) { + Timber.w("Target path despire no collision exists, deleting: %s", targetFile) + } + } catch (e: SQLiteConstraintException) { + Timber.e(e, "Insertion collision? Overwriting for %s", newKeyFile) + delete(listOf(newKeyFile)) + + Timber.d(e, "Retrying insertion for %s", newKeyFile) + cacheKeyFilesDao.insertEntry(newKeyFile) + } + + // This can't be null unless our cache dir is root `/` + val targetParent = targetFile.parentFile!! + if (!targetParent.exists()) { + Timber.w("Parent folder doesn't exist, cache cleared? %s", targetParent) + targetParent.mkdirs() + } + + return newKeyFile to targetFile + } + + suspend fun markKeyComplete(cachedKeyFile: CachedKeyFile, checksumMD5: String) { + val update = cachedKeyFile.toDownloadCompleted(checksumMD5) + cacheKeyFilesDao.updateDownloadState(update) + } + + suspend fun delete(keyFiles: Collection) { + Timber.d("delete(keyFiles=%s)", keyFiles) + keyFiles.forEach { key -> + cacheKeyFilesDao.deleteEntry(key) + Timber.v("Deleted %s", key) + val path = getPathForKey(key) + if (path.delete()) Timber.v("Deleted cache key file at %s", path) + } + } + + suspend fun clear() { + Timber.i("clear()") + delete(getAllCachedKeys()) + } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheDao.kt similarity index 97% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt rename to Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheDao.kt index ec812964a59..f67ae089d6c 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheDao.kt @@ -17,7 +17,7 @@ * under the License. * ******************************************************************************/ -package de.rki.coronawarnapp.storage.keycache +package de.rki.coronawarnapp.diagnosiskeys.storage.legacy import androidx.room.Dao import androidx.room.Delete diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheEntity.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntity.kt similarity index 96% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheEntity.kt rename to Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntity.kt index 8bd2ac55df3..449c423c991 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheEntity.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntity.kt @@ -17,7 +17,7 @@ * under the License. * ******************************************************************************/ -package de.rki.coronawarnapp.storage.keycache +package de.rki.coronawarnapp.diagnosiskeys.storage.legacy import androidx.room.Entity import androidx.room.Index diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpClientDefault.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpClientDefault.kt new file mode 100644 index 00000000000..dca6b3f7826 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpClientDefault.kt @@ -0,0 +1,8 @@ +package de.rki.coronawarnapp.http + +import javax.inject.Qualifier + +@Qualifier +@MustBeDocumented +@Retention(AnnotationRetention.RUNTIME) +annotation class HttpClientDefault diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt new file mode 100644 index 00000000000..055746655bc --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt @@ -0,0 +1,57 @@ +package de.rki.coronawarnapp.http + +import dagger.Module +import dagger.Provides +import dagger.Reusable +import de.rki.coronawarnapp.BuildConfig +import de.rki.coronawarnapp.http.config.HTTPVariables +import de.rki.coronawarnapp.http.interceptor.RetryInterceptor +import de.rki.coronawarnapp.http.interceptor.WebSecurityVerificationInterceptor +import de.rki.coronawarnapp.risk.TimeVariables +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.logging.HttpLoggingInterceptor +import retrofit2.converter.gson.GsonConverterFactory +import retrofit2.converter.protobuf.ProtoConverterFactory +import timber.log.Timber +import java.util.concurrent.TimeUnit + +@Module +class HttpModule { + + @Reusable + @HttpClientDefault + @Provides + fun defaultHttpClient(): OkHttpClient { + val interceptors: List = listOf( + WebSecurityVerificationInterceptor(), + HttpLoggingInterceptor(object : HttpLoggingInterceptor.Logger { + override fun log(message: String) { + Timber.tag("OkHttp").v(message) + } + }).apply { + if (BuildConfig.DEBUG) setLevel(HttpLoggingInterceptor.Level.BODY) + }, + RetryInterceptor(), + HttpErrorParser() + ) + + return OkHttpClient.Builder().apply { + connectTimeout(HTTPVariables.getHTTPConnectionTimeout(), TimeUnit.MILLISECONDS) + readTimeout(HTTPVariables.getHTTPReadTimeout(), TimeUnit.MILLISECONDS) + writeTimeout(HTTPVariables.getHTTPWriteTimeout(), TimeUnit.MILLISECONDS) + callTimeout(TimeVariables.getTransactionTimeout(), TimeUnit.MILLISECONDS) + + interceptors.forEach { addInterceptor(it) } + }.build() + + } + + @Reusable + @Provides + fun provideGSONConverter(): GsonConverterFactory = GsonConverterFactory.create() + + @Reusable + @Provides + fun provideProtoonverter(): ProtoConverterFactory = ProtoConverterFactory.create() +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt index 9e383ff44cf..364e057eaa1 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt @@ -4,172 +4,37 @@ import android.webkit.URLUtil import de.rki.coronawarnapp.BuildConfig import de.rki.coronawarnapp.CoronaWarnApplication import de.rki.coronawarnapp.exception.http.ServiceFactoryException -import de.rki.coronawarnapp.http.config.HTTPVariables -import de.rki.coronawarnapp.http.interceptor.RetryInterceptor -import de.rki.coronawarnapp.http.interceptor.WebSecurityVerificationInterceptor -import de.rki.coronawarnapp.http.service.DistributionService import de.rki.coronawarnapp.http.service.SubmissionService import de.rki.coronawarnapp.http.service.VerificationService -import de.rki.coronawarnapp.risk.TimeVariables import okhttp3.Cache import okhttp3.CipherSuite -import okhttp3.ConnectionPool import okhttp3.ConnectionSpec -import okhttp3.Interceptor import okhttp3.OkHttpClient import okhttp3.TlsVersion -import okhttp3.logging.HttpLoggingInterceptor import retrofit2.Retrofit import retrofit2.converter.gson.GsonConverterFactory import retrofit2.converter.protobuf.ProtoConverterFactory -import timber.log.Timber import java.io.File -import java.util.concurrent.TimeUnit - -class ServiceFactory { - companion object { - /** - * 10 MiB - */ - private const val HTTP_CACHE_SIZE = 10L * 1024L * 1024L - - /** - * Cache file name - */ - private const val HTTP_CACHE_NAME = "http_cache" - } - - /** - * List of interceptors, e.g. logging - */ - private val mInterceptors: List = listOf( - WebSecurityVerificationInterceptor(), - HttpLoggingInterceptor(object : HttpLoggingInterceptor.Logger { - override fun log(message: String) { - Timber.tag("OkHttp").v(message) - } - }).apply { - if (BuildConfig.DEBUG) setLevel(HttpLoggingInterceptor.Level.BODY) - }, - RetryInterceptor(), - HttpErrorParser() - ) - - /** - * connection pool held in-memory, especially useful for key retrieval - */ - private val conPool = ConnectionPool() - - /** - * Basic disk cache backed by LRU - */ - private val cache = Cache( - directory = File(CoronaWarnApplication.getAppContext().cacheDir, HTTP_CACHE_NAME), - maxSize = HTTP_CACHE_SIZE - ) - - private val gsonConverterFactory = GsonConverterFactory.create() - private val protoConverterFactory = ProtoConverterFactory.create() - - private val okHttpClient by lazy { - val clientBuilder = OkHttpClient.Builder() - - clientBuilder.connectTimeout( - HTTPVariables.getHTTPConnectionTimeout(), - TimeUnit.MILLISECONDS +import javax.inject.Inject + +class ServiceFactory @Inject constructor( + private val gsonConverterFactory: GsonConverterFactory, + private val protoConverterFactory: ProtoConverterFactory, + @HttpClientDefault private val defaultHttpClient: OkHttpClient +) { + + private val cache by lazy { + Cache( + directory = File(CoronaWarnApplication.getAppContext().cacheDir, HTTP_CACHE_FOLDER), + maxSize = HTTP_CACHE_SIZE ) - clientBuilder.readTimeout( - HTTPVariables.getHTTPReadTimeout(), - TimeUnit.MILLISECONDS - ) - clientBuilder.writeTimeout( - HTTPVariables.getHTTPWriteTimeout(), - TimeUnit.MILLISECONDS - ) - clientBuilder.callTimeout( - TimeVariables.getTransactionTimeout(), - TimeUnit.MILLISECONDS - ) - - clientBuilder.connectionPool(conPool) - - cache.evictAll() - clientBuilder.cache(cache) - - mInterceptors.forEach { clientBuilder.addInterceptor(it) } - - clientBuilder.build() } - - /** - * For the CDN we want to ensure maximum Compatibility. - */ - private fun getCDNSpecs(): List = listOf( - ConnectionSpec.Builder(ConnectionSpec.COMPATIBLE_TLS) - .tlsVersions( - TlsVersion.TLS_1_0, - TlsVersion.TLS_1_1, - TlsVersion.TLS_1_2, - TlsVersion.TLS_1_3 - ) - .allEnabledCipherSuites() - .build() - ) - - /** - * For Submission and Verification we want to limit our specifications for TLS. - */ - private fun getRestrictedSpecs(): List = listOf( - ConnectionSpec.Builder(ConnectionSpec.RESTRICTED_TLS) - .tlsVersions( - TlsVersion.TLS_1_2, - TlsVersion.TLS_1_3 - ) - .cipherSuites( - // TLS 1.2 with Perfect Forward Secrecy (BSI TR-02102-2) - CipherSuite.TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256, - CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, - CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, - CipherSuite.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, - CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - CipherSuite.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - CipherSuite.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, - CipherSuite.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, - CipherSuite.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, - CipherSuite.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, - CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, - CipherSuite.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, - CipherSuite.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, - CipherSuite.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - // TLS 1.3 (BSI TR-02102-2) - CipherSuite.TLS_AES_128_GCM_SHA256, - CipherSuite.TLS_AES_256_GCM_SHA384, - CipherSuite.TLS_AES_128_CCM_SHA256 - ) - .build() - ) - - /** - * Helper function to create a new client from an existent Client with New Specs. - * - * @param specs - */ - private fun OkHttpClient.buildClientWithNewSpecs(specs: List) = - this.newBuilder().connectionSpecs(specs).build() - - private val downloadCdnUrl - get() = getValidUrl(BuildConfig.DOWNLOAD_CDN_URL) - - fun distributionService(): DistributionService = distributionService - private val distributionService by lazy { - Retrofit.Builder() - .client(okHttpClient.buildClientWithNewSpecs(getCDNSpecs())) - .baseUrl(downloadCdnUrl) - .addConverterFactory(gsonConverterFactory) + private val okHttpClient by lazy { + defaultHttpClient + .newBuilder() + .connectionSpecs(getRestrictedSpecs()) + .cache(cache) .build() - .create(DistributionService::class.java) } private val verificationCdnUrl @@ -178,7 +43,7 @@ class ServiceFactory { fun verificationService(): VerificationService = verificationService private val verificationService by lazy { Retrofit.Builder() - .client(okHttpClient.buildClientWithNewSpecs(getRestrictedSpecs())) + .client(okHttpClient) .baseUrl(verificationCdnUrl) .addConverterFactory(gsonConverterFactory) .build() @@ -191,7 +56,7 @@ class ServiceFactory { fun submissionService(): SubmissionService = submissionService private val submissionService by lazy { Retrofit.Builder() - .client(okHttpClient.buildClientWithNewSpecs(getRestrictedSpecs())) + .client(okHttpClient) .baseUrl(submissionCdnUrl) .addConverterFactory(protoConverterFactory) .addConverterFactory(gsonConverterFactory) @@ -205,4 +70,45 @@ class ServiceFactory { } return url } + + companion object { + private const val HTTP_CACHE_SIZE = 10L * 1024L * 1024L // 10 MiB + private const val HTTP_CACHE_FOLDER = "http_cache" // /cache/http_cache + + + /** + * For Submission and Verification we want to limit our specifications for TLS. + */ + private fun getRestrictedSpecs(): List = listOf( + ConnectionSpec.Builder(ConnectionSpec.RESTRICTED_TLS) + .tlsVersions( + TlsVersion.TLS_1_2, + TlsVersion.TLS_1_3 + ) + .cipherSuites( + // TLS 1.2 with Perfect Forward Secrecy (BSI TR-02102-2) + CipherSuite.TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256, + CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, + CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + CipherSuite.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + CipherSuite.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + CipherSuite.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, + CipherSuite.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, + CipherSuite.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, + CipherSuite.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, + CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, + CipherSuite.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + CipherSuite.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, + CipherSuite.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, + // TLS 1.3 (BSI TR-02102-2) + CipherSuite.TLS_AES_128_GCM_SHA256, + CipherSuite.TLS_AES_256_GCM_SHA384, + CipherSuite.TLS_AES_128_CCM_SHA256 + ) + .build() + ) + } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt index a92142a6781..d0bee06128a 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt @@ -21,43 +21,27 @@ package de.rki.coronawarnapp.http import KeyExportFormat import com.google.protobuf.ByteString -import com.google.protobuf.InvalidProtocolBufferException -import de.rki.coronawarnapp.exception.ApplicationConfigurationCorruptException -import de.rki.coronawarnapp.exception.ApplicationConfigurationInvalidException import de.rki.coronawarnapp.http.requests.RegistrationRequest import de.rki.coronawarnapp.http.requests.RegistrationTokenRequest import de.rki.coronawarnapp.http.requests.TanRequestBody -import de.rki.coronawarnapp.http.service.DistributionService import de.rki.coronawarnapp.http.service.SubmissionService import de.rki.coronawarnapp.http.service.VerificationService -import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass.ApplicationConfiguration import de.rki.coronawarnapp.service.diagnosiskey.DiagnosisKeyConstants import de.rki.coronawarnapp.service.submission.KeyType import de.rki.coronawarnapp.service.submission.SubmissionConstants -import de.rki.coronawarnapp.storage.FileStorageHelper -import de.rki.coronawarnapp.util.TimeAndDateExtensions.toServerFormat -import de.rki.coronawarnapp.util.ZipHelper.unzip +import de.rki.coronawarnapp.util.di.AppInjector import de.rki.coronawarnapp.util.security.HashHelper -import de.rki.coronawarnapp.util.security.VerificationKeys import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import timber.log.Timber -import java.io.File -import java.util.Date -import java.util.Locale -import java.util.UUID import kotlin.math.max class WebRequestBuilder( - private val distributionService: DistributionService, private val verificationService: VerificationService, - private val submissionService: SubmissionService, - private val verificationKeys: VerificationKeys + private val submissionService: SubmissionService ) { companion object { private val TAG: String? = WebRequestBuilder::class.simpleName - private const val EXPORT_BINARY_FILE_NAME = "export.bin" - private const val EXPORT_SIGNATURE_FILE_NAME = "export.sig" @Volatile private var instance: WebRequestBuilder? = null @@ -69,98 +53,14 @@ class WebRequestBuilder( } private fun buildWebRequestBuilder(): WebRequestBuilder { - val serviceFactory = ServiceFactory() + val serviceFactory = AppInjector.component.serviceFactory return WebRequestBuilder( - serviceFactory.distributionService(), serviceFactory.verificationService(), - serviceFactory.submissionService(), - VerificationKeys() + serviceFactory.submissionService() ) } } - /** - * Gets the country index which is then filtered by given filter param or if param not set - * @param wantedCountries (array of country codes) used to filter - * only wanted countries of the country index (case insensitive) - */ - suspend fun asyncGetCountryIndex( - wantedCountries: List - ): List = - withContext(Dispatchers.IO) { - return@withContext distributionService - .getDateIndex(DiagnosisKeyConstants.AVAILABLE_COUNTRIES_URL) - .filter { - wantedCountries.map { c -> c.toUpperCase(Locale.ROOT) } - .contains(it.toUpperCase(Locale.ROOT)) - } - .toList() - } - - suspend fun asyncGetHourIndex(day: Date): List = withContext(Dispatchers.IO) { - return@withContext distributionService - .getHourIndex( - DiagnosisKeyConstants.AVAILABLE_DATES_URL + - "/${day.toServerFormat()}/${DiagnosisKeyConstants.HOUR}" - ) - .toList() - } - - /** - * Get the date index based on the given country - * @param country the country where the date index should be requested - */ - suspend fun asyncGetDateIndex(country: String): List = withContext(Dispatchers.IO) { - return@withContext distributionService - .getDateIndex("${DiagnosisKeyConstants.AVAILABLE_COUNTRIES_URL}/$country/${DiagnosisKeyConstants.DATE}") - .toList() - } - - /** - * Retrieves Key Files from the Server based on a URL - * - * @param url the given URL - */ - suspend fun asyncGetKeyFilesFromServer( - url: String - ): File = withContext(Dispatchers.IO) { - val fileName = "${UUID.nameUUIDFromBytes(url.toByteArray())}.zip" - val file = File(FileStorageHelper.keyExportDirectory, fileName) - file.outputStream().use { - Timber.v("Added $url to queue.") - distributionService.getKeyFiles(url).byteStream().copyTo(it, DEFAULT_BUFFER_SIZE) - Timber.v("key file request successful.") - } - return@withContext file - } - - suspend fun asyncGetApplicationConfigurationFromServer(): ApplicationConfiguration = - withContext(Dispatchers.IO) { - var exportBinary: ByteArray? = null - var exportSignature: ByteArray? = null - - distributionService.getApplicationConfiguration( - DiagnosisKeyConstants.COUNTRY_APPCONFIG_DOWNLOAD_URL - ).byteStream().unzip { entry, entryContent -> - if (entry.name == EXPORT_BINARY_FILE_NAME) exportBinary = entryContent.copyOf() - if (entry.name == EXPORT_SIGNATURE_FILE_NAME) exportSignature = - entryContent.copyOf() - } - if (exportBinary == null || exportSignature == null) { - throw ApplicationConfigurationInvalidException() - } - - if (verificationKeys.hasInvalidSignature(exportBinary, exportSignature)) { - throw ApplicationConfigurationCorruptException() - } - - try { - return@withContext ApplicationConfiguration.parseFrom(exportBinary) - } catch (e: InvalidProtocolBufferException) { - throw ApplicationConfigurationInvalidException() - } - } - suspend fun asyncGetRegistrationToken( key: String, keyType: KeyType diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/service/DistributionService.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/service/DistributionService.kt deleted file mode 100644 index 1571fc4b66c..00000000000 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/service/DistributionService.kt +++ /dev/null @@ -1,22 +0,0 @@ -package de.rki.coronawarnapp.http.service - -import okhttp3.ResponseBody -import retrofit2.http.GET -import retrofit2.http.Streaming -import retrofit2.http.Url - -interface DistributionService { - - @GET - suspend fun getDateIndex(@Url url: String): List - - @GET - suspend fun getHourIndex(@Url url: String): List - - @Streaming - @GET - suspend fun getKeyFiles(@Url url: String): ResponseBody - - @GET - suspend fun getApplicationConfiguration(@Url url: String): ResponseBody -} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt index 314b7c1197d..007e491d1bf 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt @@ -1,14 +1,14 @@ package de.rki.coronawarnapp.service.applicationconfiguration import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration -import de.rki.coronawarnapp.http.WebRequestBuilder import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass.ApplicationConfiguration import de.rki.coronawarnapp.util.CWADebug +import de.rki.coronawarnapp.util.di.AppInjector object ApplicationConfigurationService { suspend fun asyncRetrieveApplicationConfiguration(): ApplicationConfiguration { - return WebRequestBuilder.getInstance().asyncGetApplicationConfigurationFromServer().let { - return if (CWADebug.isDebugBuildOrMode) { + return AppInjector.component.downloadServer.downloadAppConfig().let { + if (CWADebug.isDebugBuildOrMode) { // TODO: THIS IS A MOCK -> Remove after Backend is providing this information. it.toBuilder() .clearSupportedCountries() diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstants.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstants.kt index e36b9111864..8acfd1c5ef2 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstants.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstants.kt @@ -26,61 +26,17 @@ object DiagnosisKeyConstants { /** version resource variable for REST-like Service Calls */ private const val VERSION = "version" - /** parameter resource variable for REST-like Service Calls */ - private const val PARAMETERS = "parameters" - private const val APPCONFIG = "configuration" - /** diagnosis keys resource variable for REST-like Service Calls */ private const val DIAGNOSIS_KEYS = "diagnosis-keys" - /** country resource variable for REST-like Service Calls */ - private const val COUNTRY = "country" - - /** date resource variable for REST-like Service Calls */ - const val DATE = "date" - - /** hour resource variable for REST-like Service Calls */ - const val HOUR = "hour" - - private const val INDEX_FILE_NAME = "index.txt" - /** resource variables but non-static context */ private var CURRENT_VERSION = "v1" - private const val CURRENT_COUNTRY = "DE" - - /** Distribution URL built from CDN URL's and REST resources */ - private var VERSIONED_DISTRIBUTION_CDN_URL = "$VERSION/$CURRENT_VERSION" /** Submission URL built from CDN URL's and REST resources */ private var VERSIONED_SUBMISSION_CDN_URL = "$VERSION/$CURRENT_VERSION" - /** Parameter Download URL built from CDN URL's and REST resources */ - private val PARAMETERS_DOWNLOAD_URL = "$VERSIONED_DISTRIBUTION_CDN_URL/$PARAMETERS" - private val APPCONFIG_DOWNLOAD_URL = "$VERSIONED_DISTRIBUTION_CDN_URL/$APPCONFIG" - - /** Index Download URL built from CDN URL's and REST resources */ - val INDEX_DOWNLOAD_URL = "$VERSIONED_DISTRIBUTION_CDN_URL/$INDEX_FILE_NAME" - - /** Diagnosis key Download URL built from CDN URL's and REST resources */ - val DIAGNOSIS_KEYS_DOWNLOAD_URL = "$VERSIONED_DISTRIBUTION_CDN_URL/$DIAGNOSIS_KEYS" - /** Diagnosis key Submission URL built from CDN URL's and REST resources */ val DIAGNOSIS_KEYS_SUBMISSION_URL = "$VERSIONED_SUBMISSION_CDN_URL/$DIAGNOSIS_KEYS" - /** Country-Specific Parameter URL built from CDN URL's and REST resources */ - val PARAMETERS_COUNTRY_DOWNLOAD_URL = "$PARAMETERS_DOWNLOAD_URL/$COUNTRY" - val APPCONFIG_COUNTRY_DOWNLOAD_URL = "$APPCONFIG_DOWNLOAD_URL/$COUNTRY" - - val COUNTRY_APPCONFIG_DOWNLOAD_URL = - "$APPCONFIG_COUNTRY_DOWNLOAD_URL/$CURRENT_COUNTRY/app_config" - - /** Dynamic URL to be able to specify which country should be used - * instead of $CURRENT_COUNTRY - **/ - val AVAILABLE_COUNTRIES_URL = "$DIAGNOSIS_KEYS_DOWNLOAD_URL/$COUNTRY" - - /** Available Dates URL built from CDN URL's and REST resources */ - val AVAILABLE_DATES_URL = "$AVAILABLE_COUNTRIES_URL/$CURRENT_COUNTRY/$DATE" - const val SERVER_ERROR_CODE_403 = 403 } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt index ce4640c15db..c2e7735be24 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt @@ -7,14 +7,15 @@ import androidx.room.Database import androidx.room.Room import androidx.room.RoomDatabase import androidx.room.TypeConverters -import de.rki.coronawarnapp.storage.keycache.KeyCacheDao -import de.rki.coronawarnapp.storage.keycache.KeyCacheEntity -import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheDao +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheEntity import de.rki.coronawarnapp.storage.tracing.TracingIntervalDao import de.rki.coronawarnapp.storage.tracing.TracingIntervalEntity import de.rki.coronawarnapp.storage.tracing.TracingIntervalRepository -import de.rki.coronawarnapp.util.Converters +import de.rki.coronawarnapp.util.database.CommonConverters +import de.rki.coronawarnapp.util.di.AppInjector import de.rki.coronawarnapp.util.security.SecurityHelper +import kotlinx.coroutines.runBlocking import net.sqlcipher.database.SupportFactory import java.io.File @@ -23,7 +24,7 @@ import java.io.File version = 1, exportSchema = true ) -@TypeConverters(Converters::class) +@TypeConverters(CommonConverters::class) abstract class AppDatabase : RoomDatabase() { abstract fun exposureSummaryDao(): ExposureSummaryDao @@ -54,7 +55,8 @@ abstract class AppDatabase : RoomDatabase() { resetInstance() // reset also the repo instances - KeyCacheRepository.resetInstance() + val keyRepository = AppInjector.component.keyCacheRepository + runBlocking { keyRepository.clear() } // TODO this is not nice TracingIntervalRepository.resetInstance() ExposureSummaryRepository.resetInstance() } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt deleted file mode 100644 index 9e3ce000310..00000000000 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt +++ /dev/null @@ -1,95 +0,0 @@ -/****************************************************************************** - * Corona-Warn-App * - * * - * SAP SE and all other contributors / * - * copyright owners license this file to you under the Apache * - * License, Version 2.0 (the "License"); you may not use this * - * file except in compliance with the License. * - * You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ******************************************************************************/ - -package de.rki.coronawarnapp.storage.keycache - -import android.content.Context -import de.rki.coronawarnapp.storage.AppDatabase -import timber.log.Timber -import java.io.File -import java.net.URI - -class KeyCacheRepository(private val keyCacheDao: KeyCacheDao) { - companion object { - @Volatile - private var instance: KeyCacheRepository? = null - - private fun getInstance(keyCacheDao: KeyCacheDao) = - instance ?: synchronized(this) { - instance - ?: KeyCacheRepository(keyCacheDao) - .also { instance = it } - } - - fun resetInstance() = synchronized(this) { - instance = null - } - - fun getDateRepository(context: Context): KeyCacheRepository { - return getInstance( - AppDatabase.getInstance(context.applicationContext) - .dateDao() - ) - } - } - - enum class DateEntryType { - DAY, - HOUR - } - - suspend fun createEntry(key: String, uri: URI, type: DateEntryType) = keyCacheDao.insertEntry( - KeyCacheEntity().apply { - this.id = key - this.path = uri.rawPath - this.type = type.ordinal - } - ) - - suspend fun deleteOutdatedEntries(validEntries: List) = - keyCacheDao.getAllEntries().forEach { - Timber.v("valid entries for cache from server: $validEntries") - val file = File(it.path) - if (!validEntries.contains(it.id) || !file.exists()) { - Timber.w("${it.id} will be deleted from the cache") - deleteFileForEntry(it) - keyCacheDao.deleteEntry(it) - } - } - - private fun deleteFileForEntry(entry: KeyCacheEntity) = File(entry.path).delete() - - suspend fun getDates() = keyCacheDao.getDates() - suspend fun getHours() = keyCacheDao.getHours() - - suspend fun clearHours() { - getHours().forEach { deleteFileForEntry(it) } - keyCacheDao.clearHours() - } - - suspend fun clear() { - keyCacheDao.getAllEntries().forEach { deleteFileForEntry(it) } - keyCacheDao.clear() - } - - suspend fun getFilesFromEntries() = keyCacheDao - .getAllEntries() - .map { File(it.path) } - .filter { it.exists() } -} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt index cdbbe414155..64e2b890633 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt @@ -20,12 +20,12 @@ package de.rki.coronawarnapp.transaction import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration -import de.rki.coronawarnapp.CoronaWarnApplication +import de.rki.coronawarnapp.diagnosiskeys.download.KeyFileDownloader +import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.nearby.InternalExposureNotificationClient import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigurationService import de.rki.coronawarnapp.storage.FileStorageHelper import de.rki.coronawarnapp.storage.LocalData -import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.API_SUBMISSION import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.CLOSE import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.FETCH_DATE_UPDATE @@ -36,12 +36,12 @@ import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.Retriev import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.rollback import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.start import de.rki.coronawarnapp.util.CWADebug -import de.rki.coronawarnapp.util.CachedKeyFileHolder import de.rki.coronawarnapp.util.di.AppInjector import de.rki.coronawarnapp.worker.BackgroundWorkHelper import org.joda.time.DateTime import org.joda.time.DateTimeZone import org.joda.time.Instant +import org.joda.time.LocalDate import timber.log.Timber import java.io.File import java.util.Date @@ -123,6 +123,12 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { private val transactionScope: TransactionCoroutineScope by lazy { AppInjector.component.transRetrieveKeysInjection.transactionScope } + private val keyCacheRepository: KeyCacheRepository by lazy { + AppInjector.component.keyCacheRepository + } + private val keyFileDownloader: KeyFileDownloader by lazy { + AppInjector.component.keyFileDownloader + } var onApiSubmissionStarted: (() -> Unit)? = null var onApiSubmissionFinished: (() -> Unit)? = null @@ -148,7 +154,7 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { } /** initiates the transaction. This suspend function guarantees a successful transaction once completed. - * @param countries defines which countries (country codes) should be used. If not filled the + * @param requestedCountries defines which countries (country codes) should be used. If not filled the * country codes will be loaded from the ApplicationConfigurationService */ suspend fun start( @@ -267,8 +273,7 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { private suspend fun rollbackFilesFromWebRequests() { Timber.tag(TAG).v("rollback $FILES_FROM_WEB_REQUESTS") - KeyCacheRepository.getDateRepository(CoronaWarnApplication.getAppContext()) - .clear() + keyCacheRepository.clear() } /** @@ -307,7 +312,8 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { countries: List ) = executeState(FILES_FROM_WEB_REQUESTS) { FileStorageHelper.initializeExportSubDirectory() - CachedKeyFileHolder.asyncFetchFiles(currentDate, countries) + val convertedDate = LocalDate.fromDateFields(currentDate) // TODO confirm + keyFileDownloader.asyncFetchFiles(convertedDate, countries) } /** diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt deleted file mode 100644 index 5330e409d92..00000000000 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt +++ /dev/null @@ -1,283 +0,0 @@ -/****************************************************************************** - * Corona-Warn-App * - * * - * SAP SE and all other contributors / * - * copyright owners license this file to you under the Apache * - * License, Version 2.0 (the "License"); you may not use this * - * file except in compliance with the License. * - * You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ******************************************************************************/ - -package de.rki.coronawarnapp.util - -import de.rki.coronawarnapp.CoronaWarnApplication -import de.rki.coronawarnapp.http.WebRequestBuilder -import de.rki.coronawarnapp.service.diagnosiskey.DiagnosisKeyConstants -import de.rki.coronawarnapp.storage.FileStorageHelper -import de.rki.coronawarnapp.storage.LocalData -import de.rki.coronawarnapp.storage.keycache.KeyCacheEntity -import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository -import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository.DateEntryType.DAY -import de.rki.coronawarnapp.util.CachedKeyFileHolder.asyncFetchFiles -import de.rki.coronawarnapp.util.CachedKeyFileHolder.generateCacheKeyFromString -import de.rki.coronawarnapp.util.TimeAndDateExtensions.toServerFormat -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.withContext -import timber.log.Timber -import java.io.File -import java.util.Date -import java.util.UUID - -/** - * Singleton used for accessing key files via combining cached entries from existing files and new requests. - * made explicitly with [asyncFetchFiles] in mind - */ -object CachedKeyFileHolder { - private val TAG: String? = CachedKeyFileHolder::class.simpleName - - /** - * the key cache instance used to store queried dates and hours - */ - private val keyCache = - KeyCacheRepository.getDateRepository(CoronaWarnApplication.getAppContext()) - - /** - * Fetches all necessary Files from the Cached KeyFile Entries out of the [KeyCacheRepository] and - * adds to that all open Files currently available from the Server. - * - * Assumptions made about the implementation: - * - the app initializes with an empty cache and draws in every available data set in the beginning - * - the difference can only work properly if the date from the device is synchronized through the net - * - the difference in timezone is taken into account by using UTC in the Conversion from the Date to Server format - * - the missing days and hours are stored in one table as the actual stored data amount is low - * - the underlying repository from the database has no error and is reliable as source of truth - * - * @param currentDate the current date - if this is adjusted by the calendar, the cache is affected. - * @return list of all files from both the cache and the diff query - */ - suspend fun asyncFetchFiles( - currentDate: Date, - countries: List - ): List = withContext(Dispatchers.IO) { - // Initiate key-cache folder needed for saving downloaded key files - FileStorageHelper.initializeExportSubDirectory() - - checkForFreeSpace() - - // Build pair of country to date - val serverDates = getCountriesFromServer(countries).map { - CountryDataWrapper(it, getDatesFromServer((it))) - } - - if (CWADebug.isDebugBuildOrMode && LocalData.last3HoursMode()) { - asyncHandleLast3HoursFilesFetch(currentDate, serverDates) - } else { - asyncHandleFilesFetch(serverDates) - } - } - - private fun checkForFreeSpace() = FileStorageHelper.checkFileStorageFreeSpace() - - /** - * Fetches files given by serverDates by respecting countries - * @param serverDates pair of dates per country code - */ - private suspend fun asyncHandleFilesFetch( - serverDates: List - ): List = withContext(Dispatchers.IO) { - // Build flat map of uuids from dates of countries - val uuidListFromServer = serverDates.flatMap { countryWrapper -> - countryWrapper.dates.map { date -> - countryWrapper.getURLForDay(date) - } - } - - Timber.v("${uuidListFromServer.size} available dates from server for ${serverDates.size} countries") - - // queries will be executed after the "query plan" was set - val deferredQueries: MutableCollection> = mutableListOf() - keyCache.deleteOutdatedEntries(uuidListFromServer) - - val countryWithMissingDays = getMissingDaysFromDiff(serverDates) - if (countryWithMissingDays.isNotEmpty()) { - // we have a date difference - countryWithMissingDays - .flatMap { country -> - country.dates.map { country.getURLForDay(it) } - } - .map { url -> - async { - keyCache.createEntry( - url.generateCacheKeyFromString(), - WebRequestBuilder.getInstance().asyncGetKeyFilesFromServer(url).toURI(), - DAY - ) - } - } - .toList() - .also { deferredQueries.addAll(it) } - } - - // execute the query plan - try { - deferredQueries.awaitAll() - } catch (e: Exception) { - // For an error we clear the cache to try again - keyCache.clear() - throw e - } - - keyCache.getFilesFromEntries() - .also { it.forEach { file -> Timber.v("cached file:${file.path}") } } - } - - /** - * Fetches files given by serverDates by respecting countries - * @param currentDate base for where only dates within 3 hours before will be fetched - * @param serverDates pair of dates per country code - */ - private suspend fun asyncHandleLast3HoursFilesFetch( - currentDate: Date, - serverDates: List - ): List = withContext(Dispatchers.IO) { - Timber.v("Last 3 Hours will be Fetched. Only use for Debugging!") - val currentDateServerFormat = currentDate.toServerFormat() - - // just fetch the hours if the date is available - // extend fetch for all dates in all countries provided in serverDates - val packagesWithCurrentDate = serverDates - .filter { countryWithDates -> - countryWithDates.dates.contains(currentDateServerFormat) - } - - if (packagesWithCurrentDate.isEmpty()) { - throw IllegalStateException( - "you cannot use the last 3 hour mode if the date index " + - "does not contain any data for today" - ) - } - - return@withContext serverDates - .flatMap { countryWithDates -> - getLast3Hours(currentDate) - .map { hour -> - countryWithDates.getURLForHour(currentDate.toServerFormat(), hour) - } - .map { url -> - async { - return@async WebRequestBuilder.getInstance() - .asyncGetKeyFilesFromServer(url) - } - } - } - .awaitAll() - } - - /** - * Calculates the missing days based on current missing entries in the cache - * with respect to all countries defined - */ - private suspend fun getMissingDaysFromDiff( - serverCountryData: Collection - ): Collection { - val cacheEntries = keyCache.getDates() - return serverCountryData - .also { Timber.d("Server country data: %s", it) } - .map { availCountry -> - CountryDataWrapper( - availCountry.country, - availCountry.getMissingDates(cacheEntries) - ) - } - .filter { countryData -> - // Only return countries with missing dates. - countryData.dates.isNotEmpty() - } - .also { Timber.d("Locally missing country data: %s", it) } - } - - private const val LATEST_HOURS_NEEDED = 3 - - /** - * Calculates the last 3 hours - */ - private suspend fun getLast3Hours(day: Date): List = getHoursFromServer(day) - .also { Timber.v("${it.size} hours from server, but only latest 3 hours needed") } - .filter { TimeAndDateExtensions.getCurrentHourUTC() - LATEST_HOURS_NEEDED <= it.toInt() } - .toList() - .also { Timber.d("${it.size} missing hours") } - - /** - * Generates a unique key name (UUIDv3) for the cache entry based out of a string (e.g. an url) - */ - fun String.generateCacheKeyFromString() = - "${UUID.nameUUIDFromBytes(this.toByteArray())}".also { - Timber.v("$this mapped to cache entry $it") - } - - /** - * Get all dates of a country from server based as formatted dates - * - * @param country the country where the dates are got from - */ - private suspend fun getDatesFromServer(country: String) = - WebRequestBuilder.getInstance().asyncGetDateIndex(country) - - /** - * Get all hours from server based as formatted dates - */ - private suspend fun getHoursFromServer(day: Date) = - WebRequestBuilder.getInstance().asyncGetHourIndex(day) - - /** - * Get all countries from the server - */ - private suspend fun getCountriesFromServer(countries: List) = - WebRequestBuilder.getInstance().asyncGetCountryIndex(countries) -} - -internal data class CountryDataWrapper(val country: String, val dates: Collection) { - - /** - * Return a filtered list that contains all dates which are part of this wrapper, but not in the parameter. - */ - fun getMissingDates(cachedKeys: Collection): Collection { - val cacheIds = cachedKeys.map { it.id } - - return dates.filterNot { date -> - val formattedDateUrl = getURLForDay(date) - val cacheKeyFromUrl = formattedDateUrl.generateCacheKeyFromString() - // If the cache doesn't contain our ID, it's a missing date - cacheIds.contains(cacheKeyFromUrl) - } - } - - /** - * Gets the correct URL String for querying a day bucket - * - * @param formattedDate the formatted date - */ - fun getURLForDay(formattedDate: String) = - "${DiagnosisKeyConstants.AVAILABLE_COUNTRIES_URL}/$country/${DiagnosisKeyConstants.DATE}/$formattedDate" - - /** - * Gets the correct URL String for querying an hour bucket - * - * @param formattedDate the formatted date for the hour bucket request - * @param formattedHour the formatted hour - */ - fun getURLForHour(formattedDate: String, formattedHour: String) = - "${getURLForDay(formattedDate)}/${DiagnosisKeyConstants.HOUR}/$formattedHour" -} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt new file mode 100644 index 00000000000..6c1a64b76a1 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt @@ -0,0 +1,40 @@ +package de.rki.coronawarnapp.util + +import java.io.File +import java.security.MessageDigest +import java.util.Locale + +internal object HashExtensions { + + fun String.toSHA256() = this.hashString("SHA-256") + + fun String.toSHA1() = this.hashString("SHA-1") + + fun String.toMD5() = this.hashString("MD5") + + private fun ByteArray.formatHash(): String = this + .joinToString(separator = "") { String.format("%02X", it) } + .toLowerCase(Locale.ROOT) + + private fun String.hashString(type: String): String = MessageDigest + .getInstance(type) + .digest(this.toByteArray()) + .formatHash() + + fun File.hashToMD5(): String = this.hashTo("MD5") + + private fun File.hashTo(type: String): String = MessageDigest + .getInstance(type) + .let { md -> + inputStream().use { stream -> + val buffer = ByteArray(8192) + var read: Int + while (stream.read(buffer).also { read = it } > 0) { + md.update(buffer, 0, read) + } + } + md.digest() + } + .formatHash() + +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt new file mode 100644 index 00000000000..29dae7d3e37 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt @@ -0,0 +1,13 @@ +package de.rki.coronawarnapp.util + +import org.joda.time.Instant +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class TimeStamper @Inject constructor() { + + val nowUTC: Instant + get() = Instant.now() + +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/database/CommonConverters.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/database/CommonConverters.kt new file mode 100644 index 00000000000..cbbf8366f3d --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/database/CommonConverters.kt @@ -0,0 +1,81 @@ +/****************************************************************************** + * Corona-Warn-App * + * * + * SAP SE and all other contributors / * + * copyright owners license this file to you under the Apache * + * License, Version 2.0 (the "License"); you may not use this * + * file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ******************************************************************************/ + +package de.rki.coronawarnapp.util.database + +import androidx.room.TypeConverter +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import org.joda.time.Instant +import org.joda.time.LocalDate +import org.joda.time.LocalTime +import java.io.File +import java.util.UUID + +class CommonConverters { + private val gson = Gson() + + @TypeConverter + fun toIntList(value: String?): List { + val listType = object : TypeToken?>() {}.type + return gson.fromJson(value, listType) + } + + @TypeConverter + fun fromIntList(list: List?): String { + return gson.toJson(list) + } + + @TypeConverter + fun toUUID(value: String?): UUID? = value?.let { UUID.fromString(it) } + + @TypeConverter + fun fromUUID(uuid: UUID?): String? = uuid?.toString() + + @TypeConverter + fun toPath(value: String?): File? = value?.let { File(it) } + + @TypeConverter + fun fromPath(path: File?): String? = path?.path + + @TypeConverter + fun toLocalDate(value: String?): LocalDate? = value?.let { LocalDate.parse(it) } + + @TypeConverter + fun fromLocalDate(date: LocalDate?): String? = date?.toString() + + @TypeConverter + fun toLocalTime(value: String?): LocalTime? = value?.let { LocalTime.parse(it) } + + @TypeConverter + fun fromLocalTime(date: LocalTime?): String? = date?.toString() + + @TypeConverter + fun toInstant(value: String?): Instant? = value?.let { Instant.parse(it) } + + @TypeConverter + fun fromInstant(date: Instant?): String? = date?.toString() + + @TypeConverter + fun toLocationCode(value: String?): LocationCode? = value?.let { LocationCode(it) } + + @TypeConverter + fun fromLocationCode(code: LocationCode?): String? = code?.identifier +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/debug/TimeMeasurement.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/debug/TimeMeasurement.kt new file mode 100644 index 00000000000..58c06b520ba --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/debug/TimeMeasurement.kt @@ -0,0 +1,7 @@ +package de.rki.coronawarnapp.util.debug + +inline fun measureTimeMillisWithResult(block: () -> T): Pair { + val start = System.currentTimeMillis() + val result = block() + return result to (System.currentTimeMillis() - start) +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt index b175c93c808..79506c91d57 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt @@ -5,6 +5,12 @@ import dagger.Component import dagger.android.AndroidInjector import dagger.android.support.AndroidSupportInjectionModule import de.rki.coronawarnapp.CoronaWarnApplication +import de.rki.coronawarnapp.diagnosiskeys.DiagnosisKeysModule +import de.rki.coronawarnapp.diagnosiskeys.download.KeyFileDownloader +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer +import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import de.rki.coronawarnapp.http.HttpModule +import de.rki.coronawarnapp.http.ServiceFactory import de.rki.coronawarnapp.receiver.ReceiverBinder import de.rki.coronawarnapp.risk.RiskModule import de.rki.coronawarnapp.service.ServiceBinder @@ -28,7 +34,9 @@ import javax.inject.Singleton ActivityBinder::class, RiskModule::class, UtilModule::class, - DeviceModule::class + DeviceModule::class, + HttpModule::class, + DiagnosisKeysModule::class ] ) interface ApplicationComponent : AndroidInjector { @@ -42,6 +50,12 @@ interface ApplicationComponent : AndroidInjector { val settingsRepository: SettingsRepository + val keyCacheRepository: KeyCacheRepository + val keyFileDownloader: KeyFileDownloader + val serviceFactory: ServiceFactory + + val downloadServer: DownloadServer + @Component.Factory interface Factory { fun create(@BindsInstance app: CoronaWarnApplication): ApplicationComponent diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/security/VerificationKeys.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/security/VerificationKeys.kt index b70af82ba6d..03ec87d302f 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/security/VerificationKeys.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/security/VerificationKeys.kt @@ -8,8 +8,11 @@ import timber.log.Timber import java.security.KeyFactory import java.security.Signature import java.security.spec.X509EncodedKeySpec +import javax.inject.Inject +import javax.inject.Singleton -class VerificationKeys { +@Singleton +class VerificationKeys @Inject constructor() { companion object { private const val KEY_DELIMITER = "," } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt new file mode 100644 index 00000000000..bb10f86d968 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt @@ -0,0 +1,74 @@ +package de.rki.coronawarnapp.diagnosiskeys.download + +import org.junit.jupiter.api.Test +import testhelpers.BaseTest + +class CountryDataTest : BaseTest() { + + + // @Test +// fun testGetMissingDaysFromDiff() { +// val c1 = KeyCacheEntity() +// c1.id = "10008bf0-8890-356d-a4a4-dc375553160a" +// c1.path = +// "/data/user/0/de.rki.coronawarnapp.dev/cache/key-export/10008bf0-8890-356d-a4a4-dc375553160a.zip" +// c1.type = KeyCacheRepository.DateEntryType.DAY.ordinal +// +// val c2 = KeyCacheEntity() +// c2.id = "a8cc7b31-843e-3924-b918-023c386aec69" +// c2.path = +// "/data/user/0/de.rki.coronawarnapp.dev/cache/key-export/a8cc7b31-843e-3924-b918-023c386aec69.zip" +// c2.type = KeyCacheRepository.DateEntryType.DAY.ordinal +// +// val cacheEntries: Collection = listOf(c1, c2) +// +// val countryDataWrapper = +// CountryDataWrapper("DE", listOf("2020-08-29", "2020-08-26", "2020-08-28")) +// +// val result = countryDataWrapper.getMissingDates(cacheEntries) +// +// result.size shouldBe 1 +// result.elementAt(0) shouldBe "2020-08-28" +// } + + @Test + fun `missing hours default`() { + TODO() + } + + @Test + fun `missing hours empty`() { + TODO() + } + + @Test + fun `missing hours disjunct`() { + TODO() + } + + @Test + fun `missing hours none missing`() { + TODO() + } + + @Test + fun `missing days default`() { + TODO() + } + + @Test + fun `missing days empty`() { + TODO() + } + + @Test + fun `missing days disjunct`() { + TODO() + } + + @Test + fun `missing days none missing`() { + TODO() + } + +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt new file mode 100644 index 00000000000..87d38e2878f --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -0,0 +1,162 @@ +package de.rki.coronawarnapp.diagnosiskeys.download + +import android.content.Context +import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import io.mockk.impl.annotations.MockK +import org.junit.jupiter.api.Test +import testhelpers.BaseTest + +/** + * CachedKeyFileHolder test. + */ +class KeyFileDownloaderTest : BaseTest() { + + @MockK + private lateinit var keyCacheRepository: KeyCacheRepository + + @MockK + private lateinit var context: Context + +// @Before +// fun setUp() { +// MockKAnnotations.init(this) +// mockkObject(CoronaWarnApplication.Companion) +// mockkObject(KeyCacheRepository.Companion) +// every { CoronaWarnApplication.getAppContext() } returns context +// every { KeyCacheRepository.getDateRepository(any()) } returns keyCacheRepository +// mockkObject(KeyFileDownloader) +// coEvery { keyCacheRepository.deleteOutdatedEntries(any()) } just Runs +// } +// +// /** +// * Test call order is correct. +// */ +// @Test +// fun testAsyncFetchFiles() { +// val date = Date() +// val countries = listOf("DE") +// val country = "DE" +// +// mockkObject(CWADebug) +// +// coEvery { keyCacheRepository.getDates() } returns listOf() +// coEvery { keyCacheRepository.getFilesFromEntries() } returns listOf() +// every { CWADebug.isDebugBuildOrMode } returns false +// every { KeyFileDownloader["checkForFreeSpace"]() } returns Unit +// every { KeyFileDownloader["getDatesFromServer"](country) } returns arrayListOf() +// +// every { CoronaWarnApplication.getAppContext().cacheDir } returns File("./") +// every { KeyFileDownloader["getCountriesFromServer"](countries) } returns countries +// +// runBlocking { +// +// KeyFileDownloader.asyncFetchFiles(date, countries) +// +// coVerifyOrder { +// KeyFileDownloader.asyncFetchFiles(date, countries) +// KeyFileDownloader["getCountriesFromServer"](countries) +// KeyFileDownloader["getDatesFromServer"](country) +// KeyFileDownloader["asyncHandleFilesFetch"]( +// listOf( +// CountryDataWrapper( +// country, +// listOf() +// ) +// ) +// ) +// keyCacheRepository.deleteOutdatedEntries(any()) +// KeyFileDownloader["getMissingDaysFromDiff"]( +// listOf( +// CountryDataWrapper( +// country, +// listOf() +// ) +// ) +// ) +// keyCacheRepository.getDates() +// keyCacheRepository.getFilesFromEntries() +// } +// } +// } + +// +// @After +// fun cleanUp() { +// unmockkAll() +// } + + @Test + fun `error during country index fetch`() { + TODO() + } + + @Test + fun `fetched country index is empty`() { + TODO() + } + + @Test + fun `day fetch without prior data`() { + TODO() + } + + @Test + fun `day fetch with existing data`() { + TODO() + } + + @Test + fun `day fetch deletes stale data`() { + TODO() + } + + @Test + fun `day fetch marks downloads as complete`() { + TODO() + } + + @Test + fun `day fetch skips single download failures`() { + TODO() + } + + @Test + fun `last3Hours fetch without prior data`() { + TODO() + } + + @Test + fun `last3Hours fetch with prior data`() { + TODO() + } + + @Test + fun `last3Hours fetch deletes stale data`() { + TODO() + } + + @Test + fun `last3Hours fetch marks downloads as complete`() { + TODO() + } + + @Test + fun `last3Hours fetch skips single download failures`() { + TODO() + } + + @Test + fun `storage is checked before fetching`() { + TODO() + } + + @Test + fun `fetching is aborted if not enough free storage`() { + TODO() + } + + @Test + fun `not completed cache entries are overwritten`() { + TODO() + } +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt new file mode 100644 index 00000000000..428a1a7d0ac --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt @@ -0,0 +1,18 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest + +class CDNModuleTest : BaseIOTest() { + + @Test + fun `home country should be DE`() { + TODO() + } + + @Test + fun `CDN URL comes from BuildConfig`() { + TODO() + } + +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt new file mode 100644 index 00000000000..49d46164239 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt @@ -0,0 +1,37 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest + +class CDNServerTest : BaseIOTest() { + + @Test + fun `application config download`() { + TODO() + } + + @Test + fun `download key files for day`() { + TODO() + } + + @Test + fun `download key files for hour`() { + TODO() + } + + @Test + fun `download country index`() { + TODO() + } + + @Test + fun `download day index for country`() { + TODO() + } + + @Test + fun `download hour index for country and day`() { + TODO() + } +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt new file mode 100644 index 00000000000..4f8bd2a714c --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt @@ -0,0 +1,27 @@ +package de.rki.coronawarnapp.diagnosiskeys.storage + +import org.junit.jupiter.api.Test +import testhelpers.BaseTest + +class CachedKeyFileTest : BaseTest() { + @Test + fun `secondary constructor`() { + TODO() + } + + @Test + fun `keyfile id calculation`() { + TODO() + } + + @Test + fun `to completion`() { + TODO() + } + + @Test + fun `trip changed typing`() { + // Let this test fail incase someone accidentally changes those values. + TODO() + } +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt new file mode 100644 index 00000000000..dde22a05855 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt @@ -0,0 +1,142 @@ +package de.rki.coronawarnapp.diagnosiskeys.storage + +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheDao +import io.mockk.impl.annotations.MockK +import org.junit.jupiter.api.Test +import testhelpers.BaseTest + +class KeyCacheRepositoryTest : BaseTest() { + + @MockK + private lateinit var keyCacheDao: KeyCacheDao + + private lateinit var keyCacheRepository: KeyCacheRepository + +// @Before +// fun setUp() { +// MockKAnnotations.init(this) +// keyCacheRepository = KeyCacheRepository(keyCacheDao) +// +// // DAO tests in another test +// coEvery { keyCacheDao.getAllEntries() } returns listOf() +// coEvery { keyCacheDao.getHours() } returns listOf() +// coEvery { keyCacheDao.clear() } just Runs +// coEvery { keyCacheDao.clearHours() } just Runs +// coEvery { keyCacheDao.insertEntry(any()) } returns 0 +// } +// +// /** +// * Test clear order. +// */ +// @Test +// fun testClear() { +// runBlocking { +// keyCacheRepository.clear() +// +// coVerifyOrder { +// keyCacheDao.getAllEntries() +// +// keyCacheDao.clear() +// } +// } +// +// runBlocking { +// keyCacheRepository.clearHours() +// +// coVerifyOrder { +// keyCacheDao.getHours() +// +// keyCacheDao.clearHours() +// } +// } +// } +// +// /** +// * Test insert order. +// */ +// @Test +// fun testInsert() { +// runBlocking { +// keyCacheRepository.createCacheEntry( +// key = "1", +// type = KeyCacheRepository.DateEntryType.DAY, +// uri = URI("1") +// ) +// +// coVerify { +// keyCacheDao.insertEntry(any()) +// } +// } +// } +// +// @After +// fun cleanUp() { +// unmockkAll() +// } + + @Test + fun `migration of old data`() { + TODO() + } + + @Test + fun `migration does nothing when there is no old data`() { + TODO() + } + + @Test + fun `migration consumes old data and runs only once`() { + TODO() + } + + @Test + fun `migration runs before creation`() { + TODO() + } + + @Test + fun `migration runs before download update`() { + TODO() + } + + @Test + fun `health check runs before data creation`() { + TODO() + } + + @Test + fun `health check runs before data update`() { + TODO() + } + + @Test + fun `insert and retrieve`() { + TODO() + } + + @Test + fun `update download state`() { + TODO() + } + + @Test + fun `delete only selected entries`() { + TODO() + } + + @Test + fun `clear all files`() { + TODO() + } + + @Test + fun `path is based on private cache dir`() { + TODO() + } + + @Test + fun `check for missing key files and change download state`() { + TODO() + } + +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt index 0f63f670412..3bcbdf82600 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt @@ -1,6 +1,5 @@ package de.rki.coronawarnapp.http -import de.rki.coronawarnapp.http.service.DistributionService import de.rki.coronawarnapp.http.service.SubmissionService import de.rki.coronawarnapp.http.service.VerificationService import de.rki.coronawarnapp.service.diagnosiskey.DiagnosisKeyConstants @@ -12,8 +11,9 @@ import io.mockk.coVerify import io.mockk.impl.annotations.MockK import io.mockk.unmockkAll import kotlinx.coroutines.runBlocking +import org.joda.time.LocalDate +import org.joda.time.LocalTime import org.junit.After -import org.junit.Assert import org.junit.Before import org.junit.Test import java.util.Date @@ -50,15 +50,18 @@ class WebRequestBuilderTest { @Test fun retrievingDateIndexIsSuccessful() { val urlString = DiagnosisKeyConstants.AVAILABLE_DATES_URL - coEvery { distributionService.getDateIndex(urlString) } - .returns(listOf("1900-01-01", "2000-01-01")) + coEvery { distributionService.getDateIndex(urlString) } returns listOf( + "1900-01-01", + "2000-01-01" + ) runBlocking { - val expectedResult = listOf("1900-01-01", "2000-01-01") - Assert.assertEquals(webRequestBuilder.asyncGetDateIndex("DE"), expectedResult) - coVerify { - distributionService.getDateIndex(urlString) - } + webRequestBuilder.asyncGetDateIndex("DE") shouldBe listOf( + LocalDate.parse("1900-01-01"), + LocalDate.parse("2000-01-01") + ) + + coVerify(exactly = 1) { distributionService.getDateIndex(urlString) } } } @@ -68,15 +71,15 @@ class WebRequestBuilderTest { val urlString = DiagnosisKeyConstants.AVAILABLE_DATES_URL + "/${day.toServerFormat()}/${DiagnosisKeyConstants.HOUR}" - coEvery { distributionService.getHourIndex(urlString) } - .returns(listOf("1", "2")) + coEvery { distributionService.getHourIndex(urlString) } returns listOf("1", "2") runBlocking { - val expectedResult = listOf("1", "2") - Assert.assertEquals(webRequestBuilder.asyncGetHourIndex(day), expectedResult) - coVerify { - distributionService.getHourIndex(urlString) - } + webRequestBuilder.asyncGetHourIndex(day) shouldBe listOf( + LocalTime.parse("01:00"), + LocalTime.parse("02:00") + ) + + coVerify(exactly = 1) { distributionService.getHourIndex(urlString) } } } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstantsTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstantsTest.kt index a7a85c26881..2bb68c7fbbf 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstantsTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/diagnosiskey/DiagnosisKeyConstantsTest.kt @@ -7,17 +7,7 @@ class DiagnosisKeyConstantsTest { @Test fun allDiagnosisKeyConstants() { - Assert.assertEquals(DiagnosisKeyConstants.HOUR, "hour") Assert.assertEquals(DiagnosisKeyConstants.SERVER_ERROR_CODE_403, 403) - Assert.assertEquals(DiagnosisKeyConstants.INDEX_DOWNLOAD_URL, "version/v1/index.txt") - Assert.assertEquals(DiagnosisKeyConstants.DIAGNOSIS_KEYS_DOWNLOAD_URL, "version/v1/diagnosis-keys") Assert.assertEquals(DiagnosisKeyConstants.DIAGNOSIS_KEYS_SUBMISSION_URL, "version/v1/diagnosis-keys") - Assert.assertEquals(DiagnosisKeyConstants.PARAMETERS_COUNTRY_DOWNLOAD_URL, "version/v1/parameters/country") - Assert.assertEquals(DiagnosisKeyConstants.APPCONFIG_COUNTRY_DOWNLOAD_URL, "version/v1/configuration/country") - Assert.assertEquals( - DiagnosisKeyConstants.COUNTRY_APPCONFIG_DOWNLOAD_URL, - "version/v1/configuration/country/DE/app_config" - ) - Assert.assertEquals(DiagnosisKeyConstants.AVAILABLE_DATES_URL, "version/v1/diagnosis-keys/country/DE/date") } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepositoryTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepositoryTest.kt deleted file mode 100644 index ea58f3a5f15..00000000000 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepositoryTest.kt +++ /dev/null @@ -1,88 +0,0 @@ -package de.rki.coronawarnapp.storage.keycache - -import io.mockk.MockKAnnotations -import io.mockk.Runs -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.coVerifyOrder -import io.mockk.impl.annotations.MockK -import io.mockk.just -import io.mockk.unmockkAll -import kotlinx.coroutines.runBlocking -import org.junit.After -import org.junit.Before -import org.junit.Test -import java.net.URI - -/** - * KeyCacheRepository test. - */ -class KeyCacheRepositoryTest { - - @MockK - private lateinit var keyCacheDao: KeyCacheDao - - private lateinit var keyCacheRepository: KeyCacheRepository - - @Before - fun setUp() { - MockKAnnotations.init(this) - keyCacheRepository = KeyCacheRepository(keyCacheDao) - - // DAO tests in another test - coEvery { keyCacheDao.getAllEntries() } returns listOf() - coEvery { keyCacheDao.getHours() } returns listOf() - coEvery { keyCacheDao.clear() } just Runs - coEvery { keyCacheDao.clearHours() } just Runs - coEvery { keyCacheDao.insertEntry(any()) } returns 0 - } - - /** - * Test clear order. - */ - @Test - fun testClear() { - runBlocking { - keyCacheRepository.clear() - - coVerifyOrder { - keyCacheDao.getAllEntries() - - keyCacheDao.clear() - } - } - - runBlocking { - keyCacheRepository.clearHours() - - coVerifyOrder { - keyCacheDao.getHours() - - keyCacheDao.clearHours() - } - } - } - - /** - * Test insert order. - */ - @Test - fun testInsert() { - runBlocking { - keyCacheRepository.createEntry( - key = "1", - type = KeyCacheRepository.DateEntryType.DAY, - uri = URI("1") - ) - - coVerify { - keyCacheDao.insertEntry(any()) - } - } - } - - @After - fun cleanUp() { - unmockkAll() - } -} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt index f5da25fac19..a378dc9cf4b 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt @@ -1,14 +1,9 @@ package de.rki.coronawarnapp.transaction import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration -import de.rki.coronawarnapp.http.WebRequestBuilder import de.rki.coronawarnapp.nearby.InternalExposureNotificationClient -import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigurationService import de.rki.coronawarnapp.storage.LocalData -import de.rki.coronawarnapp.util.CWADebug -import de.rki.coronawarnapp.util.CachedKeyFileHolder -import io.kotest.matchers.shouldBe import de.rki.coronawarnapp.util.di.AppInjector import de.rki.coronawarnapp.util.di.ApplicationComponent import io.mockk.Runs diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt deleted file mode 100644 index a9a1e502d98..00000000000 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/CachedKeyFileHolderTest.kt +++ /dev/null @@ -1,126 +0,0 @@ -package de.rki.coronawarnapp.util - -import android.content.Context -import de.rki.coronawarnapp.CoronaWarnApplication -import de.rki.coronawarnapp.storage.keycache.KeyCacheEntity -import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository -import io.kotest.matchers.shouldBe -import io.mockk.MockKAnnotations -import io.mockk.Runs -import io.mockk.coEvery -import io.mockk.coVerifyOrder -import io.mockk.every -import io.mockk.impl.annotations.MockK -import io.mockk.just -import io.mockk.mockkObject -import io.mockk.unmockkAll -import kotlinx.coroutines.runBlocking -import org.junit.After -import org.junit.Before -import org.junit.Test -import java.io.File -import java.util.Date - -/** - * CachedKeyFileHolder test. - */ -class CachedKeyFileHolderTest { - - @MockK - private lateinit var keyCacheRepository: KeyCacheRepository - - @MockK - private lateinit var context: Context - - @Before - fun setUp() { - MockKAnnotations.init(this) - mockkObject(CoronaWarnApplication.Companion) - mockkObject(KeyCacheRepository.Companion) - every { CoronaWarnApplication.getAppContext() } returns context - every { KeyCacheRepository.getDateRepository(any()) } returns keyCacheRepository - mockkObject(CachedKeyFileHolder) - coEvery { keyCacheRepository.deleteOutdatedEntries(any()) } just Runs - } - - /** - * Test call order is correct. - */ - @Test - fun testAsyncFetchFiles() { - val date = Date() - val countries = listOf("DE") - val country = "DE" - - mockkObject(CWADebug) - - coEvery { keyCacheRepository.getDates() } returns listOf() - coEvery { keyCacheRepository.getFilesFromEntries() } returns listOf() - every { CWADebug.isDebugBuildOrMode } returns false - every { CachedKeyFileHolder["checkForFreeSpace"]() } returns Unit - every { CachedKeyFileHolder["getDatesFromServer"](country) } returns arrayListOf() - - every { CoronaWarnApplication.getAppContext().cacheDir } returns File("./") - every { CachedKeyFileHolder["getCountriesFromServer"](countries) } returns countries - - runBlocking { - - CachedKeyFileHolder.asyncFetchFiles(date, countries) - - coVerifyOrder { - CachedKeyFileHolder.asyncFetchFiles(date, countries) - CachedKeyFileHolder["getCountriesFromServer"](countries) - CachedKeyFileHolder["getDatesFromServer"](country) - CachedKeyFileHolder["asyncHandleFilesFetch"]( - listOf( - CountryDataWrapper( - country, - listOf() - ) - ) - ) - keyCacheRepository.deleteOutdatedEntries(any()) - CachedKeyFileHolder["getMissingDaysFromDiff"]( - listOf( - CountryDataWrapper( - country, - listOf() - ) - ) - ) - keyCacheRepository.getDates() - keyCacheRepository.getFilesFromEntries() - } - } - } - - @Test - fun testGetMissingDaysFromDiff() { - val c1 = KeyCacheEntity() - c1.id = "10008bf0-8890-356d-a4a4-dc375553160a" - c1.path = - "/data/user/0/de.rki.coronawarnapp.dev/cache/key-export/10008bf0-8890-356d-a4a4-dc375553160a.zip" - c1.type = KeyCacheRepository.DateEntryType.DAY.ordinal - - val c2 = KeyCacheEntity() - c2.id = "a8cc7b31-843e-3924-b918-023c386aec69" - c2.path = - "/data/user/0/de.rki.coronawarnapp.dev/cache/key-export/a8cc7b31-843e-3924-b918-023c386aec69.zip" - c2.type = KeyCacheRepository.DateEntryType.DAY.ordinal - - val cacheEntries: Collection = listOf(c1, c2) - - val countryDataWrapper = - CountryDataWrapper("DE", listOf("2020-08-29", "2020-08-26", "2020-08-28")) - - val result = countryDataWrapper.getMissingDates(cacheEntries) - - result.size shouldBe 1 - result.elementAt(0) shouldBe "2020-08-28" - } - - @After - fun cleanUp() { - unmockkAll() - } -} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/HashExtensionsTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/HashExtensionsTest.kt new file mode 100644 index 00000000000..186d2973e59 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/HashExtensionsTest.kt @@ -0,0 +1,60 @@ +package de.rki.coronawarnapp.util + +import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 +import de.rki.coronawarnapp.util.HashExtensions.toMD5 +import de.rki.coronawarnapp.util.HashExtensions.toSHA1 +import de.rki.coronawarnapp.util.HashExtensions.toSHA256 +import io.kotest.matchers.shouldBe +import io.mockk.clearAllMocks +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest +import java.io.File + +class HashExtensionsTest : BaseIOTest() { + + private val testInput = "The Cake Is A Lie" + private val testDir = File(IO_TEST_BASEDIR, this::class.simpleName!!) + + @BeforeEach + fun setup() { + testDir.mkdirs() + } + + @AfterEach + fun teardown() { + clearAllMocks() + + testDir.deleteRecursively() + } + + @Test + fun `hash string to MD5`() { + testInput.toMD5() shouldBe "e42997e37d8d70d4927b0b396254c179" + } + + @Test + fun `hash string to SHA256`() { + testInput.toSHA256() shouldBe "3afc82e0c5df81d1733fe0c289538a1a1f7a5038d5c261860a5c83952f4bcb61" + } + + @Test + fun `hash string to SHA1`() { + testInput.toSHA1() shouldBe "4d57f806e5f714ebdb5a74a12fda9523fae21d76" + } + + @Test + fun `hash file to md5`() { + val fileName = "FileToMD5.txt" + val testFile = File(testDir, fileName) + try { + testFile.printWriter().use { out -> + out.println("This is a test") + } + testFile.hashToMD5() shouldBe "ff22941336956098ae9a564289d1bf1b" + } finally { + testFile.delete() + } + } +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/MockWebServerUtil.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/MockWebServerUtil.kt index 5a908210db8..0dd704f7313 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/MockWebServerUtil.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/MockWebServerUtil.kt @@ -3,10 +3,8 @@ package de.rki.coronawarnapp.util import de.rki.coronawarnapp.http.HttpErrorParser import de.rki.coronawarnapp.http.WebRequestBuilder import de.rki.coronawarnapp.http.interceptor.RetryInterceptor -import de.rki.coronawarnapp.http.service.DistributionService import de.rki.coronawarnapp.http.service.SubmissionService import de.rki.coronawarnapp.http.service.VerificationService -import de.rki.coronawarnapp.util.security.VerificationKeys import okhttp3.OkHttpClient import okhttp3.logging.HttpLoggingInterceptor import okhttp3.mockwebserver.MockWebServer @@ -29,13 +27,10 @@ fun MockWebServer.newWebRequestBuilder(): WebRequestBuilder { .addConverterFactory(GsonConverterFactory.create()) return WebRequestBuilder( - retrofit.baseUrl(this.url("/distribution/")).build() - .create(DistributionService::class.java), retrofit.baseUrl(this.url("/verification/")).build() .create(VerificationService::class.java), retrofit.baseUrl(this.url("/submission/")).build() - .create(SubmissionService::class.java), - VerificationKeys() + .create(SubmissionService::class.java) ) } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/Converters.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt similarity index 72% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/Converters.kt rename to Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt index 7b5c36d3937..28d4e67da46 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/Converters.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt @@ -17,23 +17,45 @@ * under the License. * ******************************************************************************/ -package de.rki.coronawarnapp.util +package de.rki.coronawarnapp.util.database -import androidx.room.TypeConverter -import com.google.gson.Gson -import com.google.gson.reflect.TypeToken +import org.junit.jupiter.api.Test -class Converters { - private val gson = Gson() +class CommonConvertersTest { - @TypeConverter - fun fromString(value: String?): List { - val listType = object : TypeToken?>() {}.type - return gson.fromJson(value, listType) + @Test + fun `int list conversion`() { + TODO() } - @TypeConverter - fun fromArrayList(list: List?): String { - return gson.toJson(list) + @Test + fun `UUID conversion`() { + TODO() + } + + + @Test + fun `path conversion`() { + TODO() + } + + @Test + fun `local date conversion`() { + TODO() + } + + @Test + fun `local time conversion`() { + TODO() + } + + @Test + fun `instant conversion`() { + TODO() + } + + @Test + fun `LocationCode conversion`() { + TODO() } } From c67eaf9b68efe220e2578593a56be5bb3fc154b3 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Tue, 8 Sep 2020 18:22:56 +0200 Subject: [PATCH 02/29] First batch of unit tests and some fixes for incorrect behavior that the tests surfaced. --- Corona-Warn-App/build.gradle | 2 + .../storage/KeyCacheDatabaseTest.kt | 40 +++ .../util/security/VerificationKeysTest.kt | 71 +++++ .../TestForAPIFragment.kt | 2 +- .../diagnosiskeys/DiagnosisKeysModule.kt | 19 +- .../diagnosiskeys/download/CountryData.kt | 10 +- .../download/KeyFileDownloader.kt | 56 ++-- .../diagnosiskeys/server/DownloadApiV1.kt | 1 - .../diagnosiskeys/server/DownloadServer.kt | 19 +- .../{CachedKeyFile.kt => CachedKeyInfo.kt} | 6 +- .../diagnosiskeys/storage/KeyCacheDatabase.kt | 16 +- .../storage/KeyCacheRepository.kt | 68 +++-- .../RetrieveDiagnosisKeysTransaction.kt | 2 +- .../diagnosiskeys/DiagnosisKeysModuleTest.kt | 23 ++ .../diagnosiskeys/download/CountryDataTest.kt | 247 +++++++++++++--- .../download/KeyFileDownloaderTest.kt | 5 + .../diagnosiskeys/server/CDNModuleTest.kt | 18 -- .../diagnosiskeys/server/CDNServerTest.kt | 37 --- .../diagnosiskeys/server/DownloadAPITest.kt | 142 +++++++++ .../server/DownloadServerTest.kt | 235 +++++++++++++++ .../storage/CachedKeyFileTest.kt | 45 ++- .../storage/KeyCacheRepositoryTest.kt | 272 +++++++++++------- .../storage/legacy/KeyCacheEntityTest.kt | 5 + .../http/WebRequestBuilderTest.kt | 52 +--- .../ApplicationConfigurationServiceTest.kt | 14 +- .../util/database/CommonConvertersTest.kt | 60 +++- 26 files changed, 1123 insertions(+), 344 deletions(-) create mode 100644 Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt create mode 100644 Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/VerificationKeysTest.kt rename Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/{CachedKeyFile.kt => CachedKeyInfo.kt} (94%) create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModuleTest.kt delete mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt delete mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntityTest.kt diff --git a/Corona-Warn-App/build.gradle b/Corona-Warn-App/build.gradle index 1d3229d5b65..384bcddf02b 100644 --- a/Corona-Warn-App/build.gradle +++ b/Corona-Warn-App/build.gradle @@ -276,6 +276,8 @@ dependencies { testImplementation "io.kotest:kotest-runner-junit5:4.2.0" testImplementation "io.kotest:kotest-assertions-core-jvm:4.2.0" testImplementation "io.kotest:kotest-property-jvm:4.2.0" + androidTestImplementation "io.kotest:kotest-assertions-core-jvm:4.2.0" + androidTestImplementation "io.kotest:kotest-property-jvm:4.2.0" // Testing - Instrumentation androidTestImplementation 'androidx.test:runner:1.2.0' diff --git a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt new file mode 100644 index 00000000000..a4591b02db4 --- /dev/null +++ b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt @@ -0,0 +1,40 @@ +package de.rki.coronawarnapp.diagnosiskeys.storage + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class KeyCacheDatabaseTest { + + @Test + fun createEntry() { + TODO() + } + + @Test + fun deleteEntry() { + TODO() + } + + @Test + fun clear() { + TODO() + } + + @Test + fun getAll() { + TODO() + } + + @Test + fun getAllDayData() { + TODO() + } + + @Test + fun getAllHourData() { + TODO() + } + +} diff --git a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/VerificationKeysTest.kt b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/VerificationKeysTest.kt new file mode 100644 index 00000000000..f43f22095c6 --- /dev/null +++ b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/VerificationKeysTest.kt @@ -0,0 +1,71 @@ +package de.rki.coronawarnapp.util.security + +import de.rki.coronawarnapp.exception.CwaSecurityException +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import okio.ByteString.Companion.decodeHex +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class VerificationKeysTest { + + private fun createTool() = VerificationKeys() + + @Test + fun goodBinaryAndSignature() { + val tool = createTool() + tool.hasInvalidSignature( + GOOD_BINARY.decodeHex().toByteArray(), + GOOD_SIGNATURE.decodeHex().toByteArray() + ) shouldBe false + } + + @Test + fun badBinaryGoodSignature() { + val tool = createTool() + tool.hasInvalidSignature( + "123ABC".decodeHex().toByteArray(), + GOOD_SIGNATURE.decodeHex().toByteArray() + ) shouldBe true + } + + @Test + fun goodBinaryBadSignature() { + val tool = createTool() + shouldThrow { + tool.hasInvalidSignature( + GOOD_BINARY.decodeHex().toByteArray(), + "123ABC".decodeHex().toByteArray() + ) + } + } + + @Test + fun badEverything() { + val tool = createTool() + shouldThrow { + tool.hasInvalidSignature( + "123ABC".decodeHex().toByteArray(), + "123ABC".decodeHex().toByteArray() + ) + } + } + + companion object { + private const val GOOD_BINARY = + "080b124d0a230a034c4f57180f221a68747470733a2f2f7777772e636f726f6e617761726e2e6170700a26" + + "0a0448494748100f1848221a68747470733a2f2f7777772e636f726f6e617761726e2e6170701a" + + "640a10080110021803200428053006380740081100000000000049401a0a200128013001380140" + + "012100000000000049402a10080510051805200528053005380540053100000000000034403a0e" + + "1001180120012801300138014001410000000000004940221c0a040837103f1212090000000000" + + "00f03f11000000000000e03f20192a1a0a0a0a041008180212021005120c0a0408011804120408" + + "011804" + private const val GOOD_SIGNATURE = + "0a87010a380a1864652e726b692e636f726f6e617761726e6170702d6465761a02763122033236322a1331" + + "2e322e3834302e31303034352e342e332e321001180122473045022100cf32ff24ea18a1ffcc7f" + + "f4c9fe8d1808cecbc5a37e3e1d4c9ce682120450958c022064bf124b6973a9b510a43d479ff93e" + + "0ef97a5b893c7af4abc4a8d399969cd8a0" + } +} diff --git a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt index ed9e923bafb..971aeb324b2 100644 --- a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt +++ b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt @@ -323,7 +323,7 @@ class TestForAPIFragment : Fragment(), InternalExposureNotificationPermissionHel // Trigger asyncFetchFiles which will use all Countries passed as parameter val currentDate = LocalDate.now() lifecycleScope.launch { - AppInjector.component.keyFileDownloader.asyncFetchFiles(currentDate, countryCodes) + AppInjector.component.keyFileDownloader.asyncFetchKeyFiles(currentDate, countryCodes) updateCountryStatusLabel() } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt index 1e1ddd71329..2f8c17f2daa 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt @@ -1,6 +1,5 @@ package de.rki.coronawarnapp.diagnosiskeys -import android.webkit.URLUtil import dagger.Module import dagger.Provides import dagger.Reusable @@ -24,7 +23,7 @@ class DiagnosisKeysModule { @Singleton @DownloadHomeCountry @Provides - fun provideCDNHomeCountry(): LocationCode = LocationCode("DE") + fun provideDiagnosisHomeCountry(): LocationCode = LocationCode("DE") @Reusable @DownloadHttpClient @@ -34,13 +33,13 @@ class DiagnosisKeysModule { @Singleton @Provides - fun cdnApi( - @DownloadHttpClient cdnHttpClient: OkHttpClient, - @DownloadServerUrl cdnUrl: String, + fun provideDownloadApi( + @DownloadHttpClient client: OkHttpClient, + @DownloadServerUrl url: String, gsonConverterFactory: GsonConverterFactory ): DownloadApiV1 = Retrofit.Builder() - .client(cdnHttpClient) - .baseUrl(cdnUrl) + .client(client) + .baseUrl(url) .addConverterFactory(gsonConverterFactory) .build() .create(DownloadApiV1::class.java) @@ -48,8 +47,10 @@ class DiagnosisKeysModule { @Singleton @DownloadServerUrl @Provides - fun provideCDNUrl(): String = BuildConfig.DOWNLOAD_CDN_URL.also { - if (!URLUtil.isHttpsUrl(it)) throw IllegalArgumentException("the url is invalid") + fun provideDownloadServerUrl(): String { + val url = BuildConfig.DOWNLOAD_CDN_URL + if (!url.startsWith("https://")) throw IllegalStateException("Innvalid: $url") + return url } companion object { diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt index 551f3be985e..01116c9c97a 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt @@ -1,7 +1,7 @@ package de.rki.coronawarnapp.diagnosiskeys.download import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode -import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyFile +import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import org.joda.time.LocalDate import org.joda.time.LocalTime @@ -19,7 +19,7 @@ internal data class CountryDays( /** * Return a filtered list that contains all dates which are part of this wrapper, but not in the parameter. */ - fun getMissingDays(cachedKeys: List): Collection? { + fun getMissingDays(cachedKeys: List): Collection? { val cachedCountryDates = cachedKeys .filter { it.location == country } .map { it.day } @@ -33,7 +33,7 @@ internal data class CountryDays( * Create a new country object that only contains those elements, * that are part of this wrapper, but not in the cache. */ - fun toMissingDays(cachedKeys: List): CountryDays? { + fun toMissingDays(cachedKeys: List): CountryDays? { val missingDays = this.getMissingDays(cachedKeys) if (missingDays == null || missingDays.isEmpty()) return null @@ -46,7 +46,7 @@ internal data class CountryHours( val hourData: Map> ) : CountryData() { - fun getMissingHours(cachedKeys: List): Map>? { + fun getMissingHours(cachedKeys: List): Map>? { val cachedHours = cachedKeys .filter { it.location == country } @@ -58,7 +58,7 @@ internal data class CountryHours( }.toMap() } - fun toMissingHours(cachedKeys: List): CountryHours? { + fun toMissingHours(cachedKeys: List): CountryHours? { val missingHours = this.getMissingHours(cachedKeys) if (missingHours == null || missingHours.isEmpty()) return null diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 24517d9394a..4d7d02456e1 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -22,7 +22,7 @@ package de.rki.coronawarnapp.diagnosiskeys.download import dagger.Reusable import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode -import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyFile +import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.storage.FileStorageHelper import de.rki.coronawarnapp.storage.LocalData @@ -62,7 +62,7 @@ class KeyFileDownloader @Inject constructor( * @param currentDate the current date - if this is adjusted by the calendar, the cache is affected. * @return list of all files from both the cache and the diff query */ - suspend fun asyncFetchFiles( + suspend fun asyncFetchKeyFiles( currentDate: LocalDate, countries: List ): List = withContext(Dispatchers.IO) { @@ -74,11 +74,25 @@ class KeyFileDownloader @Inject constructor( val availableCountries = downloadServer.getCountryIndex(countries) Timber.tag(TAG).v("Available server data: %s", availableCountries) - if (CWADebug.isDebugBuildOrMode && LocalData.last3HoursMode()) { - asyncHandleLast3HoursFilesFetch(currentDate, availableCountries) + val availableKeys = if (CWADebug.isDebugBuildOrMode && LocalData.last3HoursMode()) { + fetchMissing3Hours(currentDate, availableCountries) + keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) } else { - asyncHandleFilesFetch(availableCountries) + fetchMissingDays(availableCountries) + keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) } + + return@withContext availableKeys + .filter { it.first.isDownloadComplete && it.second.exists() } + .mapNotNull { (keyInfo, path) -> + if (!path.exists()) { + Timber.tag(TAG).w("Missing keyfile for : %s", keyInfo) + null + } else { + path + } + } + .also { Timber.tag(TAG).d("Returning %d available keyfiles", it.size) } } // TODO replace @@ -88,17 +102,18 @@ class KeyFileDownloader @Inject constructor( * Fetches files given by serverDates by respecting countries * @param availableCountries pair of dates per country code */ - private suspend fun asyncHandleFilesFetch( + private suspend fun fetchMissingDays( availableCountries: List - ): List = withContext(Dispatchers.IO) { + ) = withContext(Dispatchers.IO) { val availableCountriesWithDays = availableCountries.map { val days = downloadServer.getDayIndex(it) CountryDays(it, days) } val cachedDays = keyCache - .getEntriesForType(CachedKeyFile.Type.COUNTRY_DAY) - .filter { it.isDownloadComplete } // We overwrite not completed ones + .getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) + .filter { it.first.isDownloadComplete && it.second.exists() } // We overwrite not completed ones + .map { it.first } // All cached files that are no longer on the server are considered stale val staleKeyFiles = cachedDays.filter { cachedKeyFile -> @@ -136,7 +151,7 @@ class KeyFileDownloader @Inject constructor( location = countryWrapper.country, dayIdentifier = dayDate, hourIdentifier = null, - type = CachedKeyFile.Type.COUNTRY_DAY + type = CachedKeyInfo.Type.COUNTRY_DAY ) return@async try { @@ -159,10 +174,12 @@ class KeyFileDownloader @Inject constructor( (System.currentTimeMillis() - batchDownloadStart) ) - return@withContext downloadedDays.map { (keyInfo, path) -> + downloadedDays.map { (keyInfo, path) -> Timber.tag(TAG).v("Downloaded keyfile: %s to %s", keyInfo, path) path } + + Unit } /** @@ -170,10 +187,10 @@ class KeyFileDownloader @Inject constructor( * @param currentDate base for where only dates within 3 hours before will be fetched * @param availableCountries pair of dates per country code */ - private suspend fun asyncHandleLast3HoursFilesFetch( + private suspend fun fetchMissing3Hours( currentDate: LocalDate, availableCountries: List - ): List = withContext(Dispatchers.IO) { + ) = withContext(Dispatchers.IO) { Timber.tag(TAG).v( "asyncHandleLast3HoursFilesFetch(currentDate=%s, availableCountries=%s)", currentDate, availableCountries @@ -188,8 +205,9 @@ class KeyFileDownloader @Inject constructor( } val cachedHours = keyCache - .getEntriesForType(CachedKeyFile.Type.COUNTRY_HOUR) - .filter { it.isDownloadComplete } // We overwrite not completed ones + .getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) + .filter { it.first.isDownloadComplete && it.second.exists() } // We overwrite not completed ones + .map { it.first } // All cached files that are no longer on the server are considered stale val staleHours = cachedHours.filter { cachedHour -> @@ -227,7 +245,7 @@ class KeyFileDownloader @Inject constructor( location = country.country, dayIdentifier = day, hourIdentifier = missingHour, - type = CachedKeyFile.Type.COUNTRY_HOUR + type = CachedKeyInfo.Type.COUNTRY_HOUR ) downloadKeyFile(keyInfo, path) @@ -241,13 +259,15 @@ class KeyFileDownloader @Inject constructor( Timber.tag(TAG).d("Waiting for %d missing hour downloads.", hourDownloads.size) val downloadedHours = hourDownloads.awaitAll() - return@withContext downloadedHours.map { (keyInfo, path) -> + downloadedHours.map { (keyInfo, path) -> Timber.tag(TAG).d("Downloaded keyfile: %s to %s", keyInfo, path) path } + + Unit } - private suspend fun downloadKeyFile(keyInfo: CachedKeyFile, path: File) { + private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, path: File) { downloadServer.downloadKeyFile(keyInfo.location, keyInfo.day, keyInfo.hour, path) Timber.tag(TAG).v("Dowwnload finished: %s -> %s", keyInfo, path) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt index 7281c15455a..de3863d2521 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt @@ -16,7 +16,6 @@ interface DownloadApiV1 { @GET("/version/v1/diagnosis-keys/country/{country}/date") suspend fun getDayIndex(@Path("country") country: String): List // TODO Let retrofit format this to LocalDate - @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}/hour") suspend fun getHourIndex( @Path("country") country: String, diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt index 59602d4fdc2..4b49dc0022a 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt @@ -20,20 +20,19 @@ import javax.inject.Singleton @Singleton class DownloadServer @Inject constructor( - private val DownloadAPI: Lazy, + private val downloadAPI: Lazy, private val verificationKeys: VerificationKeys, @DownloadHomeCountry private val homeCountry: LocationCode ) { - private val apiDownload: DownloadApiV1 - get() = DownloadAPI.get() + private val api: DownloadApiV1 + get() = downloadAPI.get() suspend fun downloadAppConfig(): ApplicationConfigurationOuterClass.ApplicationConfiguration = withContext(Dispatchers.IO) { var exportBinary: ByteArray? = null var exportSignature: ByteArray? = null - - apiDownload.getApplicationConfiguration(homeCountry.identifier).byteStream() + api.getApplicationConfiguration(homeCountry.identifier).byteStream() .unzip { entry, entryContent -> if (entry.name == EXPORT_BINARY_FILE_NAME) exportBinary = entryContent.copyOf() @@ -65,7 +64,7 @@ class DownloadServer @Inject constructor( suspend fun getCountryIndex( wantedCountries: List ): List = withContext(Dispatchers.IO) { - apiDownload + api .getCountryIndex().filter { wantedCountries .map { c -> c.toUpperCase(Locale.ROOT) } @@ -75,7 +74,7 @@ class DownloadServer @Inject constructor( } suspend fun getDayIndex(location: LocationCode): List = withContext(Dispatchers.IO) { - apiDownload + api .getDayIndex(location.identifier) .map { dayString -> // 2020-08-19 LocalDate.parse(dayString, DAY_FORMATTER) @@ -84,7 +83,7 @@ class DownloadServer @Inject constructor( suspend fun getHourIndex(location: LocationCode, day: LocalDate): List = withContext(Dispatchers.IO) { - apiDownload + api .getHourIndex(location.identifier, day.toString(DAY_FORMATTER)) .map { hourString -> LocalTime.parse(hourString, HOUR_FORMATTER) } } @@ -112,13 +111,13 @@ class DownloadServer @Inject constructor( saveTo.outputStream().use { val streamingBody = if (hour != null) { - apiDownload.downloadKeyFileForHour( + api.downloadKeyFileForHour( locationCode.identifier, day.toString(DAY_FORMATTER), hour.toString(HOUR_FORMATTER) ) } else { - apiDownload.downloadKeyFileForDay( + api.downloadKeyFileForDay( locationCode.identifier, day.toString(DAY_FORMATTER) ) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFile.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt similarity index 94% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFile.kt rename to Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt index fbc40571bde..9c738bf566f 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFile.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt @@ -11,7 +11,7 @@ import org.joda.time.LocalDate import org.joda.time.LocalTime @Entity(tableName = "keyfiles") -data class CachedKeyFile( +data class CachedKeyInfo( @PrimaryKey @ColumnInfo(name = "id") val id: String, @ColumnInfo(name = "type") val type: Type, @ColumnInfo(name = "location") val location: LocationCode, // i.e. "DE" @@ -42,10 +42,10 @@ data class CachedKeyFile( @Transient val fileName: String = "$id.zip" - fun toDownloadCompleted(checksumMD5: String): DownloadUpdate = DownloadUpdate( + fun toDownloadUpdate(checksumMD5: String?): DownloadUpdate = DownloadUpdate( id = id, checksumMD5 = checksumMD5, - isDownloadComplete = true + isDownloadComplete = checksumMD5 != null ) companion object { diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt index f26fa21839a..77cdbc22265 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt @@ -15,11 +15,11 @@ import de.rki.coronawarnapp.util.database.CommonConverters import javax.inject.Inject @Database( - entities = [CachedKeyFile::class], + entities = [CachedKeyInfo::class], version = 1, exportSchema = true ) -@TypeConverters(CommonConverters::class, CachedKeyFile.Type.Converter::class) +@TypeConverters(CommonConverters::class, CachedKeyInfo.Type.Converter::class) abstract class KeyCacheDatabase : RoomDatabase() { abstract fun cachedKeyFiles(): CachedKeyFileDao @@ -27,22 +27,22 @@ abstract class KeyCacheDatabase : RoomDatabase() { @Dao interface CachedKeyFileDao { @Query("SELECT * FROM keyfiles") - suspend fun getAllEntries(): List + suspend fun getAllEntries(): List @Query("SELECT * FROM keyfiles WHERE type = :type") - suspend fun getEntriesForType(type: String): List + suspend fun getEntriesForType(type: String): List @Query("DELETE FROM keyfiles") suspend fun clear() @Insert(onConflict = OnConflictStrategy.ABORT) - suspend fun insertEntry(cachedKeyFile: CachedKeyFile) + suspend fun insertEntry(cachedKeyInfo: CachedKeyInfo) @Delete - suspend fun deleteEntry(cachedKeyFile: CachedKeyFile) + suspend fun deleteEntry(cachedKeyInfo: CachedKeyInfo) - @Update(entity = CachedKeyFile::class) - suspend fun updateDownloadState(update: CachedKeyFile.DownloadUpdate) + @Update(entity = CachedKeyInfo::class) + suspend fun updateDownloadState(update: CachedKeyInfo.DownloadUpdate) } class Factory @Inject constructor(private val context: Context) { diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt index bdda1b53c0c..8f2eb08f769 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt @@ -51,36 +51,54 @@ class KeyCacheRepository @Inject constructor( private val database by lazy { databaseFactory.create() } - private val cacheKeyFilesDao: KeyCacheDatabase.CachedKeyFileDao - get() = database.cachedKeyFiles() + private var isHouseKeepingDone = false + private var isMigrationDone = false - private fun tryMigration() { - // TODO() from key-export + @Synchronized + private suspend fun getDao(): KeyCacheDatabase.CachedKeyFileDao { + val dao = database.cachedKeyFiles() + + tryMigration() + tryHouseKeeping() + + return dao + } + + private suspend fun tryHouseKeeping() { + if (isHouseKeepingDone) return + isHouseKeepingDone = true + + val dirtyInfos = getDao().getAllEntries().filter { + it.isDownloadComplete && !getPathForKey(it).exists() + } + delete(dirtyInfos) } - private fun checkCacheHealth() { -// TODO() + private fun tryMigration() { + if (isMigrationDone) return + isMigrationDone = true + // TODO() from key-export } - fun getPathForKey(cachedKeyFile: CachedKeyFile): File { - return File(storageDir, cachedKeyFile.fileName) + fun getPathForKey(cachedKeyInfo: CachedKeyInfo): File { + return File(storageDir, cachedKeyInfo.fileName) } - suspend fun getAllCachedKeys(): List { - return cacheKeyFilesDao.getAllEntries() + suspend fun getAllCachedKeys(): List> { + return getDao().getAllEntries().map { it to getPathForKey(it) } } - suspend fun getEntriesForType(type: CachedKeyFile.Type): List { - return cacheKeyFilesDao.getEntriesForType(type.typeValue) + suspend fun getEntriesForType(type: CachedKeyInfo.Type): List> { + return getDao().getEntriesForType(type.typeValue).map { it to getPathForKey(it) } } suspend fun createCacheEntry( - type: CachedKeyFile.Type, + type: CachedKeyInfo.Type, location: LocationCode, dayIdentifier: LocalDate, hourIdentifier: LocalTime? - ): Pair { - val newKeyFile = CachedKeyFile( + ): Pair { + val newKeyFile = CachedKeyInfo( type = type, location = location, day = dayIdentifier, @@ -91,7 +109,7 @@ class KeyCacheRepository @Inject constructor( val targetFile = getPathForKey(newKeyFile) try { - cacheKeyFilesDao.insertEntry(newKeyFile) + getDao().insertEntry(newKeyFile) if (targetFile.exists()) { Timber.w("Target path despire no collision exists, deleting: %s", targetFile) } @@ -100,7 +118,7 @@ class KeyCacheRepository @Inject constructor( delete(listOf(newKeyFile)) Timber.d(e, "Retrying insertion for %s", newKeyFile) - cacheKeyFilesDao.insertEntry(newKeyFile) + getDao().insertEntry(newKeyFile) } // This can't be null unless our cache dir is root `/` @@ -113,15 +131,15 @@ class KeyCacheRepository @Inject constructor( return newKeyFile to targetFile } - suspend fun markKeyComplete(cachedKeyFile: CachedKeyFile, checksumMD5: String) { - val update = cachedKeyFile.toDownloadCompleted(checksumMD5) - cacheKeyFilesDao.updateDownloadState(update) + suspend fun markKeyComplete(cachedKeyInfo: CachedKeyInfo, checksumMD5: String) { + val update = cachedKeyInfo.toDownloadUpdate(checksumMD5) + getDao().updateDownloadState(update) } - suspend fun delete(keyFiles: Collection) { - Timber.d("delete(keyFiles=%s)", keyFiles) - keyFiles.forEach { key -> - cacheKeyFilesDao.deleteEntry(key) + suspend fun delete(keyInfos: Collection) { + Timber.d("delete(keyFiles=%s)", keyInfos) + keyInfos.forEach { key -> + getDao().deleteEntry(key) Timber.v("Deleted %s", key) val path = getPathForKey(key) if (path.delete()) Timber.v("Deleted cache key file at %s", path) @@ -130,6 +148,6 @@ class KeyCacheRepository @Inject constructor( suspend fun clear() { Timber.i("clear()") - delete(getAllCachedKeys()) + delete(getDao().getAllEntries()) } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt index 64e2b890633..3ddcdc5f90d 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt @@ -313,7 +313,7 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { ) = executeState(FILES_FROM_WEB_REQUESTS) { FileStorageHelper.initializeExportSubDirectory() val convertedDate = LocalDate.fromDateFields(currentDate) // TODO confirm - keyFileDownloader.asyncFetchFiles(convertedDate, countries) + keyFileDownloader.asyncFetchKeyFiles(convertedDate, countries) } /** diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModuleTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModuleTest.kt new file mode 100644 index 00000000000..35331290907 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModuleTest.kt @@ -0,0 +1,23 @@ +package de.rki.coronawarnapp.diagnosiskeys + +import de.rki.coronawarnapp.BuildConfig +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import io.kotest.matchers.shouldBe +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest + +class DiagnosisKeysModuleTest : BaseIOTest() { + + private val module = DiagnosisKeysModule() + + @Test + fun `home country should be DE`() { + module.provideDiagnosisHomeCountry() shouldBe LocationCode("DE") + } + + @Test + fun `download URL comes from BuildConfig`() { + module.provideDownloadServerUrl() shouldBe BuildConfig.DOWNLOAD_CDN_URL + } + +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt index bb10f86d968..199ddb20c55 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryDataTest.kt @@ -1,74 +1,241 @@ package de.rki.coronawarnapp.diagnosiskeys.download +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import org.joda.time.LocalDate +import org.joda.time.LocalTime import org.junit.jupiter.api.Test import testhelpers.BaseTest class CountryDataTest : BaseTest() { + private val locationCode = LocationCode("DE") + private fun createCachedKey(dayString: String, hourString: String? = null): CachedKeyInfo { + return mockk().apply { + every { location } returns locationCode + every { day } returns LocalDate.parse(dayString) + every { hour } returns hourString?.let { LocalTime.parse(it) } + } + } + + @Test + fun `missing days default`() { + val availableDates = listOf( + "2222-12-30", "2222-12-31" + ).map { LocalDate.parse(it) } + val cd = CountryDays(locationCode, availableDates) - // @Test -// fun testGetMissingDaysFromDiff() { -// val c1 = KeyCacheEntity() -// c1.id = "10008bf0-8890-356d-a4a4-dc375553160a" -// c1.path = -// "/data/user/0/de.rki.coronawarnapp.dev/cache/key-export/10008bf0-8890-356d-a4a4-dc375553160a.zip" -// c1.type = KeyCacheRepository.DateEntryType.DAY.ordinal -// -// val c2 = KeyCacheEntity() -// c2.id = "a8cc7b31-843e-3924-b918-023c386aec69" -// c2.path = -// "/data/user/0/de.rki.coronawarnapp.dev/cache/key-export/a8cc7b31-843e-3924-b918-023c386aec69.zip" -// c2.type = KeyCacheRepository.DateEntryType.DAY.ordinal -// -// val cacheEntries: Collection = listOf(c1, c2) -// -// val countryDataWrapper = -// CountryDataWrapper("DE", listOf("2020-08-29", "2020-08-26", "2020-08-28")) -// -// val result = countryDataWrapper.getMissingDates(cacheEntries) -// -// result.size shouldBe 1 -// result.elementAt(0) shouldBe "2020-08-28" -// } + cd.dayData shouldBe availableDates + + val cachedDays = listOf( + createCachedKey("2222-12-30") + ) + + cd.getMissingDays(cachedDays) shouldBe listOf(availableDates[1]) + cd.toMissingDays(cachedDays) shouldBe cd.copy( + dayData = listOf(availableDates[1]) + ) + } @Test - fun `missing hours default`() { - TODO() + fun `missing days empty day data`() { + val availableDates = emptyList() + val cd = CountryDays(locationCode, availableDates) + + cd.dayData shouldBe availableDates + + val cachedDays = listOf( + createCachedKey("2222-12-30"), + createCachedKey("2222-12-31") + ) + + cd.getMissingDays(cachedDays) shouldBe emptyList() + cd.toMissingDays(cachedDays) shouldBe null } @Test - fun `missing hours empty`() { - TODO() + fun `missing days empty cache`() { + val availableDates = listOf( + "2222-11-28", "2222-11-29" + ).map { LocalDate.parse(it) } + val cd = CountryDays(locationCode, availableDates) + + cd.dayData shouldBe availableDates + + val cachedDays = emptyList() + + cd.getMissingDays(cachedDays) shouldBe availableDates + cd.toMissingDays(cachedDays) shouldBe cd } @Test - fun `missing hours disjunct`() { - TODO() + fun `missing days disjunct`() { + val availableDates = listOf( + "2222-11-28", "2222-11-29" + ).map { LocalDate.parse(it) } + val cd = CountryDays(locationCode, availableDates) + + cd.dayData shouldBe availableDates + + val cachedDays = listOf( + createCachedKey("2222-12-28"), + createCachedKey("2222-12-29") + ) + + cd.getMissingDays(cachedDays) shouldBe availableDates + cd.toMissingDays(cachedDays) shouldBe cd } @Test - fun `missing hours none missing`() { - TODO() + fun `missing days none missing`() { + val availableDates = listOf( + "2222-12-30", "2222-12-31" + ).map { LocalDate.parse(it) } + val cd = CountryDays(locationCode, availableDates) + + cd.dayData shouldBe availableDates + + val cachedDays = listOf( + createCachedKey("2222-12-30"), + createCachedKey("2222-12-31") + ) + + cd.getMissingDays(cachedDays) shouldBe emptyList() + cd.toMissingDays(cachedDays) shouldBe null } @Test - fun `missing days default`() { - TODO() + fun `missing hours default`() { + val availableHours = mapOf( + LocalDate.parse("2222-12-30") to listOf( + LocalTime.parse("19:00"), LocalTime.parse("20:00") + ), + LocalDate.parse("2222-12-31") to listOf( + LocalTime.parse("22:00"), LocalTime.parse("23:00") + ) + ) + val cd = CountryHours(locationCode, availableHours) + + cd.hourData shouldBe availableHours + + val cachedHours = listOf( + createCachedKey("2222-12-30", "19:00"), + createCachedKey("2222-12-31", "23:00") + ) + + val missingHours = mapOf( + LocalDate.parse("2222-12-30") to listOf(LocalTime.parse("20:00")), + LocalDate.parse("2222-12-31") to listOf(LocalTime.parse("22:00")) + ) + + cd.getMissingHours(cachedHours) shouldBe missingHours + cd.toMissingHours(cachedHours) shouldBe cd.copy(hourData = missingHours) } @Test - fun `missing days empty`() { - TODO() + fun `missing hours empty available hour data`() { + val availableHours: Map> = emptyMap() + val cd = CountryHours(locationCode, availableHours) + + cd.hourData shouldBe availableHours + + val cachedHours = listOf( + createCachedKey("2222-12-30", "19:00"), + createCachedKey("2222-12-31", "23:00") + ) + + cd.getMissingHours(cachedHours) shouldBe emptyMap() + cd.toMissingHours(cachedHours) shouldBe null } @Test - fun `missing days disjunct`() { - TODO() + fun `missing hours faulty hour map`() { + val availableHours = mapOf( + LocalDate.parse("2222-12-30") to emptyList() + ) + val cd = CountryHours(locationCode, availableHours) + + cd.hourData shouldBe availableHours + + val cachedHours = listOf( + createCachedKey("2222-12-30", "19:00"), + createCachedKey("2222-12-31", "23:00") + ) + + cd.getMissingHours(cachedHours) shouldBe emptyMap() + cd.toMissingHours(cachedHours) shouldBe null } @Test - fun `missing days none missing`() { - TODO() + fun `missing hours empty cache`() { + val availableHours = mapOf( + LocalDate.parse("2222-12-30") to listOf( + LocalTime.parse("19:00"), LocalTime.parse("20:00") + ), + LocalDate.parse("2222-12-31") to listOf( + LocalTime.parse("22:00"), LocalTime.parse("23:00") + ) + ) + val cd = CountryHours(locationCode, availableHours) + + cd.hourData shouldBe availableHours + + val cachedHours = emptyList() + + cd.getMissingHours(cachedHours) shouldBe availableHours + cd.toMissingHours(cachedHours) shouldBe cd.copy(hourData = availableHours) + } + + @Test + fun `missing hours disjunct`() { + val availableHours = mapOf( + LocalDate.parse("2222-12-30") to listOf( + LocalTime.parse("19:00"), LocalTime.parse("20:00") + ), + LocalDate.parse("2222-12-31") to listOf( + LocalTime.parse("22:00"), LocalTime.parse("23:00") + ) + ) + val cd = CountryHours(locationCode, availableHours) + + cd.hourData shouldBe availableHours + + val cachedHours = listOf( + createCachedKey("2022-12-30", "19:00"), + createCachedKey("2022-12-31", "23:00") + ) + + cd.getMissingHours(cachedHours) shouldBe availableHours + cd.toMissingHours(cachedHours) shouldBe cd.copy(hourData = availableHours) + } + + @Test + fun `missing hours none missing`() { + val availableHours = mapOf( + LocalDate.parse("2222-12-30") to listOf( + LocalTime.parse("19:00"), LocalTime.parse("20:00") + ), + LocalDate.parse("2222-12-31") to listOf( + LocalTime.parse("22:00"), LocalTime.parse("23:00") + ) + ) + val cd = CountryHours(locationCode, availableHours) + + cd.hourData shouldBe availableHours + + val cachedHours = listOf( + createCachedKey("2222-12-30", "19:00"), + createCachedKey("2222-12-30", "20:00"), + createCachedKey("2222-12-31", "22:00"), + createCachedKey("2222-12-31", "23:00") + ) + + + cd.getMissingHours(cachedHours) shouldBe emptyMap() + cd.toMissingHours(cachedHours) shouldBe null } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt index 87d38e2878f..6580d1032fa 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -159,4 +159,9 @@ class KeyFileDownloaderTest : BaseTest() { fun `not completed cache entries are overwritten`() { TODO() } + + @Test + fun `fetch returns all currently available keyfiles`() { + TODO() + } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt deleted file mode 100644 index 428a1a7d0ac..00000000000 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNModuleTest.kt +++ /dev/null @@ -1,18 +0,0 @@ -package de.rki.coronawarnapp.diagnosiskeys.server - -import org.junit.jupiter.api.Test -import testhelpers.BaseIOTest - -class CDNModuleTest : BaseIOTest() { - - @Test - fun `home country should be DE`() { - TODO() - } - - @Test - fun `CDN URL comes from BuildConfig`() { - TODO() - } - -} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt deleted file mode 100644 index 49d46164239..00000000000 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/CDNServerTest.kt +++ /dev/null @@ -1,37 +0,0 @@ -package de.rki.coronawarnapp.diagnosiskeys.server - -import org.junit.jupiter.api.Test -import testhelpers.BaseIOTest - -class CDNServerTest : BaseIOTest() { - - @Test - fun `application config download`() { - TODO() - } - - @Test - fun `download key files for day`() { - TODO() - } - - @Test - fun `download key files for hour`() { - TODO() - } - - @Test - fun `download country index`() { - TODO() - } - - @Test - fun `download day index for country`() { - TODO() - } - - @Test - fun `download hour index for country and day`() { - TODO() - } -} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt new file mode 100644 index 00000000000..9fbf0e68b96 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt @@ -0,0 +1,142 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import de.rki.coronawarnapp.diagnosiskeys.DiagnosisKeysModule +import de.rki.coronawarnapp.http.HttpModule +import io.kotest.matchers.shouldBe +import kotlinx.coroutines.runBlocking +import okhttp3.ConnectionSpec +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest +import java.util.concurrent.TimeUnit + +class DownloadAPITest : BaseIOTest() { + + lateinit var webServer: MockWebServer + lateinit var serverAddress: String + + @BeforeEach + fun setup() { + webServer = MockWebServer() + webServer.start() + serverAddress = "http://${webServer.hostName}:${webServer.port}" + } + + @AfterEach + fun teardown() { + webServer.shutdown() + } + + private fun createAPI(): DownloadApiV1 { + val httpModule = HttpModule() + val defaultHttpClient = httpModule.defaultHttpClient() + val gsonConverterFactory = httpModule.provideGSONConverter() + + return DiagnosisKeysModule().let { + val downloadHttpClient = it.cdnHttpClient(defaultHttpClient) + .newBuilder() + .connectionSpecs(listOf(ConnectionSpec.CLEARTEXT, ConnectionSpec.MODERN_TLS)) + .build() + it.provideDownloadApi( + client = downloadHttpClient, + url = serverAddress, + gsonConverterFactory = gsonConverterFactory + + ) + } + } + + @Test + fun `application config download`() { + val api = createAPI() + + webServer.enqueue(MockResponse().setBody("~appconfig")) + + runBlocking { + api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" + } + + val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! + request.method shouldBe "GET" + request.path shouldBe "/version/v1/configuration/country/DE/app_config" + } + + @Test + fun `download country index`() { + val api = createAPI() + + webServer.enqueue(MockResponse().setBody("[\"DE\",\"NL\"]")) + + runBlocking { + api.getCountryIndex() shouldBe listOf("DE", "NL") + } + + val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! + request.method shouldBe "GET" + request.path shouldBe "/version/v1/diagnosis-keys/country" + } + + @Test + fun `download day index for country`() { + val api = createAPI() + + webServer.enqueue(MockResponse().setBody("[\"2020-08-19\",\"2020-08-20\"]")) + + runBlocking { + api.getDayIndex("DE") shouldBe listOf("2020-08-19", "2020-08-20") + } + + val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! + request.method shouldBe "GET" + request.path shouldBe "/version/v1/diagnosis-keys/country/DE/date" + } + + @Test + fun `download hour index for country and day`() { + val api = createAPI() + + webServer.enqueue(MockResponse().setBody("[22,23]")) + + runBlocking { + api.getHourIndex("DE", "2020-08-19") shouldBe listOf("22", "23") + } + + val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! + request.method shouldBe "GET" + request.path shouldBe "/version/v1/diagnosis-keys/country/DE/date/2020-08-19/hour" + } + + @Test + fun `download key files for day`() { + val api = createAPI() + + webServer.enqueue(MockResponse().setBody("~daykeyfile")) + + runBlocking { + api.downloadKeyFileForDay("DE", "2020-09-09").string() shouldBe "~daykeyfile" + } + + val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! + request.method shouldBe "GET" + request.path shouldBe "/version/v1/diagnosis-keys/country/DE/date/2020-09-09" + } + + @Test + fun `download key files for hour`() { + val api = createAPI() + + webServer.enqueue(MockResponse().setBody("~hourkeyfile")) + + runBlocking { + api.downloadKeyFileForHour("DE", "2020-09-09", "23").string() shouldBe "~hourkeyfile" + } + + val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! + request.method shouldBe "GET" + request.path shouldBe "/version/v1/diagnosis-keys/country/DE/date/2020-09-09/hour/23" + } + +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt new file mode 100644 index 00000000000..66bac00feb8 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt @@ -0,0 +1,235 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import dagger.Lazy +import de.rki.coronawarnapp.exception.ApplicationConfigurationCorruptException +import de.rki.coronawarnapp.exception.ApplicationConfigurationInvalidException +import de.rki.coronawarnapp.util.security.VerificationKeys +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import io.mockk.MockKAnnotations +import io.mockk.clearAllMocks +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.impl.annotations.MockK +import io.mockk.verify +import kotlinx.coroutines.runBlocking +import okhttp3.ResponseBody.Companion.toResponseBody +import okio.ByteString.Companion.decodeHex +import org.joda.time.LocalDate +import org.joda.time.LocalTime +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest +import java.io.File + +class DownloadServerTest : BaseIOTest() { + + @MockK + lateinit var api: DownloadApiV1 + + @MockK + lateinit var verificationKeys: VerificationKeys + + private val testDir = File(IO_TEST_BASEDIR, this::class.simpleName!!) + + private val defaultHomeCountry = LocationCode("DE") + + @BeforeEach + fun setup() { + MockKAnnotations.init(this) + testDir.mkdirs() + testDir.exists() shouldBe true + } + + @AfterEach + fun teardown() { + clearAllMocks() + testDir.deleteRecursively() + } + + private fun createDownloadServer( + homeCountry: LocationCode = defaultHomeCountry + ) = DownloadServer( + downloadAPI = Lazy { api }, + verificationKeys = verificationKeys, + homeCountry = homeCountry + ) + + @Test + fun `application config download`() { + coEvery { api.getApplicationConfiguration("DE") } returns APPCONFIG_HEX.decodeHex() + .toResponseBody() + + every { verificationKeys.hasInvalidSignature(any(), any()) } returns false + + val downloadServer = createDownloadServer() + + runBlocking { + val config = downloadServer.downloadAppConfig() + config.apply { + // We just care here that it's non default values, i.e. conversion worked + minRiskScore shouldBe 11 + appVersion.android.latest.major shouldBe 1 + appVersion.android.latest.minor shouldBe 0 + appVersion.android.latest.patch shouldBe 4 + + } + } + + verify(exactly = 1) { verificationKeys.hasInvalidSignature(any(), any()) } + } + + @Test + fun `application config data is faulty`() { + coEvery { api.getApplicationConfiguration("DE") } returns "123ABC".decodeHex() + .toResponseBody() + + every { verificationKeys.hasInvalidSignature(any(), any()) } returns false + + val downloadServer = createDownloadServer() + + runBlocking { + shouldThrow { + downloadServer.downloadAppConfig() + } + } + } + + @Test + fun `application config verification fails`() { + coEvery { api.getApplicationConfiguration("DE") } returns APPCONFIG_HEX.decodeHex() + .toResponseBody() + + every { verificationKeys.hasInvalidSignature(any(), any()) } returns true + + val downloadServer = createDownloadServer() + + runBlocking { + shouldThrow { + downloadServer.downloadAppConfig() + } + } + } + + @Test + fun `download country index`() { + val downloadServer = createDownloadServer() + coEvery { api.getCountryIndex() } returns listOf("DE", "NL", "FR") + + runBlocking { + downloadServer.getCountryIndex(listOf("DE", "NL")) shouldBe listOf( + LocationCode("DE"), LocationCode("NL") + ) + } + + coVerify { api.getCountryIndex() } + } + + @Test + fun `download day index for country`() { + val downloadServer = createDownloadServer() + coEvery { api.getDayIndex("DE") } returns listOf( + "2000-01-01", "2000-01-02" + ) + + runBlocking { + downloadServer.getDayIndex(LocationCode("DE")) shouldBe listOf( + "2000-01-01", "2000-01-02" + ).map { LocalDate.parse(it) } + } + + coVerify { api.getDayIndex("DE") } + } + + @Test + fun `download hour index for country and day`() { + val downloadServer = createDownloadServer() + coEvery { api.getHourIndex("DE", "2000-01-01") } returns listOf( + "20", "21" + ) + + runBlocking { + downloadServer.getHourIndex( + LocationCode("DE"), + LocalDate.parse("2000-01-01") + ) shouldBe listOf( + "20:00", "21:00" + ).map { LocalTime.parse(it) } + } + + coVerify { api.getHourIndex("DE", "2000-01-01") } + } + + + @Test + fun `download key files for day`() { + val downloadServer = createDownloadServer() + coEvery { + api.downloadKeyFileForDay( + "DE", + "2000-01-01" + ) + } returns "testdata-day".toResponseBody() + + val targetFile = File(testDir, "day-keys") + + runBlocking { + downloadServer.downloadKeyFile( + locationCode = LocationCode("DE"), + day = LocalDate.parse("2000-01-01"), + hour = null, + saveTo = targetFile + ) + } + + targetFile.exists() shouldBe true + targetFile.readText() shouldBe "testdata-day" + } + + @Test + fun `download key files for hour`() { + val downloadServer = createDownloadServer() + coEvery { + api.downloadKeyFileForHour( + "DE", + "2000-01-01", + "01" + ) + } returns "testdata-hour".toResponseBody() + + val targetFile = File(testDir, "hour-keys") + + runBlocking { + downloadServer.downloadKeyFile( + locationCode = LocationCode("DE"), + day = LocalDate.parse("2000-01-01"), + hour = LocalTime.parse("01:00"), + saveTo = targetFile + ) + } + + targetFile.exists() shouldBe true + targetFile.readText() shouldBe "testdata-hour" + } + + companion object { + private const val APPCONFIG_HEX = + "504b0304140008080800856b22510000000000000000000000000a0000006578706f72742e62696ee3e016" + + "f2e552e662f6f10f97e05792ca28292928b6d2d72f2f2fd74bce2fcacf4b2c4f2ccad34b2c28e0" + + "52e362f1f074f710e097f0c0a74e2a854b80835180498259814583d580cd82dd814390010c3c1d" + + "a4b8141835180d182d181d181561825a021cac02ac12ac0aac40f5ac16ac0eac86102913072b3e" + + "01460946841e47981e25192e160e73017b21214e88d0077ba8250fec1524b5a4b8b8b858043824" + + "98849804588578806a19255884c02400504b0708df2c788daf000000f1000000504b0304140008" + + "080800856b22510000000000000000000000000a0000006578706f72742e736967018a0075ff0a" + + "87010a380a1864652e726b692e636f726f6e617761726e6170702d6465761a0276312203323632" + + "2a13312e322e3834302e31303034352e342e332e321001180122473045022100cf32ff24ea18a1" + + "ffcc7ff4c9fe8d1808cecbc5a37e3e1d4c9ce682120450958c022064bf124b6973a9b510a43d47" + + "9ff93e0ef97a5b893c7af4abc4a8d399969cd8a0504b070813c517c68f0000008a000000504b01" + + "021400140008080800856b2251df2c788daf000000f10000000a00000000000000000000000000" + + "000000006578706f72742e62696e504b01021400140008080800856b225113c517c68f0000008a" + + "0000000a00000000000000000000000000e70000006578706f72742e736967504b050600000000" + + "0200020070000000ae0100000000" + } +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt index 4f8bd2a714c..cf1d1ebe08d 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyFileTest.kt @@ -1,27 +1,62 @@ package de.rki.coronawarnapp.diagnosiskeys.storage +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import io.kotest.matchers.shouldBe +import org.joda.time.Instant +import org.joda.time.LocalDate +import org.joda.time.LocalTime import org.junit.jupiter.api.Test import testhelpers.BaseTest class CachedKeyFileTest : BaseTest() { + private val type = CachedKeyInfo.Type.COUNTRY_DAY + private val location = LocationCode("DE") + private val day = LocalDate.parse("2222-12-31") + private val hour = LocalTime.parse("23:59") + private val now = Instant.EPOCH + @Test fun `secondary constructor`() { - TODO() + val key = CachedKeyInfo(type, location, day, hour, now) + + key.id shouldBe CachedKeyInfo.calcluateId(location, day, hour, type) + key.checksumMD5 shouldBe null + key.isDownloadComplete shouldBe false } @Test fun `keyfile id calculation`() { - TODO() + val calculatedId1 = CachedKeyInfo.calcluateId(location, day, hour, type) + val calculatedId2 = CachedKeyInfo.calcluateId(location, day, hour, type) + calculatedId1 shouldBe calculatedId2 + + calculatedId1 shouldBe "550b64773e052b9ddf232998a92846833ed3f907" } @Test fun `to completion`() { - TODO() + val key = CachedKeyInfo(type, location, day, hour, now) + val testChecksum = "testchecksum" + val downloadCompleteUpdate = key.toDownloadUpdate(testChecksum) + + downloadCompleteUpdate shouldBe CachedKeyInfo.DownloadUpdate( + id = downloadCompleteUpdate.id, + isDownloadComplete = true, + checksumMD5 = testChecksum + ) + + val resetDownloadUpdate = key.toDownloadUpdate(null) + + resetDownloadUpdate shouldBe CachedKeyInfo.DownloadUpdate( + id = downloadCompleteUpdate.id, + isDownloadComplete = false, + checksumMD5 = null + ) } @Test fun `trip changed typing`() { - // Let this test fail incase someone accidentally changes those values. - TODO() + CachedKeyInfo.Type.COUNTRY_DAY.typeValue shouldBe "country_day" + CachedKeyInfo.Type.COUNTRY_HOUR.typeValue shouldBe "country_hour" } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt index dde22a05855..f03f1058aa1 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt @@ -1,142 +1,212 @@ package de.rki.coronawarnapp.diagnosiskeys.storage -import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheDao +import android.content.Context +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.util.TimeStamper +import io.kotest.matchers.shouldBe +import io.mockk.MockKAnnotations +import io.mockk.clearAllMocks +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every import io.mockk.impl.annotations.MockK +import kotlinx.coroutines.runBlocking +import org.joda.time.Instant +import org.joda.time.LocalDate +import org.joda.time.LocalTime +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import testhelpers.BaseTest +import testhelpers.BaseIOTest +import java.io.File -class KeyCacheRepositoryTest : BaseTest() { +class KeyCacheRepositoryTest : BaseIOTest() { + @MockK + lateinit var context: Context @MockK - private lateinit var keyCacheDao: KeyCacheDao - - private lateinit var keyCacheRepository: KeyCacheRepository - -// @Before -// fun setUp() { -// MockKAnnotations.init(this) -// keyCacheRepository = KeyCacheRepository(keyCacheDao) -// -// // DAO tests in another test -// coEvery { keyCacheDao.getAllEntries() } returns listOf() -// coEvery { keyCacheDao.getHours() } returns listOf() -// coEvery { keyCacheDao.clear() } just Runs -// coEvery { keyCacheDao.clearHours() } just Runs -// coEvery { keyCacheDao.insertEntry(any()) } returns 0 -// } -// -// /** -// * Test clear order. -// */ -// @Test -// fun testClear() { -// runBlocking { -// keyCacheRepository.clear() -// -// coVerifyOrder { -// keyCacheDao.getAllEntries() -// -// keyCacheDao.clear() -// } -// } -// -// runBlocking { -// keyCacheRepository.clearHours() -// -// coVerifyOrder { -// keyCacheDao.getHours() -// -// keyCacheDao.clearHours() -// } -// } -// } -// -// /** -// * Test insert order. -// */ -// @Test -// fun testInsert() { -// runBlocking { -// keyCacheRepository.createCacheEntry( -// key = "1", -// type = KeyCacheRepository.DateEntryType.DAY, -// uri = URI("1") -// ) -// -// coVerify { -// keyCacheDao.insertEntry(any()) -// } -// } -// } -// -// @After -// fun cleanUp() { -// unmockkAll() -// } + lateinit var timeStamper: TimeStamper - @Test - fun `migration of old data`() { - TODO() - } + @MockK + lateinit var databaseFactory: KeyCacheDatabase.Factory - @Test - fun `migration does nothing when there is no old data`() { - TODO() - } + @MockK + lateinit var database: KeyCacheDatabase - @Test - fun `migration consumes old data and runs only once`() { - TODO() - } + @MockK + lateinit var keyfileDAO: KeyCacheDatabase.CachedKeyFileDao - @Test - fun `migration runs before creation`() { - TODO() + private val testDir = File(IO_TEST_BASEDIR, this::class.simpleName!!) + + @BeforeEach + fun setup() { + MockKAnnotations.init(this) + testDir.mkdirs() + testDir.exists() shouldBe true + + every { timeStamper.nowUTC } returns Instant.EPOCH + every { context.cacheDir } returns testDir + + every { databaseFactory.create() } returns database + every { database.cachedKeyFiles() } returns keyfileDAO + + coEvery { keyfileDAO.getAllEntries() } returns emptyList() } - @Test - fun `migration runs before download update`() { - TODO() + @AfterEach + fun teardown() { + clearAllMocks() + testDir.deleteRecursively() } + private fun createRepo(): KeyCacheRepository = KeyCacheRepository( + context = context, + databaseFactory = databaseFactory, + timeStamper = timeStamper + ) + @Test - fun `health check runs before data creation`() { + fun `migration runs before data access`() { TODO() } @Test - fun `health check runs before data update`() { - TODO() + fun `housekeeping runs before data access`() { + val lostKey = CachedKeyInfo( + location = LocationCode("DE"), + day = LocalDate.now(), + hour = LocalTime.now(), + type = CachedKeyInfo.Type.COUNTRY_HOUR, + createdAt = Instant.now() + ).copy( + isDownloadComplete = true, + checksumMD5 = "checksum" + ) + + val existingKey = CachedKeyInfo( + location = LocationCode("NL"), + day = LocalDate.now(), + hour = LocalTime.now(), + type = CachedKeyInfo.Type.COUNTRY_HOUR, + createdAt = Instant.now() + ) + + File(testDir, "diagnosis_keys/${existingKey.id}.zip").apply { + parentFile!!.mkdirs() + createNewFile() + } + + coEvery { keyfileDAO.getAllEntries() } returns listOf(lostKey, existingKey) + coEvery { keyfileDAO.updateDownloadState(any()) } returns Unit + coEvery { keyfileDAO.deleteEntry(lostKey) } returns Unit + + val repo = createRepo() + + coVerify(exactly = 0) { keyfileDAO.updateDownloadState(any()) } + + runBlocking { + repo.getAllCachedKeys() + coVerify(exactly = 2) { keyfileDAO.getAllEntries() } + coVerify { keyfileDAO.deleteEntry(lostKey) } + } } @Test fun `insert and retrieve`() { - TODO() + val repo = createRepo() + + coEvery { keyfileDAO.insertEntry(any()) } returns Unit + + runBlocking { + val (keyFile, path) = repo.createCacheEntry( + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-09"), + hourIdentifier = LocalTime.parse("23:00"), + type = CachedKeyInfo.Type.COUNTRY_HOUR + ) + + path shouldBe File(testDir, "diagnosis_keys/${keyFile.id}.zip") + + coVerify { keyfileDAO.insertEntry(keyFile) } + } } @Test fun `update download state`() { - TODO() + val repo = createRepo() + + coEvery { keyfileDAO.insertEntry(any()) } returns Unit + coEvery { keyfileDAO.updateDownloadState(any()) } returns Unit + + runBlocking { + val (keyFile, _) = repo.createCacheEntry( + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-09"), + hourIdentifier = LocalTime.parse("23:00"), + type = CachedKeyInfo.Type.COUNTRY_HOUR + ) + + repo.markKeyComplete(keyFile, "checksum") + + coVerify { + keyfileDAO.insertEntry(keyFile) + keyfileDAO.updateDownloadState(keyFile.toDownloadUpdate("checksum")) + } + } } @Test fun `delete only selected entries`() { - TODO() + val repo = createRepo() + + coEvery { keyfileDAO.insertEntry(any()) } returns Unit + coEvery { keyfileDAO.deleteEntry(any()) } returns Unit + + runBlocking { + val (keyFile, path) = repo.createCacheEntry( + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-09"), + hourIdentifier = LocalTime.parse("23:00"), + type = CachedKeyInfo.Type.COUNTRY_HOUR + ) + + path.createNewFile() shouldBe true + path.exists() shouldBe true + + repo.delete(listOf(keyFile)) + + coVerify { keyfileDAO.deleteEntry(keyFile) } + + path.exists() shouldBe false + } } @Test fun `clear all files`() { - TODO() - } + val repo = createRepo() - @Test - fun `path is based on private cache dir`() { - TODO() - } + val keyFileToClear = CachedKeyInfo( + location = LocationCode("DE"), + day = LocalDate.now(), + hour = LocalTime.now(), + type = CachedKeyInfo.Type.COUNTRY_HOUR, + createdAt = Instant.now() + ) - @Test - fun `check for missing key files and change download state`() { - TODO() + coEvery { keyfileDAO.getAllEntries() } returns listOf(keyFileToClear) + coEvery { keyfileDAO.deleteEntry(any()) } returns Unit + + val keyFilePath = repo.getPathForKey(keyFileToClear) + keyFilePath.createNewFile() shouldBe true + keyFilePath.exists() shouldBe true + + runBlocking { + repo.clear() + + coVerify { keyfileDAO.deleteEntry(keyFileToClear) } + + keyFilePath.exists() shouldBe false + } } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntityTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntityTest.kt new file mode 100644 index 00000000000..311923c1019 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntityTest.kt @@ -0,0 +1,5 @@ +package de.rki.coronawarnapp.diagnosiskeys.storage.legacy + +import testhelpers.BaseTest + +class KeyCacheEntityTest : BaseTest() diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt index 3bcbdf82600..cb1bb4a51e2 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/http/WebRequestBuilderTest.kt @@ -2,29 +2,17 @@ package de.rki.coronawarnapp.http import de.rki.coronawarnapp.http.service.SubmissionService import de.rki.coronawarnapp.http.service.VerificationService -import de.rki.coronawarnapp.service.diagnosiskey.DiagnosisKeyConstants -import de.rki.coronawarnapp.util.TimeAndDateExtensions.toServerFormat import de.rki.coronawarnapp.util.security.VerificationKeys import io.mockk.MockKAnnotations -import io.mockk.coEvery -import io.mockk.coVerify import io.mockk.impl.annotations.MockK import io.mockk.unmockkAll -import kotlinx.coroutines.runBlocking -import org.joda.time.LocalDate -import org.joda.time.LocalTime import org.junit.After import org.junit.Before -import org.junit.Test -import java.util.Date class WebRequestBuilderTest { @MockK private lateinit var verificationService: VerificationService - @MockK - private lateinit var distributionService: DistributionService - @MockK private lateinit var submissionService: SubmissionService @@ -37,49 +25,11 @@ class WebRequestBuilderTest { fun setUp() = run { MockKAnnotations.init(this) webRequestBuilder = WebRequestBuilder( - distributionService, verificationService, - submissionService, - verificationKeys + submissionService ) } @After fun tearDown() = unmockkAll() - - @Test - fun retrievingDateIndexIsSuccessful() { - val urlString = DiagnosisKeyConstants.AVAILABLE_DATES_URL - coEvery { distributionService.getDateIndex(urlString) } returns listOf( - "1900-01-01", - "2000-01-01" - ) - - runBlocking { - webRequestBuilder.asyncGetDateIndex("DE") shouldBe listOf( - LocalDate.parse("1900-01-01"), - LocalDate.parse("2000-01-01") - ) - - coVerify(exactly = 1) { distributionService.getDateIndex(urlString) } - } - } - - @Test - fun asyncGetHourIndex() { - val day = Date() - val urlString = DiagnosisKeyConstants.AVAILABLE_DATES_URL + - "/${day.toServerFormat()}/${DiagnosisKeyConstants.HOUR}" - - coEvery { distributionService.getHourIndex(urlString) } returns listOf("1", "2") - - runBlocking { - webRequestBuilder.asyncGetHourIndex(day) shouldBe listOf( - LocalTime.parse("01:00"), - LocalTime.parse("02:00") - ) - - coVerify(exactly = 1) { distributionService.getHourIndex(urlString) } - } - } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt index 771b9d242da..aef089e4371 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt @@ -1,8 +1,11 @@ package de.rki.coronawarnapp.service.applicationconfiguration +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer import de.rki.coronawarnapp.http.WebRequestBuilder import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass import de.rki.coronawarnapp.util.CWADebug +import de.rki.coronawarnapp.util.di.AppInjector +import de.rki.coronawarnapp.util.di.ApplicationComponent import io.kotest.matchers.shouldBe import io.mockk.coEvery import io.mockk.every @@ -23,7 +26,6 @@ class ApplicationConfigurationServiceTest : BaseTest() { CWADebug.isDebugBuildOrMode shouldBe true mockkObject(WebRequestBuilder) - val requestBuilder = mockk() val appConfig = mockk() val appConfigBuilder = mockk() @@ -36,9 +38,15 @@ class ApplicationConfigurationServiceTest : BaseTest() { every { appConfigBuilder.build() } returns appConfig - coEvery { requestBuilder.asyncGetApplicationConfigurationFromServer() } returns appConfig + val downloadServer = mockk() + coEvery { downloadServer.downloadAppConfig() } returns appConfig + + mockkObject(AppInjector) + mockk().apply { + every { this@apply.downloadServer } returns downloadServer + every { AppInjector.component } returns this@apply + } - every { WebRequestBuilder.getInstance() } returns requestBuilder runBlocking { ApplicationConfigurationService.asyncRetrieveApplicationConfiguration() diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt index 28d4e67da46..baddf45d799 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/util/database/CommonConvertersTest.kt @@ -19,43 +19,87 @@ package de.rki.coronawarnapp.util.database +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import io.kotest.matchers.shouldBe +import org.joda.time.Instant +import org.joda.time.LocalDate +import org.joda.time.LocalTime import org.junit.jupiter.api.Test +import testhelpers.BaseTest +import java.io.File +import java.util.UUID -class CommonConvertersTest { +class CommonConvertersTest : BaseTest() { + private val converters = CommonConverters() @Test fun `int list conversion`() { - TODO() + converters.apply { + val orig = listOf(1, 2, 3) + val raw = "[1,2,3]" + fromIntList(orig) shouldBe raw + toIntList(raw) shouldBe orig + } } @Test fun `UUID conversion`() { - TODO() + converters.apply { + val orig = UUID.fromString("123e4567-e89b-12d3-a456-426614174000") + val raw = "123e4567-e89b-12d3-a456-426614174000" + fromUUID(orig) shouldBe raw + toUUID(raw) shouldBe orig + } } @Test fun `path conversion`() { - TODO() + converters.apply { + val orig = File("/row/row/row/your/boat") + val raw = "/row/row/row/your/boat" + fromPath(orig) shouldBe raw + toPath(raw) shouldBe orig + } } @Test fun `local date conversion`() { - TODO() + converters.apply { + val orig = LocalDate.parse("2222-12-31") + val raw = "2222-12-31" + fromLocalDate(orig) shouldBe raw + toLocalDate(raw) shouldBe orig + } } @Test fun `local time conversion`() { - TODO() + converters.apply { + val orig = LocalTime.parse("23:59") + val raw = "23:59:00.000" + fromLocalTime(orig) shouldBe raw + toLocalTime(raw) shouldBe orig + } } @Test fun `instant conversion`() { - TODO() + converters.apply { + val orig = Instant.EPOCH + val raw = "1970-01-01T00:00:00.000Z" + fromInstant(orig) shouldBe raw + toInstant(raw) shouldBe orig + } } @Test fun `LocationCode conversion`() { - TODO() + converters.apply { + val orig = LocationCode("DE") + val raw = "DE" + fromLocationCode(orig) shouldBe raw + toLocationCode(raw) shouldBe orig + } } } From 3000bdcd7809343e87b3516bddb42aaedeb1192a Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 00:25:35 +0200 Subject: [PATCH 03/29] Added unit tests for the KeyCacheRepository TODO: Tests for downloader and migration. --- .../storage/keycache/KeyCacheDaoTest.kt | 90 ----------------- .../util/security/DBPasswordTest.kt | 53 +++++----- .../diagnosiskeys/DiagnosisKeysModule.kt | 9 ++ .../storage/KeyCacheRepository.kt | 28 +++--- .../{KeyCacheDao.kt => KeyCacheLegacyDao.kt} | 23 +---- ...CacheEntity.kt => KeyCacheLegacyEntity.kt} | 2 +- .../storage/legacy/LegacyKeyCacheMigration.kt | 98 +++++++++++++++++++ .../rki/coronawarnapp/storage/AppDatabase.kt | 8 +- .../storage/KeyCacheRepositoryTest.kt | 15 +-- .../legacy/LegacyKeyCacheMigrationTest.kt | 88 +++++++++++++++++ 10 files changed, 250 insertions(+), 164 deletions(-) delete mode 100644 Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt rename Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/{KeyCacheDao.kt => KeyCacheLegacyDao.kt} (77%) rename Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/{KeyCacheEntity.kt => KeyCacheLegacyEntity.kt} (98%) create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt diff --git a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt deleted file mode 100644 index daffbb8e9fe..00000000000 --- a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDaoTest.kt +++ /dev/null @@ -1,90 +0,0 @@ -package de.rki.coronawarnapp.storage.keycache - -import android.content.Context -import androidx.room.Room -import androidx.test.core.app.ApplicationProvider -import androidx.test.ext.junit.runners.AndroidJUnit4 -import com.google.common.truth.Truth.assertThat -import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheDao -import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheEntity -import de.rki.coronawarnapp.storage.AppDatabase -import kotlinx.coroutines.runBlocking -import org.junit.After -import org.junit.Before -import org.junit.Test -import org.junit.runner.RunWith - -/** - * KeyCacheDao test. - */ -@RunWith(AndroidJUnit4::class) -class KeyCacheDaoTest { - private lateinit var keyCacheDao: KeyCacheDao - private lateinit var db: AppDatabase - - @Before - fun setUp() { - val context = ApplicationProvider.getApplicationContext() - db = Room.inMemoryDatabaseBuilder( - context, AppDatabase::class.java - ).build() - keyCacheDao = db.dateDao() - } - - /** - * Test Create / Read / Delete DB operations. - */ - @Test - fun testCRDOperations() { - runBlocking { - val dates = KeyCacheEntity().apply { - this.id = "0" - this.path = "0" - this.type = 0 - } - val hours = KeyCacheEntity().apply { - this.id = "1" - this.path = "1" - this.type = 1 - } - - assertThat(keyCacheDao.getAllEntries().isEmpty()).isTrue() - - keyCacheDao.insertEntry(dates) - keyCacheDao.insertEntry(hours) - - var all = keyCacheDao.getAllEntries() - - assertThat(all.size).isEqualTo(2) - - val selectedDates = keyCacheDao.getDates() - assertThat(selectedDates.size).isEqualTo(1) - assertThat(selectedDates[0].type).isEqualTo(0) - assertThat(selectedDates[0].id).isEqualTo(dates.id) - - val selectedHours = keyCacheDao.getHours() - assertThat(selectedHours.size).isEqualTo(1) - assertThat(selectedHours[0].type).isEqualTo(1) - assertThat(selectedHours[0].id).isEqualTo(hours.id) - - keyCacheDao.clearHours() - - all = keyCacheDao.getAllEntries() - assertThat(all.size).isEqualTo(1) - assertThat(all[0].type).isEqualTo(0) - - keyCacheDao.insertEntry(hours) - - assertThat(keyCacheDao.getAllEntries().size).isEqualTo(2) - - keyCacheDao.clear() - - assertThat(keyCacheDao.getAllEntries().isEmpty()).isTrue() - } - } - - @After - fun closeDb() { - db.close() - } -} diff --git a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt index 3b06993c9b5..efbab656804 100644 --- a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt +++ b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/util/security/DBPasswordTest.kt @@ -21,8 +21,9 @@ package de.rki.coronawarnapp.util.security import android.content.Context import androidx.test.core.app.ApplicationProvider -import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheEntity import de.rki.coronawarnapp.storage.AppDatabase +import de.rki.coronawarnapp.storage.tracing.TracingIntervalEntity +import io.kotest.matchers.shouldBe import kotlinx.coroutines.runBlocking import net.sqlcipher.database.SQLiteException import org.hamcrest.Matchers.equalTo @@ -33,8 +34,6 @@ import org.junit.Before import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 -import java.util.UUID -import kotlin.random.Random @RunWith(JUnit4::class) class DBPasswordTest { @@ -70,16 +69,14 @@ class DBPasswordTest { @Test fun canLoadDataFromEncryptedDatabase() { runBlocking { - val id = UUID.randomUUID().toString() - val path = UUID.randomUUID().toString() - val type = Random.nextInt(1000) - - insertFakeEntity(id, path, type) - val keyCacheEntity = loadFakeEntity() - - assertThat(keyCacheEntity.id, equalTo(id)) - assertThat(keyCacheEntity.path, equalTo(path)) - assertThat(keyCacheEntity.type, equalTo(type)) + val from = 123L + val to = 456L + insertFakeEntity(from, to) + + loadFakeEntity().apply { + this.from shouldBe from + this.to shouldBe to + } } } @@ -95,34 +92,32 @@ class DBPasswordTest { @Test(expected = SQLiteException::class) fun loadingDataFromDatabaseWillFailWhenPassphraseIsIncorrect() { runBlocking { - val id = UUID.randomUUID().toString() - val path = UUID.randomUUID().toString() - val type = Random.nextInt(1000) - insertFakeEntity(id, path, type) + val from = 123L + val to = 456L + insertFakeEntity(from, to) clearSharedPreferences() AppDatabase.resetInstance() - val keyCacheEntity = loadFakeEntity() - assertThat(keyCacheEntity.id, equalTo(id)) - assertThat(keyCacheEntity.path, equalTo(path)) - assertThat(keyCacheEntity.type, equalTo(type)) + loadFakeEntity().apply { + this.from shouldBe from + this.to shouldBe to + } } } private suspend fun insertFakeEntity( - id: String, - path: String, - type: Int + from: Long, + to: Long ) { - db.dateDao().insertEntry(KeyCacheEntity().apply { - this.id = id - this.path = path - this.type = type + db.tracingIntervalDao().insertInterval(TracingIntervalEntity().apply { + this.from = from + this.to = to }) } - private suspend fun loadFakeEntity(): KeyCacheEntity = db.dateDao().getAllEntries().first() + private suspend fun loadFakeEntity(): TracingIntervalEntity = + db.tracingIntervalDao().getAllIntervals().first() private fun clearSharedPreferences() = SecurityHelper.globalEncryptedSharedPreferencesInstance.edit().clear().commit() diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt index 2f8c17f2daa..9e415bbc99a 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt @@ -1,5 +1,6 @@ package de.rki.coronawarnapp.diagnosiskeys +import android.content.Context import dagger.Module import dagger.Provides import dagger.Reusable @@ -9,7 +10,9 @@ import de.rki.coronawarnapp.diagnosiskeys.server.DownloadHomeCountry import de.rki.coronawarnapp.diagnosiskeys.server.DownloadHttpClient import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServerUrl import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheLegacyDao import de.rki.coronawarnapp.http.HttpClientDefault +import de.rki.coronawarnapp.storage.AppDatabase import okhttp3.ConnectionSpec import okhttp3.OkHttpClient import okhttp3.TlsVersion @@ -53,6 +56,12 @@ class DiagnosisKeysModule { return url } + @Singleton + @Provides + fun legacyKeyCacheDao(context: Context): KeyCacheLegacyDao { + return AppDatabase.getInstance(context).dateDao() + } + companion object { private val CDN_CONNECTION_SPECS = listOf( ConnectionSpec.Builder(ConnectionSpec.COMPATIBLE_TLS) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt index 8f2eb08f769..9365b140c34 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt @@ -23,6 +23,8 @@ import android.content.Context import android.database.sqlite.SQLiteConstraintException import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.util.TimeStamper +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import org.joda.time.LocalDate import org.joda.time.LocalTime import timber.log.Timber @@ -51,35 +53,31 @@ class KeyCacheRepository @Inject constructor( private val database by lazy { databaseFactory.create() } - private var isHouseKeepingDone = false - private var isMigrationDone = false + private var isInitDone = false + private val initMutex = Mutex() - @Synchronized private suspend fun getDao(): KeyCacheDatabase.CachedKeyFileDao { val dao = database.cachedKeyFiles() - tryMigration() - tryHouseKeeping() + if (!isInitDone) { + initMutex.withLock { + if (isInitDone) return@withLock + isInitDone = true + + doHouseKeeping() + } + } return dao } - private suspend fun tryHouseKeeping() { - if (isHouseKeepingDone) return - isHouseKeepingDone = true - + private suspend fun doHouseKeeping() { val dirtyInfos = getDao().getAllEntries().filter { it.isDownloadComplete && !getPathForKey(it).exists() } delete(dirtyInfos) } - private fun tryMigration() { - if (isMigrationDone) return - isMigrationDone = true - // TODO() from key-export - } - fun getPathForKey(cachedKeyInfo: CachedKeyInfo): File { return File(storageDir, cachedKeyInfo.fileName) } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheDao.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyDao.kt similarity index 77% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheDao.kt rename to Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyDao.kt index f67ae089d6c..4b28f4943f9 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheDao.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyDao.kt @@ -21,29 +21,16 @@ package de.rki.coronawarnapp.diagnosiskeys.storage.legacy import androidx.room.Dao import androidx.room.Delete -import androidx.room.Insert import androidx.room.Query @Dao -interface KeyCacheDao { - @Query("SELECT * FROM date WHERE type=0") - suspend fun getDates(): List - - @Query("SELECT * FROM date WHERE type=1") - suspend fun getHours(): List - +interface KeyCacheLegacyDao { @Query("SELECT * FROM date") - suspend fun getAllEntries(): List - - @Query("DELETE FROM date") - suspend fun clear() - - @Query("DELETE FROM date WHERE type=1") - suspend fun clearHours() + suspend fun getAllEntries(): List @Delete - suspend fun deleteEntry(entity: KeyCacheEntity) + suspend fun deleteEntry(entity: KeyCacheLegacyEntity) - @Insert - suspend fun insertEntry(keyCacheEntity: KeyCacheEntity): Long + @Query("DELETE FROM date") + suspend fun clear() } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntity.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyEntity.kt similarity index 98% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntity.kt rename to Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyEntity.kt index 449c423c991..0a8593c004e 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheEntity.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyEntity.kt @@ -27,7 +27,7 @@ import androidx.room.PrimaryKey tableName = "date", indices = [Index("id")] ) -class KeyCacheEntity { +class KeyCacheLegacyEntity { @PrimaryKey var id: String = "" diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt new file mode 100644 index 00000000000..2f02e34991a --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt @@ -0,0 +1,98 @@ +/****************************************************************************** + * Corona-Warn-App * + * * + * SAP SE and all other contributors / * + * copyright owners license this file to you under the Apache * + * License, Version 2.0 (the "License"); you may not use this * + * file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ******************************************************************************/ + +package de.rki.coronawarnapp.diagnosiskeys.storage.legacy + +import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import timber.log.Timber +import javax.inject.Inject + +class LegacyKeyCacheMigration @Inject constructor( + private val legacyDao: KeyCacheLegacyDao +) { + + suspend fun migrate(keyCacheRepository: KeyCacheRepository) { + val items = legacyDao.getAllEntries() + if (items.isEmpty()) { + Timber.tag(TAG).d("Nothing to migrate.") + return + } + +// val (keyInfo, path) = keyCacheRepository.createCacheEntry( +// +// ) + } + + companion object { + private val TAG = this::class.java.simpleName + } + +// +// enum class DateEntryType { +// DAY, +// HOUR +// } +// +// suspend fun createEntry(key: String, uri: URI, type: DateEntryType) = keyCacheDao.insertEntry( +// KeyCacheEntity().apply { +// this.id = key +// this.path = uri.rawPath +// this.type = type.ordinal +// } +// ) +// +// suspend fun deleteOutdatedEntries(validEntries: List) = +// keyCacheDao.getAllEntries().forEach { +// Timber.v("valid entries for cache from server: $validEntries") +// val file = File(it.path) +// if (!validEntries.contains(it.id) || !file.exists()) { +// Timber.w("${it.id} will be deleted from the cache") +// deleteFileForEntry(it) +// keyCacheDao.deleteEntry(it) +// } +// } +// +// private fun deleteFileForEntry(entry: KeyCacheEntity) = +// +// suspend fun getDates() = keyCacheDao.getDates() +// suspend fun getHours() = keyCacheDao.getHours() +// +// suspend fun clearHours() { +// getHours().forEach { deleteFileForEntry(it) } +// keyCacheDao.clearHours() +// } +// +// suspend fun clear() { +// keyCacheDao.getAllEntries().forEach { deleteFileForEntry(it) } +// keyCacheDao.clear() +// } +// +// suspend fun clear(idList: List) { +// if (idList.isNotEmpty()) { +// val entries = keyCacheDao.getAllEntries(idList) +// entries.forEach { deleteFileForEntry(it) } +// keyCacheDao.deleteEntries(entries) +// } +// } +// +// suspend fun getFilesFromEntries() = keyCacheDao +// .getAllEntries() +// .map { File(it.path) } +// .filter { it.exists() } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt index c2e7735be24..454055162dc 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppDatabase.kt @@ -7,8 +7,8 @@ import androidx.room.Database import androidx.room.Room import androidx.room.RoomDatabase import androidx.room.TypeConverters -import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheDao -import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheEntity +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheLegacyDao +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheLegacyEntity import de.rki.coronawarnapp.storage.tracing.TracingIntervalDao import de.rki.coronawarnapp.storage.tracing.TracingIntervalEntity import de.rki.coronawarnapp.storage.tracing.TracingIntervalRepository @@ -20,7 +20,7 @@ import net.sqlcipher.database.SupportFactory import java.io.File @Database( - entities = [ExposureSummaryEntity::class, KeyCacheEntity::class, TracingIntervalEntity::class], + entities = [ExposureSummaryEntity::class, KeyCacheLegacyEntity::class, TracingIntervalEntity::class], version = 1, exportSchema = true ) @@ -28,7 +28,7 @@ import java.io.File abstract class AppDatabase : RoomDatabase() { abstract fun exposureSummaryDao(): ExposureSummaryDao - abstract fun dateDao(): KeyCacheDao + abstract fun dateDao(): KeyCacheLegacyDao abstract fun tracingIntervalDao(): TracingIntervalDao companion object { diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt index f03f1058aa1..68bdc4d20a8 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt @@ -2,6 +2,7 @@ package de.rki.coronawarnapp.diagnosiskeys.storage import android.content.Context import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration import de.rki.coronawarnapp.util.TimeStamper import io.kotest.matchers.shouldBe import io.mockk.MockKAnnotations @@ -27,6 +28,9 @@ class KeyCacheRepositoryTest : BaseIOTest() { @MockK lateinit var timeStamper: TimeStamper + @MockK + lateinit var migrator: LegacyKeyCacheMigration + @MockK lateinit var databaseFactory: KeyCacheDatabase.Factory @@ -45,11 +49,13 @@ class KeyCacheRepositoryTest : BaseIOTest() { testDir.exists() shouldBe true every { timeStamper.nowUTC } returns Instant.EPOCH - every { context.cacheDir } returns testDir + every { context.cacheDir } returns File(testDir, "cache") every { databaseFactory.create() } returns database every { database.cachedKeyFiles() } returns keyfileDAO + coEvery { migrator.migrate(any()) } returns Unit + coEvery { keyfileDAO.getAllEntries() } returns emptyList() } @@ -65,11 +71,6 @@ class KeyCacheRepositoryTest : BaseIOTest() { timeStamper = timeStamper ) - @Test - fun `migration runs before data access`() { - TODO() - } - @Test fun `housekeeping runs before data access`() { val lostKey = CachedKeyInfo( @@ -125,7 +126,7 @@ class KeyCacheRepositoryTest : BaseIOTest() { type = CachedKeyInfo.Type.COUNTRY_HOUR ) - path shouldBe File(testDir, "diagnosis_keys/${keyFile.id}.zip") + path shouldBe File(context.cacheDir, "diagnosis_keys/${keyFile.id}.zip") coVerify { keyfileDAO.insertEntry(keyFile) } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt new file mode 100644 index 00000000000..4c0bd9d0360 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt @@ -0,0 +1,88 @@ +package de.rki.coronawarnapp.diagnosiskeys.storage.legacy + +import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import io.kotest.matchers.shouldBe +import io.mockk.MockKAnnotations +import io.mockk.clearAllMocks +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.impl.annotations.MockK +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest +import java.io.File + +class LegacyKeyCacheMigrationTest : BaseIOTest() { + + @MockK + lateinit var keyCacheRepository: KeyCacheRepository + + @MockK + lateinit var legacyDao: KeyCacheLegacyDao + + private val testDir = File(IO_TEST_BASEDIR, this::class.simpleName!!) + + @BeforeEach + fun setup() { + MockKAnnotations.init(this) + testDir.mkdirs() + testDir.exists() shouldBe true + + coEvery { legacyDao.getAllEntries() } returns emptyList() + } + + @AfterEach + fun teardown() { + clearAllMocks() + testDir.deleteRecursively() + } + + private fun createTool() = LegacyKeyCacheMigration( + legacyDao = legacyDao + ) + + @Test + fun `no legacy files to migrate`() { + val tool = createTool() + runBlocking { + tool.migrate(keyCacheRepository) + } + + coVerify(exactly = 0) { + keyCacheRepository.createCacheEntry( + type = any(), + location = any(), + dayIdentifier = any(), + hourIdentifier = any() + ) + } + } + + @Test + fun `migrate two legacy files`() { + val tool = createTool() + val legacyItem1 = KeyCacheLegacyEntity( + + ) + val legacyItem2 = KeyCacheLegacyEntity( + + ) + + coEvery { legacyDao.getAllEntries() } returns listOf(legacyItem1, legacyItem2) + + runBlocking { + tool.migrate(keyCacheRepository) + } + + coVerify(exactly = 0) { + keyCacheRepository.createCacheEntry( + type = any(), + location = any(), + dayIdentifier = any(), + hourIdentifier = any() + ) + } + } +} From f256fef378d09e824aac815045862bb06fb82c4a Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 01:25:27 +0200 Subject: [PATCH 04/29] Implemented POC for migration old key files. --- .../download/KeyFileDownloader.kt | 40 +++++++++++- .../diagnosiskeys/server/DownloadApiV1.kt | 5 +- .../diagnosiskeys/server/DownloadServer.kt | 41 +++++++----- .../storage/legacy/LegacyKeyCacheMigration.kt | 64 ++++++++++++++++--- .../diagnosiskeys/server/DownloadAPITest.kt | 5 +- .../server/DownloadServerTest.kt | 5 +- 6 files changed, 127 insertions(+), 33 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 4d7d02456e1..563f24468ae 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -24,6 +24,7 @@ import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration import de.rki.coronawarnapp.storage.FileStorageHelper import de.rki.coronawarnapp.storage.LocalData import de.rki.coronawarnapp.util.CWADebug @@ -34,6 +35,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.withContext +import okhttp3.Headers import org.joda.time.LocalDate import timber.log.Timber import java.io.File @@ -45,7 +47,8 @@ import javax.inject.Inject @Reusable class KeyFileDownloader @Inject constructor( private val downloadServer: DownloadServer, - private val keyCache: KeyCacheRepository + private val keyCache: KeyCacheRepository, + private val legacyKeyCache: LegacyKeyCacheMigration ) { /** @@ -268,7 +271,40 @@ class KeyFileDownloader @Inject constructor( } private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, path: File) { - downloadServer.downloadKeyFile(keyInfo.location, keyInfo.day, keyInfo.hour, path) + val headerValidation = object : DownloadServer.HeaderValidation { + override suspend fun validate(headers: Headers): Boolean { + // TODO get MD5 from better header, ETag isn't guaranteed to be the files MD5 + val fileMD5 = headers.values("ETag") + .singleOrNull() + ?.removePrefix("\"") + ?.removeSuffix("\"") + + var startDownload = true + + if (fileMD5 != null) { + legacyKeyCache.getLegacyFile(fileMD5)?.let { legacyFile -> + legacyFile.inputStream().use { from -> + path.outputStream().use { to -> + from.copyTo(to, DEFAULT_BUFFER_SIZE) + } + } + legacyKeyCache.delete(fileMD5) + startDownload = false + } + } + + return startDownload + } + } + + downloadServer.downloadKeyFile( + keyInfo.location, + keyInfo.day, + keyInfo.hour, + path, + headerValidation + ) + Timber.tag(TAG).v("Dowwnload finished: %s -> %s", keyInfo, path) val (downloadedMD5, duration) = measureTimeMillisWithResult { path.hashToMD5() } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt index de3863d2521..cdb840d860f 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt @@ -1,6 +1,7 @@ package de.rki.coronawarnapp.diagnosiskeys.server import okhttp3.ResponseBody +import retrofit2.Response import retrofit2.http.GET import retrofit2.http.Path import retrofit2.http.Streaming @@ -27,7 +28,7 @@ interface DownloadApiV1 { suspend fun downloadKeyFileForDay( @Path("country") country: String, @Path("day") day: String - ): ResponseBody + ): Response @Streaming @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}/hour/{hour}") @@ -35,6 +36,6 @@ interface DownloadApiV1 { @Path("country") country: String, @Path("day") day: String, @Path("hour") hour: String - ): ResponseBody + ): Response } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt index 4b49dc0022a..96ed0d9c453 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt @@ -9,6 +9,7 @@ import de.rki.coronawarnapp.util.ZipHelper.unzip import de.rki.coronawarnapp.util.security.VerificationKeys import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import okhttp3.Headers import org.joda.time.LocalDate import org.joda.time.LocalTime import org.joda.time.format.DateTimeFormat @@ -88,6 +89,10 @@ class DownloadServer @Inject constructor( .map { hourString -> LocalTime.parse(hourString, HOUR_FORMATTER) } } + interface HeaderValidation { + suspend fun validate(headers: Headers): Boolean = true + } + /** * Retrieves Key Files from the Server * Leave **[hour]** null to download a day package @@ -96,7 +101,8 @@ class DownloadServer @Inject constructor( locationCode: LocationCode, day: LocalDate, hour: LocalTime? = null, - saveTo: File + saveTo: File, + validator: HeaderValidation = object : HeaderValidation {} ) = withContext(Dispatchers.IO) { Timber.tag(TAG).v( "Starting download: country=%s, day=%s, hour=%s -> %s.", @@ -108,21 +114,26 @@ class DownloadServer @Inject constructor( saveTo.delete() } - saveTo.outputStream().use { + val response = if (hour != null) { + api.downloadKeyFileForHour( + locationCode.identifier, + day.toString(DAY_FORMATTER), + hour.toString(HOUR_FORMATTER) + ) + } else { + api.downloadKeyFileForDay( + locationCode.identifier, + day.toString(DAY_FORMATTER) + ) + } - val streamingBody = if (hour != null) { - api.downloadKeyFileForHour( - locationCode.identifier, - day.toString(DAY_FORMATTER), - hour.toString(HOUR_FORMATTER) - ) - } else { - api.downloadKeyFileForDay( - locationCode.identifier, - day.toString(DAY_FORMATTER) - ) - } - streamingBody.byteStream().copyTo(it, DEFAULT_BUFFER_SIZE) + if (!validator.validate(response.headers())) { + Timber.tag(TAG).d("validateHeaders() told us to abort.") + return@withContext + } + + saveTo.outputStream().use { + response.body()!!.byteStream().copyTo(it, DEFAULT_BUFFER_SIZE) } Timber.tag(TAG).v("Key file download successful: %s", saveTo) } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt index 2f02e34991a..ba753b132c1 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt @@ -19,24 +19,68 @@ package de.rki.coronawarnapp.diagnosiskeys.storage.legacy -import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import android.content.Context +import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 +import de.rki.coronawarnapp.util.TimeStamper +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.joda.time.Duration +import org.joda.time.Instant import timber.log.Timber +import java.io.File import javax.inject.Inject class LegacyKeyCacheMigration @Inject constructor( - private val legacyDao: KeyCacheLegacyDao + private val context: Context, + private val legacyDao: KeyCacheLegacyDao, + private val timeStamper: TimeStamper ) { - suspend fun migrate(keyCacheRepository: KeyCacheRepository) { - val items = legacyDao.getAllEntries() - if (items.isEmpty()) { - Timber.tag(TAG).d("Nothing to migrate.") - return + private val cacheDir by lazy { + File(context.cacheDir, "key-export") + } + + private val workMutex = Mutex() + private var isInit = false + private val legacyCacheMap = mutableMapOf() + + private suspend fun tryInit() { + if (isInit) return + isInit = true + + legacyDao.clear() + + try { + cacheDir.listFiles()?.forEach { file -> + val isExpired = Duration( + Instant.ofEpochMilli(file.lastModified()), + timeStamper.nowUTC + ).standardDays > 15 + + if (isExpired) { + Timber.tag(TAG).d("Deleting expired file: %s", file) + file.delete() + } else { + val md5 = file.hashToMD5() + Timber.tag(TAG).v("MD5 %s for %s", md5, file) + legacyCacheMap["md5"] = file + } + } + } catch (e: Exception) { + Timber.tag(TAG).e(e, "Reading legacy cached failed. Clearing.") + cacheDir.deleteRecursively() } + } -// val (keyInfo, path) = keyCacheRepository.createCacheEntry( -// -// ) + suspend fun getLegacyFile(fileMD5: String): File? = workMutex.withLock { + tryInit() + legacyCacheMap[fileMD5] + } + + suspend fun delete(fileMD5: String) = workMutex.withLock { + tryInit() + Timber.tag(TAG).v("delete(md5=%s)", fileMD5) + legacyCacheMap.remove(fileMD5) } companion object { diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt index 9fbf0e68b96..16f57bdf40e 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt @@ -116,7 +116,7 @@ class DownloadAPITest : BaseIOTest() { webServer.enqueue(MockResponse().setBody("~daykeyfile")) runBlocking { - api.downloadKeyFileForDay("DE", "2020-09-09").string() shouldBe "~daykeyfile" + api.downloadKeyFileForDay("DE", "2020-09-09").body()!!.string() shouldBe "~daykeyfile" } val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! @@ -131,7 +131,8 @@ class DownloadAPITest : BaseIOTest() { webServer.enqueue(MockResponse().setBody("~hourkeyfile")) runBlocking { - api.downloadKeyFileForHour("DE", "2020-09-09", "23").string() shouldBe "~hourkeyfile" + api.downloadKeyFileForHour("DE", "2020-09-09", "23").body()!! + .string() shouldBe "~hourkeyfile" } val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt index 66bac00feb8..12f1a52f543 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt @@ -21,6 +21,7 @@ import org.joda.time.LocalTime import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import retrofit2.Response import testhelpers.BaseIOTest import java.io.File @@ -171,7 +172,7 @@ class DownloadServerTest : BaseIOTest() { "DE", "2000-01-01" ) - } returns "testdata-day".toResponseBody() + } returns Response.success("testdata-day".toResponseBody()) val targetFile = File(testDir, "day-keys") @@ -197,7 +198,7 @@ class DownloadServerTest : BaseIOTest() { "2000-01-01", "01" ) - } returns "testdata-hour".toResponseBody() + } returns Response.success("testdata-hour".toResponseBody()) val targetFile = File(testDir, "hour-keys") From 5671e66edf396261eacec6e7af3035fed395ad4c Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 01:38:55 +0200 Subject: [PATCH 05/29] Fixed legacy file migration and cleanup, improved logging. --- .../download/KeyFileDownloader.kt | 27 +------ .../storage/legacy/KeyCacheLegacyDao.kt | 7 -- .../storage/legacy/LegacyKeyCacheMigration.kt | 77 +------------------ 3 files changed, 6 insertions(+), 105 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 563f24468ae..e1c76655896 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -1,22 +1,3 @@ -/****************************************************************************** - * Corona-Warn-App * - * * - * SAP SE and all other contributors / * - * copyright owners license this file to you under the Apache * - * License, Version 2.0 (the "License"); you may not use this * - * file except in compliance with the License. * - * You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ******************************************************************************/ - package de.rki.coronawarnapp.diagnosiskeys.download import dagger.Reusable @@ -59,8 +40,6 @@ class KeyFileDownloader @Inject constructor( * - the app initializes with an empty cache and draws in every available data set in the beginning * - the difference can only work properly if the date from the device is synchronized through the net * - the difference in timezone is taken into account by using UTC in the Conversion from the Date to Server format - * - the missing days and hours are stored in one table as the actual stored data amount is low - * - the underlying repository from the database has no error and is reliable as source of truth * * @param currentDate the current date - if this is adjusted by the calendar, the cache is affected. * @return list of all files from both the cache and the diff query @@ -271,7 +250,7 @@ class KeyFileDownloader @Inject constructor( } private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, path: File) { - val headerValidation = object : DownloadServer.HeaderValidation { + val validation = object : DownloadServer.HeaderValidation { override suspend fun validate(headers: Headers): Boolean { // TODO get MD5 from better header, ETag isn't guaranteed to be the files MD5 val fileMD5 = headers.values("ETag") @@ -283,6 +262,7 @@ class KeyFileDownloader @Inject constructor( if (fileMD5 != null) { legacyKeyCache.getLegacyFile(fileMD5)?.let { legacyFile -> + Timber.tag(TAG).i("Migrating legacy file for : %s", keyInfo) legacyFile.inputStream().use { from -> path.outputStream().use { to -> from.copyTo(to, DEFAULT_BUFFER_SIZE) @@ -302,7 +282,7 @@ class KeyFileDownloader @Inject constructor( keyInfo.day, keyInfo.hour, path, - headerValidation + validation ) Timber.tag(TAG).v("Dowwnload finished: %s -> %s", keyInfo, path) @@ -314,7 +294,6 @@ class KeyFileDownloader @Inject constructor( } companion object { - private val TAG: String? = KeyFileDownloader::class.simpleName } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyDao.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyDao.kt index 4b28f4943f9..3418ebd0cbe 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyDao.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/KeyCacheLegacyDao.kt @@ -20,17 +20,10 @@ package de.rki.coronawarnapp.diagnosiskeys.storage.legacy import androidx.room.Dao -import androidx.room.Delete import androidx.room.Query @Dao interface KeyCacheLegacyDao { - @Query("SELECT * FROM date") - suspend fun getAllEntries(): List - - @Delete - suspend fun deleteEntry(entity: KeyCacheLegacyEntity) - @Query("DELETE FROM date") suspend fun clear() } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt index ba753b132c1..1984b8c12b3 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt @@ -1,22 +1,3 @@ -/****************************************************************************** - * Corona-Warn-App * - * * - * SAP SE and all other contributors / * - * copyright owners license this file to you under the Apache * - * License, Version 2.0 (the "License"); you may not use this * - * file except in compliance with the License. * - * You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ******************************************************************************/ - package de.rki.coronawarnapp.diagnosiskeys.storage.legacy import android.content.Context @@ -63,7 +44,7 @@ class LegacyKeyCacheMigration @Inject constructor( } else { val md5 = file.hashToMD5() Timber.tag(TAG).v("MD5 %s for %s", md5, file) - legacyCacheMap["md5"] = file + legacyCacheMap[md5] = file } } } catch (e: Exception) { @@ -80,63 +61,11 @@ class LegacyKeyCacheMigration @Inject constructor( suspend fun delete(fileMD5: String) = workMutex.withLock { tryInit() Timber.tag(TAG).v("delete(md5=%s)", fileMD5) - legacyCacheMap.remove(fileMD5) + val removedFile = legacyCacheMap.remove(fileMD5) + if (removedFile?.delete() == true) Timber.tag(TAG).d("Deleted %s", removedFile) } companion object { private val TAG = this::class.java.simpleName } - -// -// enum class DateEntryType { -// DAY, -// HOUR -// } -// -// suspend fun createEntry(key: String, uri: URI, type: DateEntryType) = keyCacheDao.insertEntry( -// KeyCacheEntity().apply { -// this.id = key -// this.path = uri.rawPath -// this.type = type.ordinal -// } -// ) -// -// suspend fun deleteOutdatedEntries(validEntries: List) = -// keyCacheDao.getAllEntries().forEach { -// Timber.v("valid entries for cache from server: $validEntries") -// val file = File(it.path) -// if (!validEntries.contains(it.id) || !file.exists()) { -// Timber.w("${it.id} will be deleted from the cache") -// deleteFileForEntry(it) -// keyCacheDao.deleteEntry(it) -// } -// } -// -// private fun deleteFileForEntry(entry: KeyCacheEntity) = -// -// suspend fun getDates() = keyCacheDao.getDates() -// suspend fun getHours() = keyCacheDao.getHours() -// -// suspend fun clearHours() { -// getHours().forEach { deleteFileForEntry(it) } -// keyCacheDao.clearHours() -// } -// -// suspend fun clear() { -// keyCacheDao.getAllEntries().forEach { deleteFileForEntry(it) } -// keyCacheDao.clear() -// } -// -// suspend fun clear(idList: List) { -// if (idList.isNotEmpty()) { -// val entries = keyCacheDao.getAllEntries(idList) -// entries.forEach { deleteFileForEntry(it) } -// keyCacheDao.deleteEntries(entries) -// } -// } -// -// suspend fun getFilesFromEntries() = keyCacheDao -// .getAllEntries() -// .map { File(it.path) } -// .filter { it.exists() } } From 0a1ffb3b5eca5fc9c9679164479f68c692f19ff2 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 11:38:58 +0200 Subject: [PATCH 06/29] Added unit tests for legacy key file migration. --- .../download/KeyFileDownloader.kt | 27 +-- .../storage/legacy/LegacyKeyCacheMigration.kt | 41 +++-- .../storage/KeyCacheRepositoryTest.kt | 6 - .../legacy/LegacyKeyCacheMigrationTest.kt | 159 +++++++++++++++--- 4 files changed, 169 insertions(+), 64 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index e1c76655896..93ad62e717c 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -249,7 +249,7 @@ class KeyFileDownloader @Inject constructor( Unit } - private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, path: File) { + private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, saveTo: File) { val validation = object : DownloadServer.HeaderValidation { override suspend fun validate(headers: Headers): Boolean { // TODO get MD5 from better header, ETag isn't guaranteed to be the files MD5 @@ -258,22 +258,7 @@ class KeyFileDownloader @Inject constructor( ?.removePrefix("\"") ?.removeSuffix("\"") - var startDownload = true - - if (fileMD5 != null) { - legacyKeyCache.getLegacyFile(fileMD5)?.let { legacyFile -> - Timber.tag(TAG).i("Migrating legacy file for : %s", keyInfo) - legacyFile.inputStream().use { from -> - path.outputStream().use { to -> - from.copyTo(to, DEFAULT_BUFFER_SIZE) - } - } - legacyKeyCache.delete(fileMD5) - startDownload = false - } - } - - return startDownload + return !legacyKeyCache.tryMigration(fileMD5, saveTo) } } @@ -281,14 +266,14 @@ class KeyFileDownloader @Inject constructor( keyInfo.location, keyInfo.day, keyInfo.hour, - path, + saveTo, validation ) - Timber.tag(TAG).v("Dowwnload finished: %s -> %s", keyInfo, path) + Timber.tag(TAG).v("Dowwnload finished: %s -> %s", keyInfo, saveTo) - val (downloadedMD5, duration) = measureTimeMillisWithResult { path.hashToMD5() } - Timber.tag(TAG).v("Hashed to MD5 in %dms: %s", duration, path) + val (downloadedMD5, duration) = measureTimeMillisWithResult { saveTo.hashToMD5() } + Timber.tag(TAG).v("Hashed to MD5 in %dms: %s", duration, saveTo) keyCache.markKeyComplete(keyInfo, downloadedMD5) } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt index 1984b8c12b3..58a4e2ce37f 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt @@ -1,6 +1,7 @@ package de.rki.coronawarnapp.diagnosiskeys.storage.legacy import android.content.Context +import dagger.Lazy import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 import de.rki.coronawarnapp.util.TimeStamper import kotlinx.coroutines.sync.Mutex @@ -13,7 +14,7 @@ import javax.inject.Inject class LegacyKeyCacheMigration @Inject constructor( private val context: Context, - private val legacyDao: KeyCacheLegacyDao, + private val legacyDao: Lazy, private val timeStamper: TimeStamper ) { @@ -29,7 +30,12 @@ class LegacyKeyCacheMigration @Inject constructor( if (isInit) return isInit = true - legacyDao.clear() + try { + legacyDao.get().clear() + } catch (e: Exception) { + // Not good, but not a problem, we don't need the actual entities for migration. + Timber.tag(TAG).w(e, "Failed to clear legacy key cache from db.") + } try { cacheDir.listFiles()?.forEach { file -> @@ -53,16 +59,31 @@ class LegacyKeyCacheMigration @Inject constructor( } } - suspend fun getLegacyFile(fileMD5: String): File? = workMutex.withLock { + suspend fun tryMigration(fileMD5: String?, targetPath: File): Boolean = workMutex.withLock { + if (fileMD5 == null) return false tryInit() - legacyCacheMap[fileMD5] - } - suspend fun delete(fileMD5: String) = workMutex.withLock { - tryInit() - Timber.tag(TAG).v("delete(md5=%s)", fileMD5) - val removedFile = legacyCacheMap.remove(fileMD5) - if (removedFile?.delete() == true) Timber.tag(TAG).d("Deleted %s", removedFile) + val legacyFile = legacyCacheMap[fileMD5] ?: return false + Timber.tag(TAG).i("Migrating legacy file for %s to %s", fileMD5, targetPath) + + return try { + legacyFile.inputStream().use { from -> + targetPath.outputStream().use { to -> + from.copyTo(to, DEFAULT_BUFFER_SIZE) + } + } + true + } catch (e: Exception) { + Timber.tag(TAG).e(e, "Failed to migrate %s", legacyFile) + false + } finally { + try { + val removedFile = legacyCacheMap.remove(fileMD5) + if (removedFile?.delete() == true) Timber.tag(TAG).d("Deleted %s", removedFile) + } catch (e: Exception) { + Timber.tag(TAG).e(e, "Failed to delete %s", legacyFile) + } + } } companion object { diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt index 68bdc4d20a8..a45a79487ab 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepositoryTest.kt @@ -2,7 +2,6 @@ package de.rki.coronawarnapp.diagnosiskeys.storage import android.content.Context import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode -import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration import de.rki.coronawarnapp.util.TimeStamper import io.kotest.matchers.shouldBe import io.mockk.MockKAnnotations @@ -28,9 +27,6 @@ class KeyCacheRepositoryTest : BaseIOTest() { @MockK lateinit var timeStamper: TimeStamper - @MockK - lateinit var migrator: LegacyKeyCacheMigration - @MockK lateinit var databaseFactory: KeyCacheDatabase.Factory @@ -54,8 +50,6 @@ class KeyCacheRepositoryTest : BaseIOTest() { every { databaseFactory.create() } returns database every { database.cachedKeyFiles() } returns keyfileDAO - coEvery { migrator.migrate(any()) } returns Unit - coEvery { keyfileDAO.getAllEntries() } returns emptyList() } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt index 4c0bd9d0360..3cf5cc0de0a 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt @@ -1,28 +1,41 @@ package de.rki.coronawarnapp.diagnosiskeys.storage.legacy -import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import android.content.Context +import android.database.SQLException +import dagger.Lazy +import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 +import de.rki.coronawarnapp.util.TimeStamper import io.kotest.matchers.shouldBe import io.mockk.MockKAnnotations import io.mockk.clearAllMocks import io.mockk.coEvery import io.mockk.coVerify +import io.mockk.every import io.mockk.impl.annotations.MockK +import io.mockk.mockk import kotlinx.coroutines.runBlocking +import org.joda.time.Duration +import org.joda.time.Instant import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import testhelpers.BaseIOTest import java.io.File +import java.io.IOException class LegacyKeyCacheMigrationTest : BaseIOTest() { @MockK - lateinit var keyCacheRepository: KeyCacheRepository + lateinit var context: Context + + @MockK + lateinit var timeStamper: TimeStamper @MockK lateinit var legacyDao: KeyCacheLegacyDao private val testDir = File(IO_TEST_BASEDIR, this::class.simpleName!!) + private val legacyDir = File(testDir, "key-export") @BeforeEach fun setup() { @@ -30,7 +43,10 @@ class LegacyKeyCacheMigrationTest : BaseIOTest() { testDir.mkdirs() testDir.exists() shouldBe true - coEvery { legacyDao.getAllEntries() } returns emptyList() + every { context.cacheDir } returns testDir + every { timeStamper.nowUTC } returns Instant.EPOCH + + coEvery { legacyDao.clear() } returns Unit } @AfterEach @@ -40,49 +56,138 @@ class LegacyKeyCacheMigrationTest : BaseIOTest() { } private fun createTool() = LegacyKeyCacheMigration( - legacyDao = legacyDao + context = context, + legacyDao = Lazy { legacyDao }, + timeStamper = timeStamper ) @Test - fun `no legacy files to migrate`() { + fun `nothing happens on null checksum`() { val tool = createTool() runBlocking { - tool.migrate(keyCacheRepository) + tool.tryMigration(null, File(testDir, "something")) } - coVerify(exactly = 0) { - keyCacheRepository.createCacheEntry( - type = any(), - location = any(), - dayIdentifier = any(), - hourIdentifier = any() - ) + coVerify(exactly = 0) { legacyDao.clear() } + } + + @Test + fun `migrate a file successfully`() { + val legacyFile1 = File(legacyDir, "1234.zip") + legacyFile1.parentFile!!.mkdirs() + legacyFile1.writeText("testdata") + legacyFile1.exists() shouldBe true + + val legacyFile1MD5 = legacyFile1.hashToMD5() + legacyFile1MD5.isNotEmpty() shouldBe true + + val migrationTarget = File(testDir, "migratedkey.zip") + + val tool = createTool() + runBlocking { + tool.tryMigration(legacyFile1MD5, migrationTarget) } + + legacyFile1.exists() shouldBe false + migrationTarget.exists() shouldBe true + migrationTarget.hashToMD5() shouldBe legacyFile1MD5 + + coVerify(exactly = 1) { legacyDao.clear() } } @Test - fun `migrate two legacy files`() { + fun `migrating a single file fails gracefully`() { + val legacyFile1 = File(legacyDir, "1234.zip") + legacyFile1.parentFile!!.mkdirs() + legacyFile1.writeText("testdata") + legacyFile1.exists() shouldBe true + + val legacyFile1MD5 = legacyFile1.hashToMD5() + legacyFile1MD5.isNotEmpty() shouldBe true + + val migrationTarget = mockk() + every { migrationTarget.path } throws IOException() + val tool = createTool() - val legacyItem1 = KeyCacheLegacyEntity( + runBlocking { + tool.tryMigration(legacyFile1MD5, migrationTarget) + } - ) - val legacyItem2 = KeyCacheLegacyEntity( + legacyFile1.exists() shouldBe false - ) + coVerify(exactly = 1) { legacyDao.clear() } + } + + @Test + fun `legacy app database can crash, we don't care`() { + val legacyFile1 = File(legacyDir, "1234.zip") + legacyFile1.parentFile!!.mkdirs() + legacyFile1.writeText("testdata") + legacyFile1.exists() shouldBe true - coEvery { legacyDao.getAllEntries() } returns listOf(legacyItem1, legacyItem2) + val legacyFile1MD5 = legacyFile1.hashToMD5() + legacyFile1MD5.isNotEmpty() shouldBe true + val migrationTarget = File(testDir, "migratedkey.zip") + + coEvery { legacyDao.clear() } throws SQLException() + + val tool = createTool() runBlocking { - tool.migrate(keyCacheRepository) + tool.tryMigration(legacyFile1MD5, migrationTarget) } - coVerify(exactly = 0) { - keyCacheRepository.createCacheEntry( - type = any(), - location = any(), - dayIdentifier = any(), - hourIdentifier = any() - ) + legacyFile1.exists() shouldBe false + migrationTarget.exists() shouldBe true + migrationTarget.hashToMD5() shouldBe legacyFile1MD5 + + coVerify(exactly = 1) { legacyDao.clear() } + } + + @Test + fun `init failure causes legacy cache to be cleared`() { + val legacyFile1 = File(legacyDir, "1234.zip") + legacyFile1.parentFile!!.mkdirs() + legacyFile1.writeText("testdata") + + val legacyFile1MD5 = legacyFile1.hashToMD5() + legacyFile1MD5.isNotEmpty() shouldBe true + + legacyFile1.setReadable(false) + + val migrationTarget = File(testDir, "migratedkey.zip") + + val tool = createTool() + runBlocking { + tool.tryMigration(legacyFile1MD5, migrationTarget) } + + legacyFile1.exists() shouldBe false + migrationTarget.exists() shouldBe false + } + + @Test + fun `stale legacy files (older than 15 days) are cleaned up on init`() { + val legacyFile1 = File(legacyDir, "1234.zip") + legacyFile1.parentFile!!.mkdirs() + legacyFile1.writeText("testdata") + + val legacyFile1MD5 = legacyFile1.hashToMD5() + legacyFile1MD5.isNotEmpty() shouldBe true + + every { timeStamper.nowUTC } returns Instant.ofEpochMilli(legacyFile1.lastModified()) + .plus(Duration.standardDays(16)) + + val migrationTarget = File(testDir, "migratedkey.zip") + + coEvery { legacyDao.clear() } throws SQLException() + + val tool = createTool() + runBlocking { + tool.tryMigration(legacyFile1MD5, migrationTarget) + } + + legacyFile1.exists() shouldBe false + migrationTarget.exists() shouldBe false } } From 890553217c896622147d7c9fd42fb9738fac0f20 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 11:55:32 +0200 Subject: [PATCH 07/29] Add fallback for different file hashes in the header. --- .../diagnosiskeys/download/KeyFileDownloader.kt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 93ad62e717c..345bece66a3 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -252,11 +252,14 @@ class KeyFileDownloader @Inject constructor( private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, saveTo: File) { val validation = object : DownloadServer.HeaderValidation { override suspend fun validate(headers: Headers): Boolean { - // TODO get MD5 from better header, ETag isn't guaranteed to be the files MD5 - val fileMD5 = headers.values("ETag") - .singleOrNull() - ?.removePrefix("\"") - ?.removeSuffix("\"") + var fileMD5 = headers.values("cwa-hash-md5").singleOrNull() + if (fileMD5 == null) { + headers.values("cwa-hash").singleOrNull() + } + if (fileMD5 == null) { // Fallback + fileMD5 = headers.values("ETag").singleOrNull() + } + fileMD5 = fileMD5?.removePrefix("\"")?.removeSuffix("\"") return !legacyKeyCache.tryMigration(fileMD5, saveTo) } From 15ec9e53377b7bee836d4b6f64e51d3d46d2504e Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 15:56:07 +0200 Subject: [PATCH 08/29] Yes kLint, we know it's a long method, but for this it's better to read it in one block vs jumping to extra methods. --- .../download/KeyFileDownloader.kt | 19 ++++++++++--------- .../diagnosiskeys/server/DownloadApiV1.kt | 9 ++++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 345bece66a3..4da00edf080 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -57,10 +57,10 @@ class KeyFileDownloader @Inject constructor( Timber.tag(TAG).v("Available server data: %s", availableCountries) val availableKeys = if (CWADebug.isDebugBuildOrMode && LocalData.last3HoursMode()) { - fetchMissing3Hours(currentDate, availableCountries) + syncMissing3Hours(currentDate, availableCountries) keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) } else { - fetchMissingDays(availableCountries) + syncMissingDays(availableCountries) keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) } @@ -84,7 +84,8 @@ class KeyFileDownloader @Inject constructor( * Fetches files given by serverDates by respecting countries * @param availableCountries pair of dates per country code */ - private suspend fun fetchMissingDays( + @Suppress("LongMethod") + private suspend fun syncMissingDays( availableCountries: List ) = withContext(Dispatchers.IO) { val availableCountriesWithDays = availableCountries.map { @@ -103,11 +104,10 @@ class KeyFileDownloader @Inject constructor( it.country == cachedKeyFile.location } if (availableCountry == null) { - Timber.tag(TAG) - .w( - "Unknown location %s, assuming stale cache.", - cachedKeyFile.location - ) + Timber.tag(TAG).w( + "Unknown location %s, assuming stale cache.", + cachedKeyFile.location + ) return@filter true // It's stale } @@ -169,7 +169,8 @@ class KeyFileDownloader @Inject constructor( * @param currentDate base for where only dates within 3 hours before will be fetched * @param availableCountries pair of dates per country code */ - private suspend fun fetchMissing3Hours( + @Suppress("LongMethod") + private suspend fun syncMissing3Hours( currentDate: LocalDate, availableCountries: List ) = withContext(Dispatchers.IO) { diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt index cdb840d860f..7beb8aa5aba 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt @@ -11,17 +11,20 @@ interface DownloadApiV1 { @GET("/version/v1/configuration/country/{country}/app_config") suspend fun getApplicationConfiguration(@Path("country") country: String): ResponseBody + // TODO Let retrofit format this to CountryCode @GET("/version/v1/diagnosis-keys/country") - suspend fun getCountryIndex(): List // TODO Let retrofit format this to CountryCode + suspend fun getCountryIndex(): List + // TODO Let retrofit format this to LocalDate @GET("/version/v1/diagnosis-keys/country/{country}/date") - suspend fun getDayIndex(@Path("country") country: String): List // TODO Let retrofit format this to LocalDate + suspend fun getDayIndex(@Path("country") country: String): List + // TODO Let retrofit format this to LocalTime @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}/hour") suspend fun getHourIndex( @Path("country") country: String, @Path("day") day: String - ): List // TODO Let retrofit format this to LocalTime + ): List @Streaming @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}") From 42690190446a9385ce01d686e28d35a1368f208c Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 18:07:57 +0200 Subject: [PATCH 09/29] More linting issues, adjusting project code style prevent a few of these in the future. --- .idea/codeStyles/Project.xml | 2 ++ .../rki/coronawarnapp/diagnosiskeys/download/CountryData.kt | 1 - .../rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt | 1 - .../rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt | 4 ++-- .../rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt | 2 -- .../coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt | 2 -- .../src/main/java/de/rki/coronawarnapp/http/HttpModule.kt | 1 - .../src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt | 1 - .../transaction/RetrieveDiagnosisKeysTransaction.kt | 2 +- .../src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt | 1 - .../src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt | 1 - 11 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 32327027218..aa533f1bc24 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -125,6 +125,8 @@ diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt index 01116c9c97a..f9b4abd9485 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/CountryData.kt @@ -8,7 +8,6 @@ import org.joda.time.LocalTime sealed class CountryData { abstract val country: LocationCode - } internal data class CountryDays( diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt index 7beb8aa5aba..d7845770842 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt @@ -40,5 +40,4 @@ interface DownloadApiV1 { @Path("day") day: String, @Path("hour") hour: String ): Response - } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt index 96ed0d9c453..c7a7bbb4b25 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt @@ -77,7 +77,8 @@ class DownloadServer @Inject constructor( suspend fun getDayIndex(location: LocationCode): List = withContext(Dispatchers.IO) { api .getDayIndex(location.identifier) - .map { dayString -> // 2020-08-19 + .map { dayString -> + // 2020-08-19 LocalDate.parse(dayString, DAY_FORMATTER) } } @@ -145,6 +146,5 @@ class DownloadServer @Inject constructor( private const val EXPORT_BINARY_FILE_NAME = "export.bin" private const val EXPORT_SIGNATURE_FILE_NAME = "export.sig" - } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt index 9c738bf566f..b77f11c7be3 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/CachedKeyInfo.kt @@ -82,5 +82,3 @@ data class CachedKeyInfo( @ColumnInfo(name = "completed") val isDownloadComplete: Boolean ) } - - diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt index 77cdbc22265..e4be6ba4700 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabase.kt @@ -54,11 +54,9 @@ abstract class KeyCacheDatabase : RoomDatabase() { .databaseBuilder(context, KeyCacheDatabase::class.java, DATABASE_NAME) .fallbackToDestructiveMigrationFrom() .build() - } companion object { private const val DATABASE_NAME = "keycache.db" } - } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt index 055746655bc..a8c68ad05ff 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/HttpModule.kt @@ -44,7 +44,6 @@ class HttpModule { interceptors.forEach { addInterceptor(it) } }.build() - } @Reusable diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt index 364e057eaa1..7e9712436e2 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/ServiceFactory.kt @@ -75,7 +75,6 @@ class ServiceFactory @Inject constructor( private const val HTTP_CACHE_SIZE = 10L * 1024L * 1024L // 10 MiB private const val HTTP_CACHE_FOLDER = "http_cache" // /cache/http_cache - /** * For Submission and Verification we want to limit our specifications for TLS. */ diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt index 3ddcdc5f90d..1d9b7161797 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt @@ -312,7 +312,7 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { countries: List ) = executeState(FILES_FROM_WEB_REQUESTS) { FileStorageHelper.initializeExportSubDirectory() - val convertedDate = LocalDate.fromDateFields(currentDate) // TODO confirm + val convertedDate = LocalDate.fromDateFields(currentDate) // TODO confirm keyFileDownloader.asyncFetchKeyFiles(convertedDate, countries) } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt index 6c1a64b76a1..b99b9f2ccd8 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/HashExtensions.kt @@ -36,5 +36,4 @@ internal object HashExtensions { md.digest() } .formatHash() - } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt index 29dae7d3e37..fcbfc75d600 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/TimeStamper.kt @@ -9,5 +9,4 @@ class TimeStamper @Inject constructor() { val nowUTC: Instant get() = Instant.now() - } From e8f43fbf2fa9f187aaffe2006829e63817ccad92 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 23:02:34 +0200 Subject: [PATCH 10/29] Added missing unit tests for `KeyFileDownloader` and fixed faulty behavior that was noticed during testing. --- .../TestForAPIFragment.kt | 4 +- .../download/KeyFileDownloader.kt | 53 +- .../diagnosiskeys/server/DownloadServer.kt | 16 +- .../diagnosiskeys/server/LocationCode.kt | 9 +- .../rki/coronawarnapp/storage/AppSettings.kt | 11 + .../coronawarnapp/storage/DeviceStorage.kt | 16 +- .../RetrieveDiagnosisKeysTransaction.kt | 4 +- .../download/KeyFileDownloaderTest.kt | 635 +++++++++++++++--- .../server/DownloadServerTest.kt | 4 +- .../storage/DeviceStorageTest.kt | 131 ++-- .../RetrieveDiagnosisKeysTransactionTest.kt | 12 +- 11 files changed, 680 insertions(+), 215 deletions(-) create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppSettings.kt diff --git a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt index 971aeb324b2..016e0aafac1 100644 --- a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt +++ b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt @@ -33,6 +33,7 @@ import com.google.zxing.integration.android.IntentIntegrator import com.google.zxing.integration.android.IntentResult import com.google.zxing.qrcode.QRCodeWriter import de.rki.coronawarnapp.databinding.FragmentTestForAPIBinding +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.exception.ExceptionCategory import de.rki.coronawarnapp.exception.ExceptionCategory.INTERNAL import de.rki.coronawarnapp.exception.TransactionException @@ -323,7 +324,8 @@ class TestForAPIFragment : Fragment(), InternalExposureNotificationPermissionHel // Trigger asyncFetchFiles which will use all Countries passed as parameter val currentDate = LocalDate.now() lifecycleScope.launch { - AppInjector.component.keyFileDownloader.asyncFetchKeyFiles(currentDate, countryCodes) + val locationCodes = countryCodes.map { LocationCode(it) } + AppInjector.component.keyFileDownloader.asyncFetchKeyFiles(currentDate, locationCodes) updateCountryStatusLabel() } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 4da00edf080..0e5c78badeb 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -6,8 +6,8 @@ import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration -import de.rki.coronawarnapp.storage.FileStorageHelper -import de.rki.coronawarnapp.storage.LocalData +import de.rki.coronawarnapp.storage.AppSettings +import de.rki.coronawarnapp.storage.DeviceStorage import de.rki.coronawarnapp.util.CWADebug import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 import de.rki.coronawarnapp.util.TimeAndDateExtensions @@ -20,6 +20,7 @@ import okhttp3.Headers import org.joda.time.LocalDate import timber.log.Timber import java.io.File +import java.io.IOException import javax.inject.Inject /** @@ -27,9 +28,11 @@ import javax.inject.Inject */ @Reusable class KeyFileDownloader @Inject constructor( + private val deviceStorage: DeviceStorage, private val downloadServer: DownloadServer, private val keyCache: KeyCacheRepository, - private val legacyKeyCache: LegacyKeyCacheMigration + private val legacyKeyCache: LegacyKeyCacheMigration, + private val settings: AppSettings ) { /** @@ -46,21 +49,30 @@ class KeyFileDownloader @Inject constructor( */ suspend fun asyncFetchKeyFiles( currentDate: LocalDate, - countries: List + wantedCountries: List ): List = withContext(Dispatchers.IO) { - // Initiate key-cache folder needed for saving downloaded key files - FileStorageHelper.initializeExportSubDirectory() // TODO replace + val availableCountries = downloadServer.getCountryIndex() + val intersection = availableCountries.filter { wantedCountries.contains(it) } + Timber.tag(TAG).v( + "Available=%s; Wanted=%s; Intersect=%s", + availableCountries, wantedCountries, intersection + ) - checkForFreeSpace() // TODO replace + val storageResult = deviceStorage.checkSpacePrivateStorage( + // 512KB per day file, for 15 days, for each country ~ 65MB for 9 countries + requiredBytes = intersection.size * 15 * 512 * 1024L + ) - val availableCountries = downloadServer.getCountryIndex(countries) - Timber.tag(TAG).v("Available server data: %s", availableCountries) + Timber.tag(TAG).d("Storage check result: %s", storageResult) + if (!storageResult.isSpaceAvailable) { + throw IOException("Not enough free space (${storageResult.freeBytes}") + } - val availableKeys = if (CWADebug.isDebugBuildOrMode && LocalData.last3HoursMode()) { - syncMissing3Hours(currentDate, availableCountries) + val availableKeys = if (CWADebug.isDebugBuildOrMode && settings.isLast3HourModeEnabled) { + syncMissing3Hours(currentDate, intersection) keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) } else { - syncMissingDays(availableCountries) + syncMissingDays(intersection) keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) } @@ -77,9 +89,6 @@ class KeyFileDownloader @Inject constructor( .also { Timber.tag(TAG).d("Returning %d available keyfiles", it.size) } } - // TODO replace - private fun checkForFreeSpace() = FileStorageHelper.checkFileStorageFreeSpace() - /** * Fetches files given by serverDates by respecting countries * @param availableCountries pair of dates per country code @@ -202,7 +211,7 @@ class KeyFileDownloader @Inject constructor( return@filter true // It's stale } - val availableDay = availCountry.hourData.get(currentDate) + val availableDay = availCountry.hourData[cachedHour.day] if (availableDay == null) { Timber.d("Unknown day %s, assuming stale.", cachedHour.location) return@filter true // It's stale @@ -231,16 +240,20 @@ class KeyFileDownloader @Inject constructor( type = CachedKeyInfo.Type.COUNTRY_HOUR ) - downloadKeyFile(keyInfo, path) - - return@async keyInfo to path + return@async try { + downloadKeyFile(keyInfo, path) + keyInfo to path + } catch (e: Exception) { + Timber.tag(TAG).e(e, "Download failed: %s", keyInfo) + null + } } } } } Timber.tag(TAG).d("Waiting for %d missing hour downloads.", hourDownloads.size) - val downloadedHours = hourDownloads.awaitAll() + val downloadedHours = hourDownloads.awaitAll().filterNotNull() downloadedHours.map { (keyInfo, path) -> Timber.tag(TAG).d("Downloaded keyfile: %s to %s", keyInfo, path) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt index c7a7bbb4b25..09aba55e3ae 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt @@ -15,7 +15,6 @@ import org.joda.time.LocalTime import org.joda.time.format.DateTimeFormat import timber.log.Timber import java.io.File -import java.util.Locale import javax.inject.Inject import javax.inject.Singleton @@ -57,20 +56,9 @@ class DownloadServer @Inject constructor( } } - /** - * Gets the country index which is then filtered by given filter param or if param not set - * @param wantedCountries (array of country codes) used to filter - * only wanted countries of the country index (case insensitive) - */ - suspend fun getCountryIndex( - wantedCountries: List - ): List = withContext(Dispatchers.IO) { + suspend fun getCountryIndex(): List = withContext(Dispatchers.IO) { api - .getCountryIndex().filter { - wantedCountries - .map { c -> c.toUpperCase(Locale.ROOT) } - .contains(it.toUpperCase(Locale.ROOT)) - } + .getCountryIndex() .map { LocationCode(it) } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt index 62f7822c8a7..8201571a46b 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/LocationCode.kt @@ -1,5 +1,10 @@ package de.rki.coronawarnapp.diagnosiskeys.server +import java.util.Locale + data class LocationCode( - val identifier: String -) + private val rawIdentifier: String +) { + @Transient + val identifier: String = rawIdentifier.toUpperCase(Locale.ROOT) +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppSettings.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppSettings.kt new file mode 100644 index 00000000000..f687e24adf5 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/AppSettings.kt @@ -0,0 +1,11 @@ +package de.rki.coronawarnapp.storage + +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class AppSettings @Inject constructor() { + + val isLast3HourModeEnabled: Boolean + get() = LocalData.last3HoursMode() +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt index 40bb35e9cbd..e68516a19df 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt @@ -6,10 +6,11 @@ import android.content.Context import android.os.Build import android.os.storage.StorageManager import android.text.format.Formatter -import androidx.annotation.WorkerThread import dagger.Reusable import de.rki.coronawarnapp.util.ApiLevel import de.rki.coronawarnapp.util.storage.StatsFsProvider +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import timber.log.Timber import java.io.File import java.io.IOException @@ -75,9 +76,10 @@ class DeviceStorage @Inject constructor( ) } - @WorkerThread - @Throws(IOException::class) - private fun checkSpace(path: File, requiredBytes: Long = -1L): CheckResult { + private suspend fun checkSpace( + path: File, + requiredBytes: Long = -1L + ): CheckResult = withContext(Dispatchers.IO) { try { Timber.tag(TAG).v("checkSpace(path=%s, requiredBytes=%d)", path, requiredBytes) val result: CheckResult = if (apiLevel.hasAPILevel(Build.VERSION_CODES.O)) { @@ -92,7 +94,7 @@ class DeviceStorage @Inject constructor( } Timber.tag(TAG).d("Requested %d from %s: %s", requiredBytes, path, result) - return result + return@withContext result } catch (e: Exception) { throw IOException("checkSpace(path=$path, requiredBytes=$requiredBytes) FAILED", e) .also { Timber.tag(TAG).e(it) } @@ -108,9 +110,7 @@ class DeviceStorage @Inject constructor( * * @throws IOException if storage check or allocation fails. */ - @WorkerThread - @Throws(IOException::class) - fun checkSpacePrivateStorage(requiredBytes: Long = -1L): CheckResult = + suspend fun checkSpacePrivateStorage(requiredBytes: Long = -1L): CheckResult = checkSpace(privateStorage, requiredBytes) data class CheckResult( diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt index 1d9b7161797..73ad7ebed78 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt @@ -21,6 +21,7 @@ package de.rki.coronawarnapp.transaction import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration import de.rki.coronawarnapp.diagnosiskeys.download.KeyFileDownloader +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.nearby.InternalExposureNotificationClient import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigurationService @@ -313,7 +314,8 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { ) = executeState(FILES_FROM_WEB_REQUESTS) { FileStorageHelper.initializeExportSubDirectory() val convertedDate = LocalDate.fromDateFields(currentDate) // TODO confirm - keyFileDownloader.asyncFetchKeyFiles(convertedDate, countries) + val locationCodes = countries.map { LocationCode(it) } + keyFileDownloader.asyncFetchKeyFiles(convertedDate, locationCodes) } /** diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt index 6580d1032fa..1012ad9a321 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -1,167 +1,594 @@ package de.rki.coronawarnapp.diagnosiskeys.download -import android.content.Context +import android.database.SQLException +import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository +import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration +import de.rki.coronawarnapp.storage.AppSettings +import de.rki.coronawarnapp.storage.DeviceStorage +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import io.mockk.MockKAnnotations +import io.mockk.clearAllMocks +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every import io.mockk.impl.annotations.MockK +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import org.joda.time.Instant +import org.joda.time.LocalDate +import org.joda.time.LocalTime +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import testhelpers.BaseTest +import testhelpers.BaseIOTest +import timber.log.Timber +import java.io.File +import java.io.IOException /** * CachedKeyFileHolder test. */ -class KeyFileDownloaderTest : BaseTest() { +class KeyFileDownloaderTest : BaseIOTest() { @MockK - private lateinit var keyCacheRepository: KeyCacheRepository + private lateinit var keyCache: KeyCacheRepository @MockK - private lateinit var context: Context - -// @Before -// fun setUp() { -// MockKAnnotations.init(this) -// mockkObject(CoronaWarnApplication.Companion) -// mockkObject(KeyCacheRepository.Companion) -// every { CoronaWarnApplication.getAppContext() } returns context -// every { KeyCacheRepository.getDateRepository(any()) } returns keyCacheRepository -// mockkObject(KeyFileDownloader) -// coEvery { keyCacheRepository.deleteOutdatedEntries(any()) } just Runs -// } -// -// /** -// * Test call order is correct. -// */ -// @Test -// fun testAsyncFetchFiles() { -// val date = Date() -// val countries = listOf("DE") -// val country = "DE" -// -// mockkObject(CWADebug) -// -// coEvery { keyCacheRepository.getDates() } returns listOf() -// coEvery { keyCacheRepository.getFilesFromEntries() } returns listOf() -// every { CWADebug.isDebugBuildOrMode } returns false -// every { KeyFileDownloader["checkForFreeSpace"]() } returns Unit -// every { KeyFileDownloader["getDatesFromServer"](country) } returns arrayListOf() -// -// every { CoronaWarnApplication.getAppContext().cacheDir } returns File("./") -// every { KeyFileDownloader["getCountriesFromServer"](countries) } returns countries -// -// runBlocking { -// -// KeyFileDownloader.asyncFetchFiles(date, countries) -// -// coVerifyOrder { -// KeyFileDownloader.asyncFetchFiles(date, countries) -// KeyFileDownloader["getCountriesFromServer"](countries) -// KeyFileDownloader["getDatesFromServer"](country) -// KeyFileDownloader["asyncHandleFilesFetch"]( -// listOf( -// CountryDataWrapper( -// country, -// listOf() -// ) -// ) -// ) -// keyCacheRepository.deleteOutdatedEntries(any()) -// KeyFileDownloader["getMissingDaysFromDiff"]( -// listOf( -// CountryDataWrapper( -// country, -// listOf() -// ) -// ) -// ) -// keyCacheRepository.getDates() -// keyCacheRepository.getFilesFromEntries() -// } -// } -// } - -// -// @After -// fun cleanUp() { -// unmockkAll() -// } + private lateinit var legacyMigration: LegacyKeyCacheMigration + + @MockK + private lateinit var downloadServer: DownloadServer + + @MockK + private lateinit var deviceStorage: DeviceStorage + + @MockK + private lateinit var settings: AppSettings + + private val testDir = File(IO_TEST_BASEDIR, this::class.simpleName!!) + private val keyRepoData = mutableMapOf() + + @BeforeEach + fun setup() { + MockKAnnotations.init(this) + testDir.mkdirs() + testDir.exists() shouldBe true + + coEvery { downloadServer.getCountryIndex() } returns listOf( + LocationCode("DE"), + LocationCode("NL") + ) + coEvery { deviceStorage.checkSpacePrivateStorage(any()) } returns mockk().apply { + every { isSpaceAvailable } returns true + } + + coEvery { settings.isLast3HourModeEnabled } returns false + + + coEvery { downloadServer.getCountryIndex() } returns listOf( + LocationCode("DE"), LocationCode("NL") + ) + coEvery { downloadServer.getDayIndex(LocationCode("DE")) } returns listOf( + LocalDate.parse("2020-09-01"), LocalDate.parse("2020-09-02") + ) + coEvery { + downloadServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-01")) + } returns listOf( + LocalTime.parse("20") + ) + coEvery { + downloadServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-02")) + } returns listOf( + LocalTime.parse("20"), LocalTime.parse("21") + ) + coEvery { downloadServer.getDayIndex(LocationCode("NL")) } returns listOf( + LocalDate.parse("2020-09-02"), LocalDate.parse("2020-09-03") + ) + coEvery { + downloadServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-02")) + } returns listOf( + LocalTime.parse("22") + ) + coEvery { + downloadServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-03")) + } returns listOf( + LocalTime.parse("22"), LocalTime.parse("23") + ) + coEvery { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { + mockDownloadServerDownload(arg(0), arg(1), arg(2), arg(3), arg(4)) + } + + coEvery { keyCache.createCacheEntry(any(), any(), any(), any()) } answers { + mockKeyCacheCreateEntry(arg(0), arg(1), arg(2), arg(3)) + } + coEvery { keyCache.markKeyComplete(any(), any()) } answers { + mockKeyCacheUpdateComplete(arg(0), arg(1)) + } + coEvery { keyCache.getEntriesForType(any()) } answers { + val type = arg(0) + keyRepoData.values.filter { it.type == type }.map { it to File(testDir, it.id) } + } + coEvery { keyCache.getAllCachedKeys() } returns keyRepoData.values.map { + it to File( + testDir, + it.id + ) + } + coEvery { keyCache.delete(any()) } answers { + val keyInfos = arg>(0) + keyInfos.forEach { + keyRepoData.remove(it.id) + } + } + } + + @AfterEach + fun teardown() { + clearAllMocks() + keyRepoData.clear() + testDir.deleteRecursively() + } + + private fun mockKeyCacheCreateEntry( + type: CachedKeyInfo.Type, + location: LocationCode, + dayIdentifier: LocalDate, + hourIdentifier: LocalTime? + ): Pair { + val keyInfo = CachedKeyInfo( + type = type, + location = location, + day = dayIdentifier, + hour = hourIdentifier, + createdAt = Instant.now() + ) + Timber.i("mockKeyCacheCreateEntry(...): %s", keyInfo) + val file = File(testDir, keyInfo.id) + keyRepoData[keyInfo.id] = keyInfo + return keyInfo to file + } + + private fun mockKeyCacheUpdateComplete( + keyInfo: CachedKeyInfo, checksum: String + ) { + keyRepoData[keyInfo.id] = keyInfo.copy( + isDownloadComplete = checksum != null, checksumMD5 = checksum + ) + } + + private fun mockDownloadServerDownload( + locationCode: LocationCode, + day: LocalDate, + hour: LocalTime? = null, + saveTo: File, + validator: DownloadServer.HeaderValidation = object : DownloadServer.HeaderValidation {} + ) { + saveTo.writeText("$locationCode.$day.$hour") + } + + private fun mockAddData( + type: CachedKeyInfo.Type, + location: LocationCode, + day: LocalDate, + hour: LocalTime?, + isCompleted: Boolean + ): Pair { + val (keyInfo, file) = mockKeyCacheCreateEntry(type, location, day, hour) + if (isCompleted) { + mockDownloadServerDownload(location, day, hour, file) + mockKeyCacheUpdateComplete(keyInfo, "checksum") + } + return keyRepoData[keyInfo.id]!! to file + } + + private fun createDownloader(): KeyFileDownloader { + val downloader = KeyFileDownloader( + deviceStorage = deviceStorage, + downloadServer = downloadServer, + keyCache = keyCache, + legacyKeyCache = legacyMigration, + settings = settings + ) + Timber.i("createDownloader(): %s", downloader) + return downloader + } @Test - fun `error during country index fetch`() { - TODO() + fun `storage is checked before fetching`() { + val downloader = createDownloader() + runBlocking { + downloader.asyncFetchKeyFiles(LocalDate.now(), emptyList()) shouldBe emptyList() + } + } + + @Test + fun `fetching is aborted if not enough free storage`() { + coEvery { deviceStorage.checkSpacePrivateStorage(any()) } returns mockk().apply { + every { isSpaceAvailable } returns false + every { freeBytes } returns 1337L + } + + val downloader = createDownloader() + + runBlocking { + shouldThrow { + downloader.asyncFetchKeyFiles(LocalDate.now(), listOf(LocationCode("DE"))) + } + } } @Test - fun `fetched country index is empty`() { - TODO() + fun `error during country index fetch`() { + coEvery { downloadServer.getCountryIndex() } throws IOException() + + val downloader = createDownloader() + + runBlocking { + shouldThrow { + downloader.asyncFetchKeyFiles(LocalDate.now(), listOf(LocationCode("DE"))) + } + } } @Test fun `day fetch without prior data`() { - TODO() + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.now(), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 4 + } + + coVerify { + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-01"), + hourIdentifier = null + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = null + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = null + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = null + ) + } + keyRepoData.size shouldBe 4 + keyRepoData.values.forEach { it.isDownloadComplete shouldBe true } } @Test fun `day fetch with existing data`() { - TODO() + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + day = LocalDate.parse("2020-09-01"), + hour = null, + isCompleted = true + ) + + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("NL"), + day = LocalDate.parse("2020-09-02"), + hour = null, + isCompleted = true + ) + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.now(), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 4 + } + + coVerify { + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = null + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = null + ) + } + + coVerify(exactly = 2) { keyCache.createCacheEntry(any(), any(), any(), any()) } + coVerify(exactly = 2) { keyCache.markKeyComplete(any(), any()) } } @Test fun `day fetch deletes stale data`() { - TODO() - } + coEvery { downloadServer.getDayIndex(LocationCode("DE")) } returns listOf( + LocalDate.parse("2020-09-02") + ) + val (staleKeyInfo, _) = mockAddData( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + day = LocalDate.parse("2020-09-01"), + hour = null, + isCompleted = true + ) - @Test - fun `day fetch marks downloads as complete`() { - TODO() + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("NL"), + day = LocalDate.parse("2020-09-02"), + hour = null, + isCompleted = true + ) + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.now(), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 3 + } + + coVerify { + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = null + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = null + ) + } + coVerify(exactly = 1) { keyCache.delete(listOf(staleKeyInfo)) } + coVerify(exactly = 2) { keyCache.createCacheEntry(any(), any(), any(), any()) } + coVerify(exactly = 2) { keyCache.markKeyComplete(any(), any()) } } @Test fun `day fetch skips single download failures`() { - TODO() + var dlCounter = 0 + coEvery { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { + dlCounter++ + if (dlCounter == 2) throw IOException("Timeout") + mockDownloadServerDownload(arg(0), arg(1), arg(2), arg(3), arg(4)) + } + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.now(), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 3 + } } @Test fun `last3Hours fetch without prior data`() { - TODO() + every { settings.isLast3HourModeEnabled } returns true + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.parse("2020-09-02"), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 3 + } + + coVerify { + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = LocalTime.parse("20") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = LocalTime.parse("21") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = LocalTime.parse("22") + ) + } + coVerify(exactly = 3) { keyCache.markKeyComplete(any(), any()) } + + keyRepoData.size shouldBe 3 + keyRepoData.values.forEach { it.isDownloadComplete shouldBe true } } @Test fun `last3Hours fetch with prior data`() { - TODO() + every { settings.isLast3HourModeEnabled } returns true + + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + day = LocalDate.parse("2020-09-02"), + hour = LocalTime.parse("22"), + isCompleted = true + ) + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.parse("2020-09-02"), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 3 + } + + coVerify { + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = LocalTime.parse("20") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = LocalTime.parse("21") + ) + } + coVerify(exactly = 2) { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } } @Test fun `last3Hours fetch deletes stale data`() { - TODO() - } + every { settings.isLast3HourModeEnabled } returns true - @Test - fun `last3Hours fetch marks downloads as complete`() { - TODO() + val (staleKey1, _) = mockAddData( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + day = LocalDate.parse("2020-09-02"), + hour = LocalTime.parse("12"), // Stale hour + isCompleted = true + ) + + val (staleKey2, _) = mockAddData( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + day = LocalDate.parse("2020-09-01"), // Stale day + hour = LocalTime.parse("22"), + isCompleted = true + ) + + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + day = LocalDate.parse("2020-09-02"), + hour = LocalTime.parse("22"), + isCompleted = true + ) + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.parse("2020-09-02"), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 3 + } + + coVerify { + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = LocalTime.parse("20") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-02"), + hourIdentifier = LocalTime.parse("21") + ) + } + coVerify(exactly = 2) { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } + coVerify(exactly = 1) { keyCache.delete(listOf(staleKey1, staleKey2)) } } @Test fun `last3Hours fetch skips single download failures`() { - TODO() - } + every { settings.isLast3HourModeEnabled } returns true - @Test - fun `storage is checked before fetching`() { - TODO() - } + var dlCounter = 0 + coEvery { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { + dlCounter++ + if (dlCounter == 2) throw IOException("Timeout") + mockDownloadServerDownload(arg(0), arg(1), arg(2), arg(3), arg(4)) + } - @Test - fun `fetching is aborted if not enough free storage`() { - TODO() + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.parse("2020-09-02"), // Has 3 hour files + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 2 + } } @Test fun `not completed cache entries are overwritten`() { - TODO() + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + day = LocalDate.parse("2020-09-01"), + hour = null, + isCompleted = false + ) + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.now(), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 4 + } + + coVerify { + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-01"), + hourIdentifier = null + ) + } } @Test - fun `fetch returns all currently available keyfiles`() { - TODO() + fun `database errors do not abort the whole process`() { + var completionCounter = 0 + coEvery { keyCache.markKeyComplete(any(), any()) } answers { + completionCounter++ + if (completionCounter == 2) throw SQLException(":)") + mockKeyCacheUpdateComplete(arg(0), arg(1)) + } + + val downloader = createDownloader() + + runBlocking { + downloader.asyncFetchKeyFiles( + LocalDate.now(), + listOf(LocationCode("DE"), LocationCode("NL")) + ).size shouldBe 3 + } + + coVerify(exactly = 4) { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt index 12f1a52f543..98167f5af5c 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt @@ -117,10 +117,10 @@ class DownloadServerTest : BaseIOTest() { @Test fun `download country index`() { val downloadServer = createDownloadServer() - coEvery { api.getCountryIndex() } returns listOf("DE", "NL", "FR") + coEvery { api.getCountryIndex() } returns listOf("DE", "NL") runBlocking { - downloadServer.getCountryIndex(listOf("DE", "NL")) shouldBe listOf( + downloadServer.getCountryIndex() shouldBe listOf( LocationCode("DE"), LocationCode("NL") ) } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt index 84a551dc012..b49fe05caf5 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt @@ -15,6 +15,7 @@ import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.mockk import io.mockk.verify +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -84,14 +85,14 @@ class DeviceStorageTest : BaseIOTest() { @Test fun `check private storage space`() { val deviceStorage = buildInstance() - - deviceStorage.checkSpacePrivateStorage() shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = true, - freeBytes = defaultFreeSpace, - totalBytes = defaultTotalSpace - ) - + runBlocking { + deviceStorage.checkSpacePrivateStorage() shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = true, + freeBytes = defaultFreeSpace, + totalBytes = defaultTotalSpace + ) + } verify { storageManager.getUuidForPath(any()) } verify(exactly = 0) { statsFsProvider.createStats(any()) } @@ -101,14 +102,14 @@ class DeviceStorageTest : BaseIOTest() { @Test fun `check private storage space, sub API26`() { val deviceStorage = buildInstance(level = legacyApiLevel) - - deviceStorage.checkSpacePrivateStorage() shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = true, - freeBytes = defaultFreeSpace, - totalBytes = defaultTotalSpace - ) - + runBlocking { + deviceStorage.checkSpacePrivateStorage() shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = true, + freeBytes = defaultFreeSpace, + totalBytes = defaultTotalSpace + ) + } verify(exactly = 0) { storageManager.getUuidForPath(any()) } verify { statsFsProvider.createStats(any()) } } @@ -116,40 +117,42 @@ class DeviceStorageTest : BaseIOTest() { @Test fun `request space from private storage successfully`() { val deviceStorage = buildInstance() - - deviceStorage.checkSpacePrivateStorage(requiredBytes = defaultFreeSpace) shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = true, - freeBytes = defaultFreeSpace, - totalBytes = defaultTotalSpace - ) - + runBlocking { + deviceStorage.checkSpacePrivateStorage(requiredBytes = defaultFreeSpace) shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = true, + freeBytes = defaultFreeSpace, + totalBytes = defaultTotalSpace + ) + } verify(exactly = 0) { storageManager.allocateBytes(any(), any()) } } @Test fun `request space from private storage successfully, sub API26`() { val deviceStorage = buildInstance(level = legacyApiLevel) - - deviceStorage.checkSpacePrivateStorage(requiredBytes = defaultFreeSpace) shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = true, - freeBytes = defaultFreeSpace, - totalBytes = defaultTotalSpace - ) + runBlocking { + deviceStorage.checkSpacePrivateStorage(requiredBytes = defaultFreeSpace) shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = true, + freeBytes = defaultFreeSpace, + totalBytes = defaultTotalSpace + ) + } } @Test fun `request space from private storage wth allocation`() { val deviceStorage = buildInstance() - - val targetBytes = defaultFreeSpace + defaultAllocatableBytes - deviceStorage.checkSpacePrivateStorage(requiredBytes = targetBytes) shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = true, - freeBytes = targetBytes, - totalBytes = defaultTotalSpace - ) + runBlocking { + val targetBytes = defaultFreeSpace + defaultAllocatableBytes + deviceStorage.checkSpacePrivateStorage(requiredBytes = targetBytes) shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = true, + freeBytes = targetBytes, + totalBytes = defaultTotalSpace + ) + } verify { storageManager.allocateBytes(privateDataDirUUID, defaultAllocatableBytes) } } @@ -157,25 +160,27 @@ class DeviceStorageTest : BaseIOTest() { @Test fun `request space from private storage unsuccessfully`() { val deviceStorage = buildInstance() - - deviceStorage.checkSpacePrivateStorage(requiredBytes = Long.MAX_VALUE) shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = false, - freeBytes = defaultFreeSpace, - totalBytes = defaultTotalSpace - ) + runBlocking { + deviceStorage.checkSpacePrivateStorage(requiredBytes = Long.MAX_VALUE) shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = false, + freeBytes = defaultFreeSpace, + totalBytes = defaultTotalSpace + ) + } } @Test fun `request space from private storage unsuccessfully, sub API26`() { val deviceStorage = buildInstance(level = legacyApiLevel) - - deviceStorage.checkSpacePrivateStorage(requiredBytes = Long.MAX_VALUE) shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = false, - freeBytes = defaultFreeSpace, - totalBytes = defaultTotalSpace - ) + runBlocking { + deviceStorage.checkSpacePrivateStorage(requiredBytes = Long.MAX_VALUE) shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = false, + freeBytes = defaultFreeSpace, + totalBytes = defaultTotalSpace + ) + } } @Test @@ -183,13 +188,14 @@ class DeviceStorageTest : BaseIOTest() { every { storageManager.getUuidForPath(privateDataDir) } throws IOException("uh oh") val deviceStorage = buildInstance() - - deviceStorage.checkSpacePrivateStorage() shouldBe DeviceStorage.CheckResult( - path = privateDataDir, - isSpaceAvailable = true, - freeBytes = defaultFreeSpace, - totalBytes = defaultTotalSpace - ) + runBlocking { + deviceStorage.checkSpacePrivateStorage() shouldBe DeviceStorage.CheckResult( + path = privateDataDir, + isSpaceAvailable = true, + freeBytes = defaultFreeSpace, + totalBytes = defaultTotalSpace + ) + } verify { statsFsProvider.createStats(privateDataDir) } } @@ -199,7 +205,8 @@ class DeviceStorageTest : BaseIOTest() { every { statsFsProvider.createStats(privateDataDir) } throws IOException("uh oh") val deviceStorage = buildInstance(level = legacyApiLevel) - - shouldThrow { deviceStorage.checkSpacePrivateStorage() } + runBlocking { + shouldThrow { deviceStorage.checkSpacePrivateStorage() } + } } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt index a378dc9cf4b..f5d2817f6cc 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt @@ -6,6 +6,7 @@ import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigur import de.rki.coronawarnapp.storage.LocalData import de.rki.coronawarnapp.util.di.AppInjector import de.rki.coronawarnapp.util.di.ApplicationComponent +import io.kotest.matchers.shouldBe import io.mockk.Runs import io.mockk.coEvery import io.mockk.coVerifyOrder @@ -15,6 +16,7 @@ import io.mockk.mockk import io.mockk.mockkObject import io.mockk.unmockkAll import kotlinx.coroutines.runBlocking +import org.joda.time.LocalDate import org.junit.After import org.junit.Before import org.junit.Test @@ -89,7 +91,10 @@ class RetrieveDiagnosisKeysTransactionTest { coVerifyOrder { RetrieveDiagnosisKeysTransaction["executeSetup"]() RetrieveDiagnosisKeysTransaction["executeRetrieveRiskScoreParams"]() - RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"](any(), requestedCountries) + RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"]( + any(), + requestedCountries + ) RetrieveDiagnosisKeysTransaction["executeAPISubmission"]( any(), listOf(file), @@ -100,6 +105,11 @@ class RetrieveDiagnosisKeysTransactionTest { } } + @Test + fun `conversion from date to localdate`() { + LocalDate.fromDateFields(Date(0)) shouldBe LocalDate.parse("1970-01-01") + } + @After fun cleanUp() { unmockkAll() From 709c8fbbab48ce674c3c906cdbc4106e0c90bf06 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 23:17:28 +0200 Subject: [PATCH 11/29] CRUD (instrumentation) test for `KeyCacheDatabase` --- .../storage/KeyCacheDatabaseTest.kt | 85 ++++++++++++++----- 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt index a4591b02db4..ac0ccdefe87 100644 --- a/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt +++ b/Corona-Warn-App/src/androidTest/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheDatabaseTest.kt @@ -1,40 +1,79 @@ package de.rki.coronawarnapp.diagnosiskeys.storage +import android.content.Context +import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 +import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode +import io.kotest.matchers.shouldBe +import kotlinx.coroutines.runBlocking +import org.joda.time.Instant +import org.joda.time.LocalDate +import org.joda.time.LocalTime import org.junit.Test import org.junit.runner.RunWith @RunWith(AndroidJUnit4::class) class KeyCacheDatabaseTest { + private val database = KeyCacheDatabase.Factory( + ApplicationProvider.getApplicationContext() + ).create() + private val dao = database.cachedKeyFiles() @Test - fun createEntry() { - TODO() - } + fun crud() { + val keyDay = CachedKeyInfo( + type = CachedKeyInfo.Type.COUNTRY_DAY, + location = LocationCode("DE"), + day = LocalDate.now(), + hour = null, + createdAt = Instant.now() + ) + val keyHour = CachedKeyInfo( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + day = LocalDate.now(), + hour = LocalTime.now(), + createdAt = Instant.now() + ) + runBlocking { + dao.clear() - @Test - fun deleteEntry() { - TODO() - } + dao.insertEntry(keyDay) + dao.insertEntry(keyHour) + dao.getAllEntries() shouldBe listOf(keyDay, keyHour) + dao.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY.typeValue) shouldBe listOf(keyDay) + dao.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR.typeValue) shouldBe listOf(keyHour) - @Test - fun clear() { - TODO() - } + dao.updateDownloadState(keyDay.toDownloadUpdate("coffee")) + dao.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY.typeValue).single().apply { + isDownloadComplete shouldBe true + checksumMD5 shouldBe "coffee" + } + dao.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR.typeValue).single().apply { + isDownloadComplete shouldBe false + checksumMD5 shouldBe null + } - @Test - fun getAll() { - TODO() - } + dao.updateDownloadState(keyHour.toDownloadUpdate("with milk")) + dao.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY.typeValue).single().apply { + isDownloadComplete shouldBe true + checksumMD5 shouldBe "coffee" + } + dao.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR.typeValue).single().apply { + isDownloadComplete shouldBe true + checksumMD5 shouldBe "with milk" + } - @Test - fun getAllDayData() { - TODO() - } + dao.deleteEntry(keyDay) + dao.getAllEntries() shouldBe listOf( + keyHour.copy( + isDownloadComplete = true, + checksumMD5 = "with milk" + ) + ) - @Test - fun getAllHourData() { - TODO() + dao.clear() + dao.getAllEntries() shouldBe emptyList() + } } - } From 2ba7dc6413e0c35106dfb5bc88338163df7d3221 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 23:25:29 +0200 Subject: [PATCH 12/29] Remove unused `FileStorageHelper` and related constants+tests. --- .../TestRiskLevelCalculation.kt | 4 +- .../storage/FileStorageConstants.kt | 18 ---- .../storage/FileStorageHelper.kt | 90 ------------------- .../RetrieveDiagnosisKeysTransaction.kt | 2 - .../coronawarnapp/util/DataRetentionHelper.kt | 6 +- .../storage/FileStorageConstantsTest.kt | 14 --- 6 files changed, 6 insertions(+), 128 deletions(-) delete mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageConstants.kt delete mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageHelper.kt delete mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/FileStorageConstantsTest.kt diff --git a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt index 14ddf7a9291..a1cfbb2868e 100644 --- a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt +++ b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt @@ -27,7 +27,6 @@ import de.rki.coronawarnapp.server.protocols.AppleLegacyKeyExchange import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigurationService import de.rki.coronawarnapp.sharing.ExposureSharingService import de.rki.coronawarnapp.storage.AppDatabase -import de.rki.coronawarnapp.storage.FileStorageHelper import de.rki.coronawarnapp.storage.LocalData import de.rki.coronawarnapp.storage.RiskLevelRepository import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction @@ -36,6 +35,7 @@ import de.rki.coronawarnapp.ui.viewmodel.SettingsViewModel import de.rki.coronawarnapp.ui.viewmodel.SubmissionViewModel import de.rki.coronawarnapp.ui.viewmodel.TracingViewModel import de.rki.coronawarnapp.util.KeyFileHelper +import de.rki.coronawarnapp.util.di.AppInjector import de.rki.coronawarnapp.util.security.SecurityHelper import kotlinx.android.synthetic.deviceForTesters.fragment_test_risk_level_calculation.* import kotlinx.coroutines.Dispatchers @@ -112,7 +112,7 @@ class TestRiskLevelCalculation : Fragment() { // Database Reset AppDatabase.reset(requireContext()) // Export File Reset - FileStorageHelper.getAllFilesInKeyExportDirectory().forEach { it.delete() } + AppInjector.component.keyCacheRepository.clear() LocalData.lastCalculatedRiskLevel(RiskLevel.UNDETERMINED.raw) LocalData.lastSuccessfullyCalculatedRiskLevel(RiskLevel.UNDETERMINED.raw) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageConstants.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageConstants.kt deleted file mode 100644 index 9571157bb5f..00000000000 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageConstants.kt +++ /dev/null @@ -1,18 +0,0 @@ -package de.rki.coronawarnapp.storage - -/** - * The File Storage constants are used inside the FileStorageHelper - * - * @see FileStorageHelper - */ -object FileStorageConstants { - - /** Days to keep data in internal storage */ - const val DAYS_TO_KEEP: Long = 14 - - /** Size (Mb) threshold for free space check */ - const val FREE_SPACE_THRESHOLD = 15 - - /** Key export directory name in internal storage */ - const val KEY_EXPORT_DIRECTORY_NAME = "key-export" -} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageHelper.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageHelper.kt deleted file mode 100644 index 44442e9e61d..00000000000 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/FileStorageHelper.kt +++ /dev/null @@ -1,90 +0,0 @@ -package de.rki.coronawarnapp.storage - -import android.content.Context -import android.os.Build -import android.os.storage.StorageManager -import de.rki.coronawarnapp.CoronaWarnApplication -import de.rki.coronawarnapp.exception.NotEnoughSpaceOnDiskException -import timber.log.Timber -import java.io.File -import java.util.UUID -import java.util.concurrent.TimeUnit - -/** - * A helper class for file storage manipulation - * The helper uses externalised constants for readability. - * - * @see FileStorageConstants - */ -object FileStorageHelper { - - private val TAG: String? = FileStorageHelper::class.simpleName - private val TIME_TO_KEEP = TimeUnit.DAYS.toMillis(FileStorageConstants.DAYS_TO_KEEP) - private const val BYTES = 1048576 - - /** - * create the needed key export directory (recursively) - * - */ - fun initializeExportSubDirectory() = keyExportDirectory.mkdirs() - - /** - * Get key files export directory used to store all export files for the transaction - * Uses FileStorageConstants.KEY_EXPORT_DIRECTORY_NAME constant - * - * @return File of key export directory - */ - val keyExportDirectory = File( - CoronaWarnApplication.getAppContext().cacheDir, - FileStorageConstants.KEY_EXPORT_DIRECTORY_NAME - ) - - /** - * Checks if internal store has free memory. - * Threshold: FileStorageConstants.FREE_SPACE_THRESHOLD - * Bound to .usableSpace due to API level restrictions (minimum required level - 23) - */ - fun checkFileStorageFreeSpace() { - val availableSpace = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val storageManager = CoronaWarnApplication.getAppContext() - .getSystemService(Context.STORAGE_SERVICE) as StorageManager - val storageVolume = storageManager.primaryStorageVolume - val storageUUID = - UUID.fromString(storageVolume.uuid ?: StorageManager.UUID_DEFAULT.toString()) - storageManager.getAllocatableBytes(storageUUID) / BYTES - } else { - keyExportDirectory.usableSpace / BYTES - } - if (availableSpace < FileStorageConstants.FREE_SPACE_THRESHOLD) { - throw NotEnoughSpaceOnDiskException() - } - } - - fun getAllFilesInKeyExportDirectory(): List { - return keyExportDirectory - .walk(FileWalkDirection.BOTTOM_UP) - .filter(File::isFile) - .toList() - } - - fun File.isOutdated(): Boolean = - (System.currentTimeMillis() - lastModified() > TIME_TO_KEEP) - - private fun File.checkAndRemove(): Boolean { - return if (exists() && isDirectory) { - deleteRecursively() - } else { - false - } - } - - // LOGGING - private fun logFileRemovalResult(fileName: String, result: Boolean) = - Timber.d("File $fileName was deleted: $result") - - private fun logAvailableSpace(availableSpace: Long) = - Timber.d("Available space: $availableSpace") - - private fun logInsufficientSpace(availableSpace: Long) = - Timber.e("Not enough free space! Required: ${FileStorageConstants.FREE_SPACE_THRESHOLD} Has: $availableSpace") -} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt index 73ad7ebed78..08a80c8ab89 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt @@ -25,7 +25,6 @@ import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.nearby.InternalExposureNotificationClient import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigurationService -import de.rki.coronawarnapp.storage.FileStorageHelper import de.rki.coronawarnapp.storage.LocalData import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.API_SUBMISSION import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.CLOSE @@ -312,7 +311,6 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { currentDate: Date, countries: List ) = executeState(FILES_FROM_WEB_REQUESTS) { - FileStorageHelper.initializeExportSubDirectory() val convertedDate = LocalDate.fromDateFields(currentDate) // TODO confirm val locationCodes = countries.map { LocationCode(it) } keyFileDownloader.asyncFetchKeyFiles(convertedDate, locationCodes) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/DataRetentionHelper.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/DataRetentionHelper.kt index 46343586783..1ce93fc10fa 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/DataRetentionHelper.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/DataRetentionHelper.kt @@ -22,9 +22,10 @@ package de.rki.coronawarnapp.util import android.annotation.SuppressLint import android.content.Context import de.rki.coronawarnapp.storage.AppDatabase -import de.rki.coronawarnapp.storage.FileStorageHelper import de.rki.coronawarnapp.storage.RiskLevelRepository +import de.rki.coronawarnapp.util.di.AppInjector import de.rki.coronawarnapp.util.security.SecurityHelper +import kotlinx.coroutines.runBlocking import timber.log.Timber /** @@ -47,7 +48,8 @@ object DataRetentionHelper { // Reset the current risk level stored in LiveData RiskLevelRepository.reset() // Export File Reset - FileStorageHelper.getAllFilesInKeyExportDirectory().forEach { it.delete() } + // TODO runBlocking, but also all of the above is BLOCKING and should be called more nicely + runBlocking { AppInjector.component.keyCacheRepository.clear() } Timber.w("CWA LOCAL DATA DELETION COMPLETED.") } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/FileStorageConstantsTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/FileStorageConstantsTest.kt deleted file mode 100644 index a9d31199602..00000000000 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/FileStorageConstantsTest.kt +++ /dev/null @@ -1,14 +0,0 @@ -package de.rki.coronawarnapp.storage - -import org.junit.Assert -import org.junit.Test - -class FileStorageConstantsTest { - - @Test - fun allFileStorageConstants() { - Assert.assertEquals(FileStorageConstants.DAYS_TO_KEEP, 14) - Assert.assertEquals(FileStorageConstants.FREE_SPACE_THRESHOLD, 15) - Assert.assertEquals(FileStorageConstants.KEY_EXPORT_DIRECTORY_NAME, "key-export") - } -} From fc8fe5227d378a2f57dc034b3d2c72e6997a4cd3 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Wed, 9 Sep 2020 23:32:36 +0200 Subject: [PATCH 13/29] Fix last3Hours unit test in deviceRelease mode, we need to explicitly enable debug for these tests. --- .../diagnosiskeys/download/KeyFileDownloaderTest.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt index 1012ad9a321..bf42eb34856 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -8,6 +8,7 @@ import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration import de.rki.coronawarnapp.storage.AppSettings import de.rki.coronawarnapp.storage.DeviceStorage +import de.rki.coronawarnapp.util.CWADebug import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldBe import io.mockk.MockKAnnotations @@ -17,6 +18,7 @@ import io.mockk.coVerify import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.mockk +import io.mockk.mockkObject import kotlinx.coroutines.runBlocking import org.joda.time.Instant import org.joda.time.LocalDate @@ -58,6 +60,8 @@ class KeyFileDownloaderTest : BaseIOTest() { testDir.mkdirs() testDir.exists() shouldBe true + mockkObject(CWADebug) + coEvery { downloadServer.getCountryIndex() } returns listOf( LocationCode("DE"), LocationCode("NL") @@ -391,6 +395,7 @@ class KeyFileDownloaderTest : BaseIOTest() { @Test fun `last3Hours fetch without prior data`() { + every { CWADebug.isDebugBuildOrMode } returns true every { settings.isLast3HourModeEnabled } returns true val downloader = createDownloader() @@ -430,6 +435,7 @@ class KeyFileDownloaderTest : BaseIOTest() { @Test fun `last3Hours fetch with prior data`() { + every { CWADebug.isDebugBuildOrMode } returns true every { settings.isLast3HourModeEnabled } returns true mockAddData( @@ -468,6 +474,7 @@ class KeyFileDownloaderTest : BaseIOTest() { @Test fun `last3Hours fetch deletes stale data`() { + every { CWADebug.isDebugBuildOrMode } returns true every { settings.isLast3HourModeEnabled } returns true val (staleKey1, _) = mockAddData( @@ -523,6 +530,7 @@ class KeyFileDownloaderTest : BaseIOTest() { @Test fun `last3Hours fetch skips single download failures`() { + every { CWADebug.isDebugBuildOrMode } returns true every { settings.isLast3HourModeEnabled } returns true var dlCounter = 0 From 11bf27070d8ccbea115f7158455f581c88d7d359 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Thu, 10 Sep 2020 09:58:13 +0200 Subject: [PATCH 14/29] Until we have more information about the hashsum's format in the header, default to `ETag --- .../diagnosiskeys/download/KeyFileDownloader.kt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 0e5c78badeb..dc5bb7e2962 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -266,13 +266,14 @@ class KeyFileDownloader @Inject constructor( private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, saveTo: File) { val validation = object : DownloadServer.HeaderValidation { override suspend fun validate(headers: Headers): Boolean { - var fileMD5 = headers.values("cwa-hash-md5").singleOrNull() - if (fileMD5 == null) { - headers.values("cwa-hash").singleOrNull() - } - if (fileMD5 == null) { // Fallback - fileMD5 = headers.values("ETag").singleOrNull() - } + var fileMD5 = headers.values("ETag").singleOrNull() +// var fileMD5 = headers.values("x-amz-meta-cwa-hash-md5").singleOrNull() +// if (fileMD5 == null) { +// headers.values("x-amz-meta-cwa-hash").singleOrNull() +// } +// if (fileMD5 == null) { // Fallback +// fileMD5 = headers.values("ETag").singleOrNull() +// } fileMD5 = fileMD5?.removePrefix("\"")?.removeSuffix("\"") return !legacyKeyCache.tryMigration(fileMD5, saveTo) From 76bc35b12a264c91376037cdf43184c2d07b8739 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Thu, 10 Sep 2020 11:24:27 +0200 Subject: [PATCH 15/29] Split app config server API from diagnosis key download API, and reintroduce caching for the app config download. --- .../diagnosiskeys/DiagnosisKeysModule.kt | 31 +++- .../download/KeyFileDownloader.kt | 19 +-- .../diagnosiskeys/server/AppConfigApiV1.kt | 11 ++ .../diagnosiskeys/server/AppConfigServer.kt | 61 +++++++ ...{DownloadApiV1.kt => DiagnosisKeyApiV1.kt} | 6 +- ...ownloadServer.kt => DiagnosisKeyServer.kt} | 58 ++----- .../ApplicationConfigurationService.kt | 2 +- .../util/di/ApplicationComponent.kt | 4 +- .../download/KeyFileDownloaderTest.kt | 65 +++++--- .../diagnosiskeys/server/AppConfigApiTest.kt | 128 +++++++++++++++ ...adServerTest.kt => AppConfigServerTest.kt} | 113 +------------ ...nloadAPITest.kt => DiagnosisKeyApiTest.kt} | 22 +-- .../server/DiagnosisKeyServerTest.kt | 150 ++++++++++++++++++ .../ApplicationConfigurationServiceTest.kt | 6 +- 14 files changed, 456 insertions(+), 220 deletions(-) create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigServer.kt rename Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/{DownloadApiV1.kt => DiagnosisKeyApiV1.kt} (87%) rename Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/{DownloadServer.kt => DiagnosisKeyServer.kt} (56%) create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt rename Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/{DownloadServerTest.kt => AppConfigServerTest.kt} (61%) rename Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/{DownloadAPITest.kt => DiagnosisKeyApiTest.kt} (86%) create mode 100644 Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServerTest.kt diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt index 9e415bbc99a..88be7abab62 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/DiagnosisKeysModule.kt @@ -5,7 +5,8 @@ import dagger.Module import dagger.Provides import dagger.Reusable import de.rki.coronawarnapp.BuildConfig -import de.rki.coronawarnapp.diagnosiskeys.server.DownloadApiV1 +import de.rki.coronawarnapp.diagnosiskeys.server.AppConfigApiV1 +import de.rki.coronawarnapp.diagnosiskeys.server.DiagnosisKeyApiV1 import de.rki.coronawarnapp.diagnosiskeys.server.DownloadHomeCountry import de.rki.coronawarnapp.diagnosiskeys.server.DownloadHttpClient import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServerUrl @@ -13,11 +14,13 @@ import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.KeyCacheLegacyDao import de.rki.coronawarnapp.http.HttpClientDefault import de.rki.coronawarnapp.storage.AppDatabase +import okhttp3.Cache import okhttp3.ConnectionSpec import okhttp3.OkHttpClient import okhttp3.TlsVersion import retrofit2.Retrofit import retrofit2.converter.gson.GsonConverterFactory +import java.io.File import javax.inject.Singleton @Module @@ -36,16 +39,36 @@ class DiagnosisKeysModule { @Singleton @Provides - fun provideDownloadApi( + fun provideDiagnosisKeyApi( @DownloadHttpClient client: OkHttpClient, @DownloadServerUrl url: String, gsonConverterFactory: GsonConverterFactory - ): DownloadApiV1 = Retrofit.Builder() + ): DiagnosisKeyApiV1 = Retrofit.Builder() .client(client) .baseUrl(url) .addConverterFactory(gsonConverterFactory) .build() - .create(DownloadApiV1::class.java) + .create(DiagnosisKeyApiV1::class.java) + + @Singleton + @Provides + fun provideAppConfigApi( + context: Context, + @DownloadHttpClient client: OkHttpClient, + @DownloadServerUrl url: String, + gsonConverterFactory: GsonConverterFactory + ): AppConfigApiV1 { + val cacheSize = 1 * 1024 * 1024L // 1MB + val cacheDir = File(context.cacheDir, "http_app-config") + val cache = Cache(cacheDir, cacheSize) + val cachingClient = client.newBuilder().cache(cache).build() + return Retrofit.Builder() + .client(cachingClient) + .baseUrl(url) + .addConverterFactory(gsonConverterFactory) + .build() + .create(AppConfigApiV1::class.java) + } @Singleton @DownloadServerUrl diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index dc5bb7e2962..7b4d88cc53a 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -1,7 +1,7 @@ package de.rki.coronawarnapp.diagnosiskeys.download import dagger.Reusable -import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer +import de.rki.coronawarnapp.diagnosiskeys.server.DiagnosisKeyServer import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository @@ -29,7 +29,7 @@ import javax.inject.Inject @Reusable class KeyFileDownloader @Inject constructor( private val deviceStorage: DeviceStorage, - private val downloadServer: DownloadServer, + private val diagnosisKeyServer: DiagnosisKeyServer, private val keyCache: KeyCacheRepository, private val legacyKeyCache: LegacyKeyCacheMigration, private val settings: AppSettings @@ -51,7 +51,7 @@ class KeyFileDownloader @Inject constructor( currentDate: LocalDate, wantedCountries: List ): List = withContext(Dispatchers.IO) { - val availableCountries = downloadServer.getCountryIndex() + val availableCountries = diagnosisKeyServer.getCountryIndex() val intersection = availableCountries.filter { wantedCountries.contains(it) } Timber.tag(TAG).v( "Available=%s; Wanted=%s; Intersect=%s", @@ -98,7 +98,7 @@ class KeyFileDownloader @Inject constructor( availableCountries: List ) = withContext(Dispatchers.IO) { val availableCountriesWithDays = availableCountries.map { - val days = downloadServer.getDayIndex(it) + val days = diagnosisKeyServer.getDayIndex(it) CountryDays(it, days) } @@ -190,9 +190,10 @@ class KeyFileDownloader @Inject constructor( // This is currently used for debugging, so we only fetch 3 hours val availableHours = availableCountries.map { - val hoursForDate = downloadServer.getHourIndex(it, currentDate).filter { availHour -> - TimeAndDateExtensions.getCurrentHourUTC() - 3 <= availHour.hourOfDay - } + val hoursForDate = + diagnosisKeyServer.getHourIndex(it, currentDate).filter { availHour -> + TimeAndDateExtensions.getCurrentHourUTC() - 3 <= availHour.hourOfDay + } CountryHours(it, mapOf(currentDate to hoursForDate)) } @@ -264,7 +265,7 @@ class KeyFileDownloader @Inject constructor( } private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, saveTo: File) { - val validation = object : DownloadServer.HeaderValidation { + val validation = object : DiagnosisKeyServer.HeaderValidation { override suspend fun validate(headers: Headers): Boolean { var fileMD5 = headers.values("ETag").singleOrNull() // var fileMD5 = headers.values("x-amz-meta-cwa-hash-md5").singleOrNull() @@ -280,7 +281,7 @@ class KeyFileDownloader @Inject constructor( } } - downloadServer.downloadKeyFile( + diagnosisKeyServer.downloadKeyFile( keyInfo.location, keyInfo.day, keyInfo.hour, diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt new file mode 100644 index 00000000000..0816aa33800 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt @@ -0,0 +1,11 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import okhttp3.ResponseBody +import retrofit2.http.GET +import retrofit2.http.Path + +interface AppConfigApiV1 { + + @GET("/version/v1/configuration/country/{country}/app_config") + suspend fun getApplicationConfiguration(@Path("country") country: String): ResponseBody +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigServer.kt new file mode 100644 index 00000000000..f5bee9979a8 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigServer.kt @@ -0,0 +1,61 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import com.google.protobuf.InvalidProtocolBufferException +import dagger.Lazy +import de.rki.coronawarnapp.exception.ApplicationConfigurationCorruptException +import de.rki.coronawarnapp.exception.ApplicationConfigurationInvalidException +import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass +import de.rki.coronawarnapp.util.ZipHelper.unzip +import de.rki.coronawarnapp.util.security.VerificationKeys +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import timber.log.Timber +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class AppConfigServer @Inject constructor( + private val appConfigAPI: Lazy, + private val verificationKeys: VerificationKeys, + @DownloadHomeCountry private val homeCountry: LocationCode +) { + + private val configApi: AppConfigApiV1 + get() = appConfigAPI.get() + + suspend fun downloadAppConfig(): ApplicationConfigurationOuterClass.ApplicationConfiguration = + withContext(Dispatchers.IO) { + Timber.tag(TAG).d("Fetching app config.") + var exportBinary: ByteArray? = null + var exportSignature: ByteArray? = null + configApi.getApplicationConfiguration(homeCountry.identifier).byteStream() + .unzip { entry, entryContent -> + if (entry.name == EXPORT_BINARY_FILE_NAME) exportBinary = + entryContent.copyOf() + if (entry.name == EXPORT_SIGNATURE_FILE_NAME) exportSignature = + entryContent.copyOf() + } + if (exportBinary == null || exportSignature == null) { + throw ApplicationConfigurationInvalidException() + } + + if (verificationKeys.hasInvalidSignature(exportBinary, exportSignature)) { + throw ApplicationConfigurationCorruptException() + } + + try { + return@withContext ApplicationConfigurationOuterClass.ApplicationConfiguration.parseFrom( + exportBinary + ) + } catch (e: InvalidProtocolBufferException) { + throw ApplicationConfigurationInvalidException() + } + } + + companion object { + private val TAG = AppConfigServer::class.java.simpleName + + private const val EXPORT_BINARY_FILE_NAME = "export.bin" + private const val EXPORT_SIGNATURE_FILE_NAME = "export.sig" + } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt similarity index 87% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt rename to Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt index d7845770842..6c35fafc022 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadApiV1.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt @@ -6,11 +6,7 @@ import retrofit2.http.GET import retrofit2.http.Path import retrofit2.http.Streaming -interface DownloadApiV1 { - - @GET("/version/v1/configuration/country/{country}/app_config") - suspend fun getApplicationConfiguration(@Path("country") country: String): ResponseBody - +interface DiagnosisKeyApiV1 { // TODO Let retrofit format this to CountryCode @GET("/version/v1/diagnosis-keys/country") suspend fun getCountryIndex(): List diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServer.kt similarity index 56% rename from Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt rename to Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServer.kt index 09aba55e3ae..783cd2556cd 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServer.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServer.kt @@ -1,12 +1,6 @@ package de.rki.coronawarnapp.diagnosiskeys.server -import com.google.protobuf.InvalidProtocolBufferException import dagger.Lazy -import de.rki.coronawarnapp.exception.ApplicationConfigurationCorruptException -import de.rki.coronawarnapp.exception.ApplicationConfigurationInvalidException -import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass -import de.rki.coronawarnapp.util.ZipHelper.unzip -import de.rki.coronawarnapp.util.security.VerificationKeys import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import okhttp3.Headers @@ -19,51 +13,22 @@ import javax.inject.Inject import javax.inject.Singleton @Singleton -class DownloadServer @Inject constructor( - private val downloadAPI: Lazy, - private val verificationKeys: VerificationKeys, +class DiagnosisKeyServer @Inject constructor( + private val diagnosisKeyAPI: Lazy, @DownloadHomeCountry private val homeCountry: LocationCode ) { - private val api: DownloadApiV1 - get() = downloadAPI.get() - - suspend fun downloadAppConfig(): ApplicationConfigurationOuterClass.ApplicationConfiguration = - withContext(Dispatchers.IO) { - var exportBinary: ByteArray? = null - var exportSignature: ByteArray? = null - api.getApplicationConfiguration(homeCountry.identifier).byteStream() - .unzip { entry, entryContent -> - if (entry.name == EXPORT_BINARY_FILE_NAME) exportBinary = - entryContent.copyOf() - if (entry.name == EXPORT_SIGNATURE_FILE_NAME) exportSignature = - entryContent.copyOf() - } - if (exportBinary == null || exportSignature == null) { - throw ApplicationConfigurationInvalidException() - } - - if (verificationKeys.hasInvalidSignature(exportBinary, exportSignature)) { - throw ApplicationConfigurationCorruptException() - } - - try { - return@withContext ApplicationConfigurationOuterClass.ApplicationConfiguration.parseFrom( - exportBinary - ) - } catch (e: InvalidProtocolBufferException) { - throw ApplicationConfigurationInvalidException() - } - } + private val keyApi: DiagnosisKeyApiV1 + get() = diagnosisKeyAPI.get() suspend fun getCountryIndex(): List = withContext(Dispatchers.IO) { - api + keyApi .getCountryIndex() .map { LocationCode(it) } } suspend fun getDayIndex(location: LocationCode): List = withContext(Dispatchers.IO) { - api + keyApi .getDayIndex(location.identifier) .map { dayString -> // 2020-08-19 @@ -73,7 +38,7 @@ class DownloadServer @Inject constructor( suspend fun getHourIndex(location: LocationCode, day: LocalDate): List = withContext(Dispatchers.IO) { - api + keyApi .getHourIndex(location.identifier, day.toString(DAY_FORMATTER)) .map { hourString -> LocalTime.parse(hourString, HOUR_FORMATTER) } } @@ -104,13 +69,13 @@ class DownloadServer @Inject constructor( } val response = if (hour != null) { - api.downloadKeyFileForHour( + keyApi.downloadKeyFileForHour( locationCode.identifier, day.toString(DAY_FORMATTER), hour.toString(HOUR_FORMATTER) ) } else { - api.downloadKeyFileForDay( + keyApi.downloadKeyFileForDay( locationCode.identifier, day.toString(DAY_FORMATTER) ) @@ -128,11 +93,8 @@ class DownloadServer @Inject constructor( } companion object { - private val TAG = DownloadServer::class.java.simpleName + private val TAG = DiagnosisKeyServer::class.java.simpleName private val DAY_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd") private val HOUR_FORMATTER = DateTimeFormat.forPattern("HH") - - private const val EXPORT_BINARY_FILE_NAME = "export.bin" - private const val EXPORT_SIGNATURE_FILE_NAME = "export.sig" } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt index 007e491d1bf..e9e3a09a8f8 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationService.kt @@ -7,7 +7,7 @@ import de.rki.coronawarnapp.util.di.AppInjector object ApplicationConfigurationService { suspend fun asyncRetrieveApplicationConfiguration(): ApplicationConfiguration { - return AppInjector.component.downloadServer.downloadAppConfig().let { + return AppInjector.component.appConfigServer.downloadAppConfig().let { if (CWADebug.isDebugBuildOrMode) { // TODO: THIS IS A MOCK -> Remove after Backend is providing this information. it.toBuilder() diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt index 79506c91d57..2e6290343c5 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/di/ApplicationComponent.kt @@ -7,7 +7,7 @@ import dagger.android.support.AndroidSupportInjectionModule import de.rki.coronawarnapp.CoronaWarnApplication import de.rki.coronawarnapp.diagnosiskeys.DiagnosisKeysModule import de.rki.coronawarnapp.diagnosiskeys.download.KeyFileDownloader -import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer +import de.rki.coronawarnapp.diagnosiskeys.server.AppConfigServer import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.http.HttpModule import de.rki.coronawarnapp.http.ServiceFactory @@ -54,7 +54,7 @@ interface ApplicationComponent : AndroidInjector { val keyFileDownloader: KeyFileDownloader val serviceFactory: ServiceFactory - val downloadServer: DownloadServer + val appConfigServer: AppConfigServer @Component.Factory interface Factory { diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt index bf42eb34856..51e78d24ec1 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -1,7 +1,7 @@ package de.rki.coronawarnapp.diagnosiskeys.download import android.database.SQLException -import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer +import de.rki.coronawarnapp.diagnosiskeys.server.DiagnosisKeyServer import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository @@ -43,7 +43,7 @@ class KeyFileDownloaderTest : BaseIOTest() { private lateinit var legacyMigration: LegacyKeyCacheMigration @MockK - private lateinit var downloadServer: DownloadServer + private lateinit var diagnosisKeyServer: DiagnosisKeyServer @MockK private lateinit var deviceStorage: DeviceStorage @@ -62,7 +62,7 @@ class KeyFileDownloaderTest : BaseIOTest() { mockkObject(CWADebug) - coEvery { downloadServer.getCountryIndex() } returns listOf( + coEvery { diagnosisKeyServer.getCountryIndex() } returns listOf( LocationCode("DE"), LocationCode("NL") ) @@ -73,36 +73,36 @@ class KeyFileDownloaderTest : BaseIOTest() { coEvery { settings.isLast3HourModeEnabled } returns false - coEvery { downloadServer.getCountryIndex() } returns listOf( + coEvery { diagnosisKeyServer.getCountryIndex() } returns listOf( LocationCode("DE"), LocationCode("NL") ) - coEvery { downloadServer.getDayIndex(LocationCode("DE")) } returns listOf( + coEvery { diagnosisKeyServer.getDayIndex(LocationCode("DE")) } returns listOf( LocalDate.parse("2020-09-01"), LocalDate.parse("2020-09-02") ) coEvery { - downloadServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-01")) + diagnosisKeyServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-01")) } returns listOf( LocalTime.parse("20") ) coEvery { - downloadServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-02")) + diagnosisKeyServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-02")) } returns listOf( LocalTime.parse("20"), LocalTime.parse("21") ) - coEvery { downloadServer.getDayIndex(LocationCode("NL")) } returns listOf( + coEvery { diagnosisKeyServer.getDayIndex(LocationCode("NL")) } returns listOf( LocalDate.parse("2020-09-02"), LocalDate.parse("2020-09-03") ) coEvery { - downloadServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-02")) + diagnosisKeyServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-02")) } returns listOf( LocalTime.parse("22") ) coEvery { - downloadServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-03")) + diagnosisKeyServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-03")) } returns listOf( LocalTime.parse("22"), LocalTime.parse("23") ) - coEvery { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { + coEvery { diagnosisKeyServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { mockDownloadServerDownload(arg(0), arg(1), arg(2), arg(3), arg(4)) } @@ -169,7 +169,8 @@ class KeyFileDownloaderTest : BaseIOTest() { day: LocalDate, hour: LocalTime? = null, saveTo: File, - validator: DownloadServer.HeaderValidation = object : DownloadServer.HeaderValidation {} + validator: DiagnosisKeyServer.HeaderValidation = object : + DiagnosisKeyServer.HeaderValidation {} ) { saveTo.writeText("$locationCode.$day.$hour") } @@ -192,7 +193,7 @@ class KeyFileDownloaderTest : BaseIOTest() { private fun createDownloader(): KeyFileDownloader { val downloader = KeyFileDownloader( deviceStorage = deviceStorage, - downloadServer = downloadServer, + diagnosisKeyServer = diagnosisKeyServer, keyCache = keyCache, legacyKeyCache = legacyMigration, settings = settings @@ -227,7 +228,7 @@ class KeyFileDownloaderTest : BaseIOTest() { @Test fun `error during country index fetch`() { - coEvery { downloadServer.getCountryIndex() } throws IOException() + coEvery { diagnosisKeyServer.getCountryIndex() } throws IOException() val downloader = createDownloader() @@ -327,7 +328,7 @@ class KeyFileDownloaderTest : BaseIOTest() { @Test fun `day fetch deletes stale data`() { - coEvery { downloadServer.getDayIndex(LocationCode("DE")) } returns listOf( + coEvery { diagnosisKeyServer.getDayIndex(LocationCode("DE")) } returns listOf( LocalDate.parse("2020-09-02") ) val (staleKeyInfo, _) = mockAddData( @@ -377,7 +378,7 @@ class KeyFileDownloaderTest : BaseIOTest() { @Test fun `day fetch skips single download failures`() { var dlCounter = 0 - coEvery { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { + coEvery { diagnosisKeyServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { dlCounter++ if (dlCounter == 2) throw IOException("Timeout") mockDownloadServerDownload(arg(0), arg(1), arg(2), arg(3), arg(4)) @@ -469,7 +470,15 @@ class KeyFileDownloaderTest : BaseIOTest() { hourIdentifier = LocalTime.parse("21") ) } - coVerify(exactly = 2) { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } + coVerify(exactly = 2) { + diagnosisKeyServer.downloadKeyFile( + any(), + any(), + any(), + any(), + any() + ) + } } @Test @@ -524,7 +533,15 @@ class KeyFileDownloaderTest : BaseIOTest() { hourIdentifier = LocalTime.parse("21") ) } - coVerify(exactly = 2) { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } + coVerify(exactly = 2) { + diagnosisKeyServer.downloadKeyFile( + any(), + any(), + any(), + any(), + any() + ) + } coVerify(exactly = 1) { keyCache.delete(listOf(staleKey1, staleKey2)) } } @@ -534,7 +551,7 @@ class KeyFileDownloaderTest : BaseIOTest() { every { settings.isLast3HourModeEnabled } returns true var dlCounter = 0 - coEvery { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { + coEvery { diagnosisKeyServer.downloadKeyFile(any(), any(), any(), any(), any()) } answers { dlCounter++ if (dlCounter == 2) throw IOException("Timeout") mockDownloadServerDownload(arg(0), arg(1), arg(2), arg(3), arg(4)) @@ -597,6 +614,14 @@ class KeyFileDownloaderTest : BaseIOTest() { ).size shouldBe 3 } - coVerify(exactly = 4) { downloadServer.downloadKeyFile(any(), any(), any(), any(), any()) } + coVerify(exactly = 4) { + diagnosisKeyServer.downloadKeyFile( + any(), + any(), + any(), + any(), + any() + ) + } } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt new file mode 100644 index 00000000000..26773fa7927 --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt @@ -0,0 +1,128 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import android.content.Context +import de.rki.coronawarnapp.diagnosiskeys.DiagnosisKeysModule +import de.rki.coronawarnapp.http.HttpModule +import io.kotest.matchers.shouldBe +import io.mockk.MockKAnnotations +import io.mockk.clearAllMocks +import io.mockk.every +import io.mockk.impl.annotations.MockK +import kotlinx.coroutines.runBlocking +import okhttp3.ConnectionSpec +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import testhelpers.BaseIOTest +import java.io.File +import java.util.concurrent.TimeUnit + +class AppConfigApiTest : BaseIOTest() { + + @MockK + private lateinit var context: Context + + private lateinit var webServer: MockWebServer + private lateinit var serverAddress: String + + private val testDir = File(IO_TEST_BASEDIR, this::class.java.simpleName) + private val appConfigCacheDir = File(testDir, "http_app-config") + + @BeforeEach + fun setup() { + MockKAnnotations.init(this) + every { context.cacheDir } returns testDir + + webServer = MockWebServer() + webServer.start() + serverAddress = "http://${webServer.hostName}:${webServer.port}" + } + + @AfterEach + fun teardown() { + clearAllMocks() + webServer.shutdown() + testDir.deleteRecursively() + } + + private fun createAPI(): AppConfigApiV1 { + val httpModule = HttpModule() + val defaultHttpClient = httpModule.defaultHttpClient() + val gsonConverterFactory = httpModule.provideGSONConverter() + + return DiagnosisKeysModule().let { + val downloadHttpClient = it.cdnHttpClient(defaultHttpClient) + .newBuilder() + .connectionSpecs(listOf(ConnectionSpec.CLEARTEXT, ConnectionSpec.MODERN_TLS)) + .build() + it.provideAppConfigApi( + context = context, + client = downloadHttpClient, + url = serverAddress, + gsonConverterFactory = gsonConverterFactory + ) + } + } + + @Test + fun `application config download`() { + val api = createAPI() + + webServer.enqueue(MockResponse().setBody("~appconfig")) + + runBlocking { + api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" + } + + val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! + request.method shouldBe "GET" + request.path shouldBe "/version/v1/configuration/country/DE/app_config" + } + + @Test + fun `application config download uses cache`() { + appConfigCacheDir.exists() shouldBe false + + val api = createAPI() + + val configResponse = + MockResponse().setBody("~appconfig").addHeader("Cache-Control: max-age=2") + + webServer.enqueue(configResponse) + runBlocking { + api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" + } + appConfigCacheDir.exists() shouldBe true + appConfigCacheDir.listFiles()!!.size shouldBe 3 + + webServer.takeRequest(5, TimeUnit.SECONDS)!!.apply { + method shouldBe "GET" + path shouldBe "/version/v1/configuration/country/DE/app_config" + } + + webServer.enqueue(configResponse) + runBlocking { + api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" + } + appConfigCacheDir.exists() shouldBe true + appConfigCacheDir.listFiles()!!.size shouldBe 3 + + webServer.takeRequest(2, TimeUnit.SECONDS) shouldBe null + + Thread.sleep(4000) // Let the cache expire + + webServer.enqueue(configResponse) + runBlocking { + api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" + } + appConfigCacheDir.exists() shouldBe true + appConfigCacheDir.listFiles()!!.size shouldBe 3 + + webServer.takeRequest(5, TimeUnit.SECONDS)!!.apply { + method shouldBe "GET" + path shouldBe "/version/v1/configuration/country/DE/app_config" + } + } +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigServerTest.kt similarity index 61% rename from Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt rename to Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigServerTest.kt index 98167f5af5c..0fbd9e4c8f8 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadServerTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigServerTest.kt @@ -9,26 +9,22 @@ import io.kotest.matchers.shouldBe import io.mockk.MockKAnnotations import io.mockk.clearAllMocks import io.mockk.coEvery -import io.mockk.coVerify import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.verify import kotlinx.coroutines.runBlocking import okhttp3.ResponseBody.Companion.toResponseBody import okio.ByteString.Companion.decodeHex -import org.joda.time.LocalDate -import org.joda.time.LocalTime import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import retrofit2.Response import testhelpers.BaseIOTest import java.io.File -class DownloadServerTest : BaseIOTest() { +class AppConfigServerTest : BaseIOTest() { @MockK - lateinit var api: DownloadApiV1 + lateinit var api: AppConfigApiV1 @MockK lateinit var verificationKeys: VerificationKeys @@ -52,8 +48,8 @@ class DownloadServerTest : BaseIOTest() { private fun createDownloadServer( homeCountry: LocationCode = defaultHomeCountry - ) = DownloadServer( - downloadAPI = Lazy { api }, + ) = AppConfigServer( + appConfigAPI = Lazy { api }, verificationKeys = verificationKeys, homeCountry = homeCountry ) @@ -114,107 +110,6 @@ class DownloadServerTest : BaseIOTest() { } } - @Test - fun `download country index`() { - val downloadServer = createDownloadServer() - coEvery { api.getCountryIndex() } returns listOf("DE", "NL") - - runBlocking { - downloadServer.getCountryIndex() shouldBe listOf( - LocationCode("DE"), LocationCode("NL") - ) - } - - coVerify { api.getCountryIndex() } - } - - @Test - fun `download day index for country`() { - val downloadServer = createDownloadServer() - coEvery { api.getDayIndex("DE") } returns listOf( - "2000-01-01", "2000-01-02" - ) - - runBlocking { - downloadServer.getDayIndex(LocationCode("DE")) shouldBe listOf( - "2000-01-01", "2000-01-02" - ).map { LocalDate.parse(it) } - } - - coVerify { api.getDayIndex("DE") } - } - - @Test - fun `download hour index for country and day`() { - val downloadServer = createDownloadServer() - coEvery { api.getHourIndex("DE", "2000-01-01") } returns listOf( - "20", "21" - ) - - runBlocking { - downloadServer.getHourIndex( - LocationCode("DE"), - LocalDate.parse("2000-01-01") - ) shouldBe listOf( - "20:00", "21:00" - ).map { LocalTime.parse(it) } - } - - coVerify { api.getHourIndex("DE", "2000-01-01") } - } - - - @Test - fun `download key files for day`() { - val downloadServer = createDownloadServer() - coEvery { - api.downloadKeyFileForDay( - "DE", - "2000-01-01" - ) - } returns Response.success("testdata-day".toResponseBody()) - - val targetFile = File(testDir, "day-keys") - - runBlocking { - downloadServer.downloadKeyFile( - locationCode = LocationCode("DE"), - day = LocalDate.parse("2000-01-01"), - hour = null, - saveTo = targetFile - ) - } - - targetFile.exists() shouldBe true - targetFile.readText() shouldBe "testdata-day" - } - - @Test - fun `download key files for hour`() { - val downloadServer = createDownloadServer() - coEvery { - api.downloadKeyFileForHour( - "DE", - "2000-01-01", - "01" - ) - } returns Response.success("testdata-hour".toResponseBody()) - - val targetFile = File(testDir, "hour-keys") - - runBlocking { - downloadServer.downloadKeyFile( - locationCode = LocationCode("DE"), - day = LocalDate.parse("2000-01-01"), - hour = LocalTime.parse("01:00"), - saveTo = targetFile - ) - } - - targetFile.exists() shouldBe true - targetFile.readText() shouldBe "testdata-hour" - } - companion object { private const val APPCONFIG_HEX = "504b0304140008080800856b22510000000000000000000000000a0000006578706f72742e62696ee3e016" + diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiTest.kt similarity index 86% rename from Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt rename to Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiTest.kt index 16f57bdf40e..2c6c522ed40 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DownloadAPITest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiTest.kt @@ -13,7 +13,7 @@ import org.junit.jupiter.api.Test import testhelpers.BaseIOTest import java.util.concurrent.TimeUnit -class DownloadAPITest : BaseIOTest() { +class DiagnosisKeyApiTest : BaseIOTest() { lateinit var webServer: MockWebServer lateinit var serverAddress: String @@ -30,7 +30,7 @@ class DownloadAPITest : BaseIOTest() { webServer.shutdown() } - private fun createAPI(): DownloadApiV1 { + private fun createAPI(): DiagnosisKeyApiV1 { val httpModule = HttpModule() val defaultHttpClient = httpModule.defaultHttpClient() val gsonConverterFactory = httpModule.provideGSONConverter() @@ -40,30 +40,14 @@ class DownloadAPITest : BaseIOTest() { .newBuilder() .connectionSpecs(listOf(ConnectionSpec.CLEARTEXT, ConnectionSpec.MODERN_TLS)) .build() - it.provideDownloadApi( + it.provideDiagnosisKeyApi( client = downloadHttpClient, url = serverAddress, gsonConverterFactory = gsonConverterFactory - ) } } - @Test - fun `application config download`() { - val api = createAPI() - - webServer.enqueue(MockResponse().setBody("~appconfig")) - - runBlocking { - api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" - } - - val request = webServer.takeRequest(5, TimeUnit.SECONDS)!! - request.method shouldBe "GET" - request.path shouldBe "/version/v1/configuration/country/DE/app_config" - } - @Test fun `download country index`() { val api = createAPI() diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServerTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServerTest.kt new file mode 100644 index 00000000000..16edc6a932c --- /dev/null +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServerTest.kt @@ -0,0 +1,150 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import dagger.Lazy +import io.kotest.matchers.shouldBe +import io.mockk.MockKAnnotations +import io.mockk.clearAllMocks +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.impl.annotations.MockK +import kotlinx.coroutines.runBlocking +import okhttp3.ResponseBody.Companion.toResponseBody +import org.joda.time.LocalDate +import org.joda.time.LocalTime +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import retrofit2.Response +import testhelpers.BaseIOTest +import java.io.File + +class DiagnosisKeyServerTest : BaseIOTest() { + + @MockK + lateinit var api: DiagnosisKeyApiV1 + + private val testDir = File(IO_TEST_BASEDIR, this::class.simpleName!!) + + private val defaultHomeCountry = LocationCode("DE") + + @BeforeEach + fun setup() { + MockKAnnotations.init(this) + testDir.mkdirs() + testDir.exists() shouldBe true + } + + @AfterEach + fun teardown() { + clearAllMocks() + testDir.deleteRecursively() + } + + private fun createDownloadServer( + homeCountry: LocationCode = defaultHomeCountry + ) = DiagnosisKeyServer( + diagnosisKeyAPI = Lazy { api }, + homeCountry = homeCountry + ) + + @Test + fun `download country index`() { + val downloadServer = createDownloadServer() + coEvery { api.getCountryIndex() } returns listOf("DE", "NL") + + runBlocking { + downloadServer.getCountryIndex() shouldBe listOf( + LocationCode("DE"), LocationCode("NL") + ) + } + + coVerify { api.getCountryIndex() } + } + + @Test + fun `download day index for country`() { + val downloadServer = createDownloadServer() + coEvery { api.getDayIndex("DE") } returns listOf( + "2000-01-01", "2000-01-02" + ) + + runBlocking { + downloadServer.getDayIndex(LocationCode("DE")) shouldBe listOf( + "2000-01-01", "2000-01-02" + ).map { LocalDate.parse(it) } + } + + coVerify { api.getDayIndex("DE") } + } + + @Test + fun `download hour index for country and day`() { + val downloadServer = createDownloadServer() + coEvery { api.getHourIndex("DE", "2000-01-01") } returns listOf( + "20", "21" + ) + + runBlocking { + downloadServer.getHourIndex( + LocationCode("DE"), + LocalDate.parse("2000-01-01") + ) shouldBe listOf( + "20:00", "21:00" + ).map { LocalTime.parse(it) } + } + + coVerify { api.getHourIndex("DE", "2000-01-01") } + } + + + @Test + fun `download key files for day`() { + val downloadServer = createDownloadServer() + coEvery { + api.downloadKeyFileForDay( + "DE", + "2000-01-01" + ) + } returns Response.success("testdata-day".toResponseBody()) + + val targetFile = File(testDir, "day-keys") + + runBlocking { + downloadServer.downloadKeyFile( + locationCode = LocationCode("DE"), + day = LocalDate.parse("2000-01-01"), + hour = null, + saveTo = targetFile + ) + } + + targetFile.exists() shouldBe true + targetFile.readText() shouldBe "testdata-day" + } + + @Test + fun `download key files for hour`() { + val downloadServer = createDownloadServer() + coEvery { + api.downloadKeyFileForHour( + "DE", + "2000-01-01", + "01" + ) + } returns Response.success("testdata-hour".toResponseBody()) + + val targetFile = File(testDir, "hour-keys") + + runBlocking { + downloadServer.downloadKeyFile( + locationCode = LocationCode("DE"), + day = LocalDate.parse("2000-01-01"), + hour = LocalTime.parse("01:00"), + saveTo = targetFile + ) + } + + targetFile.exists() shouldBe true + targetFile.readText() shouldBe "testdata-hour" + } +} diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt index aef089e4371..06f4368a048 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/service/applicationconfiguration/ApplicationConfigurationServiceTest.kt @@ -1,6 +1,6 @@ package de.rki.coronawarnapp.service.applicationconfiguration -import de.rki.coronawarnapp.diagnosiskeys.server.DownloadServer +import de.rki.coronawarnapp.diagnosiskeys.server.AppConfigServer import de.rki.coronawarnapp.http.WebRequestBuilder import de.rki.coronawarnapp.server.protocols.ApplicationConfigurationOuterClass import de.rki.coronawarnapp.util.CWADebug @@ -38,12 +38,12 @@ class ApplicationConfigurationServiceTest : BaseTest() { every { appConfigBuilder.build() } returns appConfig - val downloadServer = mockk() + val downloadServer = mockk() coEvery { downloadServer.downloadAppConfig() } returns appConfig mockkObject(AppInjector) mockk().apply { - every { this@apply.downloadServer } returns downloadServer + every { this@apply.appConfigServer } returns downloadServer every { AppInjector.component } returns this@apply } From 6e96a60b65dbc1a839a114668025a2dc4f8f798c Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Thu, 10 Sep 2020 12:35:23 +0200 Subject: [PATCH 16/29] Add test to check that the cache is used on flaky connections. --- .../diagnosiskeys/server/AppConfigApiTest.kt | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt index 26773fa7927..4db01f22b51 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiTest.kt @@ -12,6 +12,7 @@ import kotlinx.coroutines.runBlocking import okhttp3.ConnectionSpec import okhttp3.mockwebserver.MockResponse import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.SocketPolicy import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -125,4 +126,32 @@ class AppConfigApiTest : BaseIOTest() { path shouldBe "/version/v1/configuration/country/DE/app_config" } } + + @Test + fun `cache is used when connection is flaky`() { + appConfigCacheDir.exists() shouldBe false + + val api = createAPI() + + val configResponse = + MockResponse().setBody("~appconfig").addHeader("Cache-Control: max-age=300") + + webServer.enqueue(configResponse) + runBlocking { + api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" + } + appConfigCacheDir.exists() shouldBe true + appConfigCacheDir.listFiles()!!.size shouldBe 3 + + webServer.takeRequest(5, TimeUnit.SECONDS)!!.apply { + method shouldBe "GET" + path shouldBe "/version/v1/configuration/country/DE/app_config" + } + + webServer.enqueue(MockResponse().setSocketPolicy(SocketPolicy.DISCONNECT_DURING_REQUEST_BODY)) + + runBlocking { + api.getApplicationConfiguration("DE").string() shouldBe "~appconfig" + } + } } From 3143c090b532892a0d99446c364e6f882caa1229 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Thu, 10 Sep 2020 22:29:56 +0200 Subject: [PATCH 17/29] Code changes based on PR comment, part #1. --- .../download/KeyFileDownloader.kt | 154 +++++++++++------- .../coronawarnapp/storage/DeviceStorage.kt | 3 + .../storage/InsufficientStorageException.kt | 7 + .../RetrieveDiagnosisKeysTransaction.kt | 2 +- .../download/KeyFileDownloaderTest.kt | 4 +- .../storage/DeviceStorageTest.kt | 5 + 6 files changed, 114 insertions(+), 61 deletions(-) create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/InsufficientStorageException.kt diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 7b4d88cc53a..30d5e72a757 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -8,6 +8,7 @@ import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration import de.rki.coronawarnapp.storage.AppSettings import de.rki.coronawarnapp.storage.DeviceStorage +import de.rki.coronawarnapp.storage.InsufficientStorageException import de.rki.coronawarnapp.util.CWADebug import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 import de.rki.coronawarnapp.util.TimeAndDateExtensions @@ -20,7 +21,6 @@ import okhttp3.Headers import org.joda.time.LocalDate import timber.log.Timber import java.io.File -import java.io.IOException import javax.inject.Inject /** @@ -35,6 +35,30 @@ class KeyFileDownloader @Inject constructor( private val settings: AppSettings ) { + private suspend fun checkStorageSpace(countries: List): DeviceStorage.CheckResult { + val storageResult = deviceStorage.checkSpacePrivateStorage( + // 512KB per day file, for 15 days, for each country ~ 65MB for 9 countries + requiredBytes = countries.size * 15 * 512 * 1024L + ) + Timber.tag(TAG).d("Storage check result: %s", storageResult) + return storageResult + } + + private suspend fun getCompletedKeyFiles(type: CachedKeyInfo.Type): List { + return keyCache + .getEntriesForType(type) + .filter { (keyInfo, file) -> + val complete = keyInfo.isDownloadComplete + val exists = file.exists() + if (complete && !exists) { + Timber.tag(TAG).v("Incomplete download, will overwrite: %s", keyInfo) + } + // We overwrite not completed ones + complete && exists + } + .map { it.first } + } + /** * Fetches all necessary Files from the Cached KeyFile Entries out of the [KeyCacheRepository] and * adds to that all open Files currently available from the Server. @@ -52,27 +76,20 @@ class KeyFileDownloader @Inject constructor( wantedCountries: List ): List = withContext(Dispatchers.IO) { val availableCountries = diagnosisKeyServer.getCountryIndex() - val intersection = availableCountries.filter { wantedCountries.contains(it) } + val filteredCountries = availableCountries.filter { wantedCountries.contains(it) } Timber.tag(TAG).v( "Available=%s; Wanted=%s; Intersect=%s", - availableCountries, wantedCountries, intersection + availableCountries, wantedCountries, filteredCountries ) - val storageResult = deviceStorage.checkSpacePrivateStorage( - // 512KB per day file, for 15 days, for each country ~ 65MB for 9 countries - requiredBytes = intersection.size * 15 * 512 * 1024L - ) - - Timber.tag(TAG).d("Storage check result: %s", storageResult) - if (!storageResult.isSpaceAvailable) { - throw IOException("Not enough free space (${storageResult.freeBytes}") - } + val storageResult = checkStorageSpace(filteredCountries) + if (!storageResult.isSpaceAvailable) throw InsufficientStorageException(storageResult) val availableKeys = if (CWADebug.isDebugBuildOrMode && settings.isLast3HourModeEnabled) { - syncMissing3Hours(currentDate, intersection) + syncMissing3Hours(currentDate, filteredCountries, DEBUG_HOUR_LIMIT) keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) } else { - syncMissingDays(intersection) + syncMissingDays(filteredCountries) keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) } @@ -89,27 +106,17 @@ class KeyFileDownloader @Inject constructor( .also { Timber.tag(TAG).d("Returning %d available keyfiles", it.size) } } - /** - * Fetches files given by serverDates by respecting countries - * @param availableCountries pair of dates per country code - */ - @Suppress("LongMethod") - private suspend fun syncMissingDays( - availableCountries: List - ) = withContext(Dispatchers.IO) { - val availableCountriesWithDays = availableCountries.map { + private suspend fun determineMissingDays(availableCountries: List): List { + val availableDays = availableCountries.map { val days = diagnosisKeyServer.getDayIndex(it) CountryDays(it, days) } - val cachedDays = keyCache - .getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) - .filter { it.first.isDownloadComplete && it.second.exists() } // We overwrite not completed ones - .map { it.first } + val cachedDays = getCompletedKeyFiles(CachedKeyInfo.Type.COUNTRY_DAY) // All cached files that are no longer on the server are considered stale - val staleKeyFiles = cachedDays.filter { cachedKeyFile -> - val availableCountry = availableCountriesWithDays.singleOrNull { + val staleDays = cachedDays.filter { cachedKeyFile -> + val availableCountry = availableDays.singleOrNull { it.country == cachedKeyFile.location } if (availableCountry == null) { @@ -124,12 +131,27 @@ class KeyFileDownloader @Inject constructor( cachedKeyFile.day == date } } - if (staleKeyFiles.isNotEmpty()) keyCache.delete(staleKeyFiles) - val nonStaleDays = cachedDays.minus(staleKeyFiles) - val countriesWithMissingDays = availableCountriesWithDays.mapNotNull { - it.toMissingDays(nonStaleDays) + if (staleDays.isNotEmpty()) { + Timber.tag(TAG).v("Deleting stale days: %s", staleDays) + keyCache.delete(staleDays) } + + val nonStaleDays = cachedDays.minus(staleDays) + + // The missing days + return availableDays.mapNotNull { it.toMissingDays(nonStaleDays) } + } + + /** + * Fetches files given by serverDates by respecting countries + * @param availableCountries pair of dates per country code + */ + private suspend fun syncMissingDays( + availableCountries: List + ) = withContext(Dispatchers.IO) { + val countriesWithMissingDays = determineMissingDays(availableCountries) + Timber.tag(TAG).d("Downloading missing days: %s", countriesWithMissingDays) val batchDownloadStart = System.currentTimeMillis() val dayDownloads = countriesWithMissingDays @@ -170,37 +192,29 @@ class KeyFileDownloader @Inject constructor( path } - Unit + return@withContext } - /** - * Fetches files given by serverDates by respecting countries - * @param currentDate base for where only dates within 3 hours before will be fetched - * @param availableCountries pair of dates per country code - */ - @Suppress("LongMethod") - private suspend fun syncMissing3Hours( + private suspend fun determineMissingHours( currentDate: LocalDate, - availableCountries: List - ) = withContext(Dispatchers.IO) { - Timber.tag(TAG).v( - "asyncHandleLast3HoursFilesFetch(currentDate=%s, availableCountries=%s)", - currentDate, availableCountries - ) - + availableCountries: List, + itemLimit: Int + ): List { // This is currently used for debugging, so we only fetch 3 hours + val availableHours = availableCountries.map { val hoursForDate = - diagnosisKeyServer.getHourIndex(it, currentDate).filter { availHour -> - TimeAndDateExtensions.getCurrentHourUTC() - 3 <= availHour.hourOfDay + diagnosisKeyServer.getHourIndex(it, currentDate).filter { availableHour -> + if (itemLimit < 0) { + true + } else { + TimeAndDateExtensions.getCurrentHourUTC() - itemLimit <= availableHour.hourOfDay + } } CountryHours(it, mapOf(currentDate to hoursForDate)) } - val cachedHours = keyCache - .getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) - .filter { it.first.isDownloadComplete && it.second.exists() } // We overwrite not completed ones - .map { it.first } + val cachedHours = getCompletedKeyFiles(CachedKeyInfo.Type.COUNTRY_HOUR) // All cached files that are no longer on the server are considered stale val staleHours = cachedHours.filter { cachedHour -> @@ -222,12 +236,33 @@ class KeyFileDownloader @Inject constructor( cachedHour.hour == time } } - if (staleHours.isNotEmpty()) keyCache.delete(staleHours) - val nonStaleHours = cachedHours.minus(staleHours) - val missingHours = availableHours.mapNotNull { - it.toMissingHours(nonStaleHours) + if (staleHours.isNotEmpty()) { + Timber.tag(TAG).v("Deleting stale hours: %s", staleHours) + keyCache.delete(staleHours) } + + val nonStaleHours = cachedHours.minus(staleHours) + + // The missing hours + return availableHours.mapNotNull { it.toMissingHours(nonStaleHours) } + } + + /** + * Fetches files given by serverDates by respecting countries + * @param currentDate base for where only dates within 3 hours before will be fetched + * @param availableCountries pair of dates per country code + */ + private suspend fun syncMissing3Hours( + currentDate: LocalDate, + availableCountries: List, + hourItemLimit: Int + ) = withContext(Dispatchers.IO) { + Timber.tag(TAG).v( + "asyncHandleLast3HoursFilesFetch(currentDate=%s, availableCountries=%s)", + currentDate, availableCountries + ) + val missingHours = determineMissingHours(currentDate, availableCountries, hourItemLimit) Timber.tag(TAG).d("Downloading missing hours: %s", missingHours) val hourDownloads = missingHours.flatMap { country -> @@ -261,7 +296,7 @@ class KeyFileDownloader @Inject constructor( path } - Unit + return@withContext } private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, saveTo: File) { @@ -299,5 +334,6 @@ class KeyFileDownloader @Inject constructor( companion object { private val TAG: String? = KeyFileDownloader::class.simpleName + private const val DEBUG_HOUR_LIMIT = 3 } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt index e68516a19df..75caa3780c9 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/DeviceStorage.kt @@ -56,6 +56,7 @@ class DeviceStorage @Inject constructor( return CheckResult( path = targetPath, isSpaceAvailable = availableBytes >= requiredBytes || requiredBytes == -1L, + requiredBytes = requiredBytes, freeBytes = availableBytes, totalBytes = totalBytes ) @@ -71,6 +72,7 @@ class DeviceStorage @Inject constructor( return CheckResult( path = targetPath, isSpaceAvailable = stats.availableBytes >= requiredBytes || requiredBytes == -1L, + requiredBytes = requiredBytes, freeBytes = stats.availableBytes, totalBytes = stats.totalBytes ) @@ -116,6 +118,7 @@ class DeviceStorage @Inject constructor( data class CheckResult( val path: File, val isSpaceAvailable: Boolean, + val requiredBytes: Long = -1L, val freeBytes: Long, val totalBytes: Long ) { diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/InsufficientStorageException.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/InsufficientStorageException.kt new file mode 100644 index 00000000000..32d18cb7c27 --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/InsufficientStorageException.kt @@ -0,0 +1,7 @@ +package de.rki.coronawarnapp.storage + +import java.io.IOException + +class InsufficientStorageException( + val result: DeviceStorage.CheckResult +) : IOException("Not enough free space (Want:${result.requiredBytes}; Have:${result.freeBytes}") diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt index 08a80c8ab89..974a243075b 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt @@ -311,7 +311,7 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { currentDate: Date, countries: List ) = executeState(FILES_FROM_WEB_REQUESTS) { - val convertedDate = LocalDate.fromDateFields(currentDate) // TODO confirm + val convertedDate = LocalDate.fromDateFields(currentDate) val locationCodes = countries.map { LocationCode(it) } keyFileDownloader.asyncFetchKeyFiles(convertedDate, locationCodes) } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt index 51e78d24ec1..e1dc0a997e6 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -8,6 +8,7 @@ import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository import de.rki.coronawarnapp.diagnosiskeys.storage.legacy.LegacyKeyCacheMigration import de.rki.coronawarnapp.storage.AppSettings import de.rki.coronawarnapp.storage.DeviceStorage +import de.rki.coronawarnapp.storage.InsufficientStorageException import de.rki.coronawarnapp.util.CWADebug import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldBe @@ -215,12 +216,13 @@ class KeyFileDownloaderTest : BaseIOTest() { coEvery { deviceStorage.checkSpacePrivateStorage(any()) } returns mockk().apply { every { isSpaceAvailable } returns false every { freeBytes } returns 1337L + every { requiredBytes } returns 5000L } val downloader = createDownloader() runBlocking { - shouldThrow { + shouldThrow { downloader.asyncFetchKeyFiles(LocalDate.now(), listOf(LocationCode("DE"))) } } diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt index b49fe05caf5..eed7854bfbf 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/storage/DeviceStorageTest.kt @@ -121,6 +121,7 @@ class DeviceStorageTest : BaseIOTest() { deviceStorage.checkSpacePrivateStorage(requiredBytes = defaultFreeSpace) shouldBe DeviceStorage.CheckResult( path = privateDataDir, isSpaceAvailable = true, + requiredBytes = defaultFreeSpace, freeBytes = defaultFreeSpace, totalBytes = defaultTotalSpace ) @@ -135,6 +136,7 @@ class DeviceStorageTest : BaseIOTest() { deviceStorage.checkSpacePrivateStorage(requiredBytes = defaultFreeSpace) shouldBe DeviceStorage.CheckResult( path = privateDataDir, isSpaceAvailable = true, + requiredBytes = defaultFreeSpace, freeBytes = defaultFreeSpace, totalBytes = defaultTotalSpace ) @@ -149,6 +151,7 @@ class DeviceStorageTest : BaseIOTest() { deviceStorage.checkSpacePrivateStorage(requiredBytes = targetBytes) shouldBe DeviceStorage.CheckResult( path = privateDataDir, isSpaceAvailable = true, + requiredBytes = targetBytes, freeBytes = targetBytes, totalBytes = defaultTotalSpace ) @@ -165,6 +168,7 @@ class DeviceStorageTest : BaseIOTest() { path = privateDataDir, isSpaceAvailable = false, freeBytes = defaultFreeSpace, + requiredBytes = Long.MAX_VALUE, totalBytes = defaultTotalSpace ) } @@ -177,6 +181,7 @@ class DeviceStorageTest : BaseIOTest() { deviceStorage.checkSpacePrivateStorage(requiredBytes = Long.MAX_VALUE) shouldBe DeviceStorage.CheckResult( path = privateDataDir, isSpaceAvailable = false, + requiredBytes = Long.MAX_VALUE, freeBytes = defaultFreeSpace, totalBytes = defaultTotalSpace ) From 949f8f27a3b19b21010b2fbfc2b67c262a9fde01 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Thu, 10 Sep 2020 23:42:49 +0200 Subject: [PATCH 18/29] Code fluff, formatting. --- .../rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt | 4 +++- .../coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt index 0816aa33800..95a99141304 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/AppConfigApiV1.kt @@ -7,5 +7,7 @@ import retrofit2.http.Path interface AppConfigApiV1 { @GET("/version/v1/configuration/country/{country}/app_config") - suspend fun getApplicationConfiguration(@Path("country") country: String): ResponseBody + suspend fun getApplicationConfiguration( + @Path("country") country: String + ): ResponseBody } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt index 6c35fafc022..258321c44c0 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyApiV1.kt @@ -13,7 +13,9 @@ interface DiagnosisKeyApiV1 { // TODO Let retrofit format this to LocalDate @GET("/version/v1/diagnosis-keys/country/{country}/date") - suspend fun getDayIndex(@Path("country") country: String): List + suspend fun getDayIndex( + @Path("country") country: String + ): List // TODO Let retrofit format this to LocalTime @GET("/version/v1/diagnosis-keys/country/{country}/date/{day}/hour") From fa196fc19b0425edc9f1bb2c9efed188e39fe5b7 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Thu, 10 Sep 2020 23:43:11 +0200 Subject: [PATCH 19/29] Handle download errors correctly. --- .../server/DiagnosisKeyServer.kt | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServer.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServer.kt index 783cd2556cd..8995751a9ee 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServer.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/DiagnosisKeyServer.kt @@ -7,6 +7,7 @@ import okhttp3.Headers import org.joda.time.LocalDate import org.joda.time.LocalTime import org.joda.time.format.DateTimeFormat +import retrofit2.HttpException import timber.log.Timber import java.io.File import javax.inject.Inject @@ -43,7 +44,7 @@ class DiagnosisKeyServer @Inject constructor( .map { hourString -> LocalTime.parse(hourString, HOUR_FORMATTER) } } - interface HeaderValidation { + interface HeaderHook { suspend fun validate(headers: Headers): Boolean = true } @@ -56,7 +57,7 @@ class DiagnosisKeyServer @Inject constructor( day: LocalDate, hour: LocalTime? = null, saveTo: File, - validator: HeaderValidation = object : HeaderValidation {} + headerHook: HeaderHook = object : HeaderHook {} ) = withContext(Dispatchers.IO) { Timber.tag(TAG).v( "Starting download: country=%s, day=%s, hour=%s -> %s.", @@ -65,7 +66,9 @@ class DiagnosisKeyServer @Inject constructor( if (saveTo.exists()) { Timber.tag(TAG).w("File existed, overwriting: %s", saveTo) - saveTo.delete() + if (saveTo.delete()) { + Timber.tag(TAG).e("%s exists, but can't be deleted.", saveTo) + } } val response = if (hour != null) { @@ -81,15 +84,20 @@ class DiagnosisKeyServer @Inject constructor( ) } - if (!validator.validate(response.headers())) { + if (!headerHook.validate(response.headers())) { Timber.tag(TAG).d("validateHeaders() told us to abort.") return@withContext } - - saveTo.outputStream().use { - response.body()!!.byteStream().copyTo(it, DEFAULT_BUFFER_SIZE) + if (response.isSuccessful) { + saveTo.outputStream().use { target -> + response.body()!!.byteStream().use { source -> + source.copyTo(target, DEFAULT_BUFFER_SIZE) + } + } + Timber.tag(TAG).v("Key file download successful: %s", saveTo) + } else { + throw HttpException(response) } - Timber.tag(TAG).v("Key file download successful: %s", saveTo) } companion object { From bcc6418128cd87dd3930219c2b6fb8b272013a84 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 00:19:22 +0200 Subject: [PATCH 20/29] Refactoring: * Remove unnecessary `currentDate` we always start with the newest date from the servers index. * Make a specialised class for header validation --- .../TestForAPIFragment.kt | 4 +- .../download/KeyFileDownloader.kt | 155 ++++++++---------- .../diagnosiskeys/server/KeyFileHeaderHook.kt | 23 +++ .../storage/KeyCacheRepository.kt | 3 +- .../RetrieveDiagnosisKeysTransaction.kt | 10 +- .../download/KeyFileDownloaderTest.kt | 113 +++++++++---- 6 files changed, 181 insertions(+), 127 deletions(-) create mode 100644 Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/KeyFileHeaderHook.kt diff --git a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt index 016e0aafac1..1e457e5ad9f 100644 --- a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt +++ b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestForAPIFragment.kt @@ -58,7 +58,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.joda.time.DateTime import org.joda.time.DateTimeZone -import org.joda.time.LocalDate import timber.log.Timber import java.io.File import java.lang.reflect.Type @@ -322,10 +321,9 @@ class TestForAPIFragment : Fragment(), InternalExposureNotificationPermissionHel lastSetCountries = countryCodes // Trigger asyncFetchFiles which will use all Countries passed as parameter - val currentDate = LocalDate.now() lifecycleScope.launch { val locationCodes = countryCodes.map { LocationCode(it) } - AppInjector.component.keyFileDownloader.asyncFetchKeyFiles(currentDate, locationCodes) + AppInjector.component.keyFileDownloader.asyncFetchKeyFiles(locationCodes) updateCountryStatusLabel() } } diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 30d5e72a757..a7810ad12d0 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -2,6 +2,7 @@ package de.rki.coronawarnapp.diagnosiskeys.download import dagger.Reusable import de.rki.coronawarnapp.diagnosiskeys.server.DiagnosisKeyServer +import de.rki.coronawarnapp.diagnosiskeys.server.KeyFileHeaderHook import de.rki.coronawarnapp.diagnosiskeys.server.LocationCode import de.rki.coronawarnapp.diagnosiskeys.storage.CachedKeyInfo import de.rki.coronawarnapp.diagnosiskeys.storage.KeyCacheRepository @@ -11,14 +12,12 @@ import de.rki.coronawarnapp.storage.DeviceStorage import de.rki.coronawarnapp.storage.InsufficientStorageException import de.rki.coronawarnapp.util.CWADebug import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 -import de.rki.coronawarnapp.util.TimeAndDateExtensions import de.rki.coronawarnapp.util.debug.measureTimeMillisWithResult import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.withContext -import okhttp3.Headers -import org.joda.time.LocalDate +import org.joda.time.LocalTime import timber.log.Timber import java.io.File import javax.inject.Inject @@ -29,7 +28,7 @@ import javax.inject.Inject @Reusable class KeyFileDownloader @Inject constructor( private val deviceStorage: DeviceStorage, - private val diagnosisKeyServer: DiagnosisKeyServer, + private val keyServer: DiagnosisKeyServer, private val keyCache: KeyCacheRepository, private val legacyKeyCache: LegacyKeyCacheMigration, private val settings: AppSettings @@ -68,67 +67,62 @@ class KeyFileDownloader @Inject constructor( * - the difference can only work properly if the date from the device is synchronized through the net * - the difference in timezone is taken into account by using UTC in the Conversion from the Date to Server format * - * @param currentDate the current date - if this is adjusted by the calendar, the cache is affected. * @return list of all files from both the cache and the diff query */ - suspend fun asyncFetchKeyFiles( - currentDate: LocalDate, - wantedCountries: List - ): List = withContext(Dispatchers.IO) { - val availableCountries = diagnosisKeyServer.getCountryIndex() - val filteredCountries = availableCountries.filter { wantedCountries.contains(it) } - Timber.tag(TAG).v( - "Available=%s; Wanted=%s; Intersect=%s", - availableCountries, wantedCountries, filteredCountries - ) - - val storageResult = checkStorageSpace(filteredCountries) - if (!storageResult.isSpaceAvailable) throw InsufficientStorageException(storageResult) - - val availableKeys = if (CWADebug.isDebugBuildOrMode && settings.isLast3HourModeEnabled) { - syncMissing3Hours(currentDate, filteredCountries, DEBUG_HOUR_LIMIT) - keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) - } else { - syncMissingDays(filteredCountries) - keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) - } - - return@withContext availableKeys - .filter { it.first.isDownloadComplete && it.second.exists() } - .mapNotNull { (keyInfo, path) -> - if (!path.exists()) { - Timber.tag(TAG).w("Missing keyfile for : %s", keyInfo) - null + suspend fun asyncFetchKeyFiles(wantedCountries: List): List = + withContext(Dispatchers.IO) { + val availableCountries = keyServer.getCountryIndex() + val filteredCountries = availableCountries.filter { wantedCountries.contains(it) } + Timber.tag(TAG).v( + "Available=%s; Wanted=%s; Intersect=%s", + availableCountries, wantedCountries, filteredCountries + ) + + val storageResult = checkStorageSpace(filteredCountries) + if (!storageResult.isSpaceAvailable) throw InsufficientStorageException(storageResult) + + val availableKeys = + if (CWADebug.isDebugBuildOrMode && settings.isLast3HourModeEnabled) { + syncMissing3Hours(filteredCountries, DEBUG_HOUR_LIMIT) + keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_HOUR) } else { - path + syncMissingDays(filteredCountries) + keyCache.getEntriesForType(CachedKeyInfo.Type.COUNTRY_DAY) } - } - .also { Timber.tag(TAG).d("Returning %d available keyfiles", it.size) } - } + + return@withContext availableKeys + .filter { it.first.isDownloadComplete && it.second.exists() } + .mapNotNull { (keyInfo, path) -> + if (!path.exists()) { + Timber.tag(TAG).w("Missing keyfile for : %s", keyInfo) + null + } else { + path + } + } + .also { Timber.tag(TAG).d("Returning %d available keyfiles", it.size) } + } private suspend fun determineMissingDays(availableCountries: List): List { val availableDays = availableCountries.map { - val days = diagnosisKeyServer.getDayIndex(it) + val days = keyServer.getDayIndex(it) CountryDays(it, days) } val cachedDays = getCompletedKeyFiles(CachedKeyInfo.Type.COUNTRY_DAY) // All cached files that are no longer on the server are considered stale - val staleDays = cachedDays.filter { cachedKeyFile -> + val staleDays = cachedDays.filter { cachedDay -> val availableCountry = availableDays.singleOrNull { - it.country == cachedKeyFile.location + it.country == cachedDay.location } if (availableCountry == null) { - Timber.tag(TAG).w( - "Unknown location %s, assuming stale cache.", - cachedKeyFile.location - ) + Timber.tag(TAG).w("Unknown location %s, assuming stale day.", cachedDay.location) return@filter true // It's stale } availableCountry.dayData.none { date -> - cachedKeyFile.day == date + cachedDay.day == date } } @@ -196,22 +190,26 @@ class KeyFileDownloader @Inject constructor( } private suspend fun determineMissingHours( - currentDate: LocalDate, availableCountries: List, itemLimit: Int ): List { - // This is currently used for debugging, so we only fetch 3 hours - val availableHours = availableCountries.map { - val hoursForDate = - diagnosisKeyServer.getHourIndex(it, currentDate).filter { availableHour -> - if (itemLimit < 0) { - true - } else { - TimeAndDateExtensions.getCurrentHourUTC() - itemLimit <= availableHour.hourOfDay - } + val availableHours = availableCountries.flatMap { location -> + var remainingItems = itemLimit + // Descending because we go backwards newest -> oldest + keyServer.getDayIndex(location).sortedDescending().mapNotNull { day -> + // Limit reached, return null (filtered out) instead of new CountryHours object + if (remainingItems <= 0) return@mapNotNull null + + val hoursForDate = mutableListOf() + for (hour in keyServer.getHourIndex(location, day).sortedDescending()) { + if (remainingItems <= 0) break + remainingItems-- + hoursForDate.add(hour) } - CountryHours(it, mapOf(currentDate to hoursForDate)) + + CountryHours(location, mapOf(day to hoursForDate)) + } } val cachedHours = getCompletedKeyFiles(CachedKeyInfo.Type.COUNTRY_HOUR) @@ -219,16 +217,16 @@ class KeyFileDownloader @Inject constructor( // All cached files that are no longer on the server are considered stale val staleHours = cachedHours.filter { cachedHour -> val availCountry = availableHours.singleOrNull { - it.country == cachedHour.location + it.country == cachedHour.location && it.hourData.containsKey(cachedHour.day) } if (availCountry == null) { - Timber.w("Unknown location %s, assuming stale.", cachedHour.location) + Timber.w("Unknown location %s, assuming stale hour.", cachedHour.location) return@filter true // It's stale } val availableDay = availCountry.hourData[cachedHour.day] if (availableDay == null) { - Timber.d("Unknown day %s, assuming stale.", cachedHour.location) + Timber.d("Unknown day %s, assuming stale hour.", cachedHour.location) return@filter true // It's stale } @@ -254,15 +252,14 @@ class KeyFileDownloader @Inject constructor( * @param availableCountries pair of dates per country code */ private suspend fun syncMissing3Hours( - currentDate: LocalDate, availableCountries: List, hourItemLimit: Int ) = withContext(Dispatchers.IO) { Timber.tag(TAG).v( - "asyncHandleLast3HoursFilesFetch(currentDate=%s, availableCountries=%s)", - currentDate, availableCountries + "asyncHandleLast3HoursFilesFetch(availableCountries=%s, hourLimit=%d)", + availableCountries, hourItemLimit ) - val missingHours = determineMissingHours(currentDate, availableCountries, hourItemLimit) + val missingHours = determineMissingHours(availableCountries, hourItemLimit) Timber.tag(TAG).d("Downloading missing hours: %s", missingHours) val hourDownloads = missingHours.flatMap { country -> @@ -300,28 +297,20 @@ class KeyFileDownloader @Inject constructor( } private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, saveTo: File) { - val validation = object : DiagnosisKeyServer.HeaderValidation { - override suspend fun validate(headers: Headers): Boolean { - var fileMD5 = headers.values("ETag").singleOrNull() -// var fileMD5 = headers.values("x-amz-meta-cwa-hash-md5").singleOrNull() -// if (fileMD5 == null) { -// headers.values("x-amz-meta-cwa-hash").singleOrNull() -// } -// if (fileMD5 == null) { // Fallback -// fileMD5 = headers.values("ETag").singleOrNull() -// } - fileMD5 = fileMD5?.removePrefix("\"")?.removeSuffix("\"") - - return !legacyKeyCache.tryMigration(fileMD5, saveTo) - } + val validation = KeyFileHeaderHook { headers -> + // tryMigration returns true when a file was migrated, meaning, no download necessary + return@KeyFileHeaderHook !legacyKeyCache.tryMigration( + headers.getPayloadChecksumMD5(), + saveTo + ) } - diagnosisKeyServer.downloadKeyFile( - keyInfo.location, - keyInfo.day, - keyInfo.hour, - saveTo, - validation + keyServer.downloadKeyFile( + locationCode = keyInfo.location, + day = keyInfo.day, + hour = keyInfo.hour, + saveTo = saveTo, + headerHook = validation ) Timber.tag(TAG).v("Dowwnload finished: %s -> %s", keyInfo, saveTo) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/KeyFileHeaderHook.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/KeyFileHeaderHook.kt new file mode 100644 index 00000000000..b7613c53b8d --- /dev/null +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/server/KeyFileHeaderHook.kt @@ -0,0 +1,23 @@ +package de.rki.coronawarnapp.diagnosiskeys.server + +import okhttp3.Headers + +class KeyFileHeaderHook( + private val onEval: suspend KeyFileHeaderHook.(Headers) -> Boolean +) : DiagnosisKeyServer.HeaderHook { + + override suspend fun validate(headers: Headers): Boolean = onEval(headers) + + fun Headers.getPayloadChecksumMD5(): String? { + // TODO Ping backend regarding alternative checksum sources + var fileMD5 = values("ETag").singleOrNull() +// var fileMD5 = headers.values("x-amz-meta-cwa-hash-md5").singleOrNull() +// if (fileMD5 == null) { +// headers.values("x-amz-meta-cwa-hash").singleOrNull() +// } +// if (fileMD5 == null) { // Fallback +// fileMD5 = headers.values("ETag").singleOrNull() +// } + return fileMD5?.removePrefix("\"")?.removeSuffix("\"") + } +} diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt index 9365b140c34..c0a2379c073 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt @@ -75,6 +75,7 @@ class KeyCacheRepository @Inject constructor( val dirtyInfos = getDao().getAllEntries().filter { it.isDownloadComplete && !getPathForKey(it).exists() } + Timber.v("HouseKeeping, deleting: %s", dirtyInfos) delete(dirtyInfos) } @@ -109,7 +110,7 @@ class KeyCacheRepository @Inject constructor( try { getDao().insertEntry(newKeyFile) if (targetFile.exists()) { - Timber.w("Target path despire no collision exists, deleting: %s", targetFile) + Timber.w("Target path despite no collision exists, deleting: %s", targetFile) } } catch (e: SQLiteConstraintException) { Timber.e(e, "Insertion collision? Overwriting for %s", newKeyFile) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt index 974a243075b..06bd08ecd9a 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt @@ -41,7 +41,6 @@ import de.rki.coronawarnapp.worker.BackgroundWorkHelper import org.joda.time.DateTime import org.joda.time.DateTimeZone import org.joda.time.Instant -import org.joda.time.LocalDate import timber.log.Timber import java.io.File import java.util.Date @@ -198,10 +197,7 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { onKeyFilesDownloadStarted = null } - val keyFiles = executeFetchKeyFilesFromServer( - currentDate, - countries - ) + val keyFiles = executeFetchKeyFilesFromServer(countries) if (CWADebug.isDebugBuildOrMode) { val totalFileSize = keyFiles.fold(0L, { acc, file -> @@ -308,12 +304,10 @@ object RetrieveDiagnosisKeysTransaction : Transaction() { * Executes the WEB_REQUESTS Transaction State */ private suspend fun executeFetchKeyFilesFromServer( - currentDate: Date, countries: List ) = executeState(FILES_FROM_WEB_REQUESTS) { - val convertedDate = LocalDate.fromDateFields(currentDate) val locationCodes = countries.map { LocationCode(it) } - keyFileDownloader.asyncFetchKeyFiles(convertedDate, locationCodes) + keyFileDownloader.asyncFetchKeyFiles(locationCodes) } /** diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt index e1dc0a997e6..3cdf69b5819 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -83,7 +83,7 @@ class KeyFileDownloaderTest : BaseIOTest() { coEvery { diagnosisKeyServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-01")) } returns listOf( - LocalTime.parse("20") + LocalTime.parse("18"), LocalTime.parse("19"), LocalTime.parse("20") ) coEvery { diagnosisKeyServer.getHourIndex(LocationCode("DE"), LocalDate.parse("2020-09-02")) @@ -96,7 +96,7 @@ class KeyFileDownloaderTest : BaseIOTest() { coEvery { diagnosisKeyServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-02")) } returns listOf( - LocalTime.parse("22") + LocalTime.parse("20"), LocalTime.parse("21"), LocalTime.parse("22") ) coEvery { diagnosisKeyServer.getHourIndex(LocationCode("NL"), LocalDate.parse("2020-09-03")) @@ -170,8 +170,8 @@ class KeyFileDownloaderTest : BaseIOTest() { day: LocalDate, hour: LocalTime? = null, saveTo: File, - validator: DiagnosisKeyServer.HeaderValidation = object : - DiagnosisKeyServer.HeaderValidation {} + validator: DiagnosisKeyServer.HeaderHook = object : + DiagnosisKeyServer.HeaderHook {} ) { saveTo.writeText("$locationCode.$day.$hour") } @@ -194,7 +194,7 @@ class KeyFileDownloaderTest : BaseIOTest() { private fun createDownloader(): KeyFileDownloader { val downloader = KeyFileDownloader( deviceStorage = deviceStorage, - diagnosisKeyServer = diagnosisKeyServer, + keyServer = diagnosisKeyServer, keyCache = keyCache, legacyKeyCache = legacyMigration, settings = settings @@ -207,7 +207,7 @@ class KeyFileDownloaderTest : BaseIOTest() { fun `storage is checked before fetching`() { val downloader = createDownloader() runBlocking { - downloader.asyncFetchKeyFiles(LocalDate.now(), emptyList()) shouldBe emptyList() + downloader.asyncFetchKeyFiles(emptyList()) shouldBe emptyList() } } @@ -223,7 +223,7 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { shouldThrow { - downloader.asyncFetchKeyFiles(LocalDate.now(), listOf(LocationCode("DE"))) + downloader.asyncFetchKeyFiles(listOf(LocationCode("DE"))) } } } @@ -236,7 +236,7 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { shouldThrow { - downloader.asyncFetchKeyFiles(LocalDate.now(), listOf(LocationCode("DE"))) + downloader.asyncFetchKeyFiles(listOf(LocationCode("DE"))) } } } @@ -247,7 +247,6 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.now(), listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 4 } @@ -304,7 +303,6 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.now(), listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 4 } @@ -353,7 +351,6 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.now(), listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 3 } @@ -390,7 +387,6 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.now(), listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 3 } @@ -405,9 +401,8 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.parse("2020-09-02"), listOf(LocationCode("DE"), LocationCode("NL")) - ).size shouldBe 3 + ).size shouldBe 6 } coVerify { @@ -415,13 +410,32 @@ class KeyFileDownloaderTest : BaseIOTest() { type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("DE"), dayIdentifier = LocalDate.parse("2020-09-02"), - hourIdentifier = LocalTime.parse("20") + hourIdentifier = LocalTime.parse("21") ) keyCache.createCacheEntry( type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("DE"), dayIdentifier = LocalDate.parse("2020-09-02"), - hourIdentifier = LocalTime.parse("21") + hourIdentifier = LocalTime.parse("20") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + dayIdentifier = LocalDate.parse("2020-09-01"), + hourIdentifier = LocalTime.parse("20") + ) + + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = LocalTime.parse("23") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = LocalTime.parse("22") ) keyCache.createCacheEntry( type = CachedKeyInfo.Type.COUNTRY_HOUR, @@ -430,9 +444,9 @@ class KeyFileDownloaderTest : BaseIOTest() { hourIdentifier = LocalTime.parse("22") ) } - coVerify(exactly = 3) { keyCache.markKeyComplete(any(), any()) } + coVerify(exactly = 6) { keyCache.markKeyComplete(any(), any()) } - keyRepoData.size shouldBe 3 + keyRepoData.size shouldBe 6 keyRepoData.values.forEach { it.isDownloadComplete shouldBe true } } @@ -441,6 +455,13 @@ class KeyFileDownloaderTest : BaseIOTest() { every { CWADebug.isDebugBuildOrMode } returns true every { settings.isLast3HourModeEnabled } returns true + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + day = LocalDate.parse("2020-09-01"), + hour = LocalTime.parse("20"), + isCompleted = true + ) mockAddData( type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("NL"), @@ -453,9 +474,8 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.parse("2020-09-02"), listOf(LocationCode("DE"), LocationCode("NL")) - ).size shouldBe 3 + ).size shouldBe 6 } coVerify { @@ -463,16 +483,29 @@ class KeyFileDownloaderTest : BaseIOTest() { type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("DE"), dayIdentifier = LocalDate.parse("2020-09-02"), - hourIdentifier = LocalTime.parse("20") + hourIdentifier = LocalTime.parse("21") ) keyCache.createCacheEntry( type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("DE"), dayIdentifier = LocalDate.parse("2020-09-02"), - hourIdentifier = LocalTime.parse("21") + hourIdentifier = LocalTime.parse("20") + ) + + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = LocalTime.parse("23") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = LocalTime.parse("22") ) } - coVerify(exactly = 2) { + coVerify(exactly = 4) { diagnosisKeyServer.downloadKeyFile( any(), any(), @@ -504,6 +537,13 @@ class KeyFileDownloaderTest : BaseIOTest() { isCompleted = true ) + mockAddData( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("DE"), + day = LocalDate.parse("2020-09-01"), + hour = LocalTime.parse("20"), + isCompleted = true + ) mockAddData( type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("NL"), @@ -516,9 +556,8 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.parse("2020-09-02"), listOf(LocationCode("DE"), LocationCode("NL")) - ).size shouldBe 3 + ).size shouldBe 6 } coVerify { @@ -526,16 +565,29 @@ class KeyFileDownloaderTest : BaseIOTest() { type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("DE"), dayIdentifier = LocalDate.parse("2020-09-02"), - hourIdentifier = LocalTime.parse("20") + hourIdentifier = LocalTime.parse("21") ) keyCache.createCacheEntry( type = CachedKeyInfo.Type.COUNTRY_HOUR, location = LocationCode("DE"), dayIdentifier = LocalDate.parse("2020-09-02"), - hourIdentifier = LocalTime.parse("21") + hourIdentifier = LocalTime.parse("20") + ) + + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = LocalTime.parse("23") + ) + keyCache.createCacheEntry( + type = CachedKeyInfo.Type.COUNTRY_HOUR, + location = LocationCode("NL"), + dayIdentifier = LocalDate.parse("2020-09-03"), + hourIdentifier = LocalTime.parse("22") ) } - coVerify(exactly = 2) { + coVerify(exactly = 4) { diagnosisKeyServer.downloadKeyFile( any(), any(), @@ -563,9 +615,8 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.parse("2020-09-02"), // Has 3 hour files listOf(LocationCode("DE"), LocationCode("NL")) - ).size shouldBe 2 + ).size shouldBe 5 } } @@ -583,7 +634,6 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.now(), listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 4 } @@ -611,7 +661,6 @@ class KeyFileDownloaderTest : BaseIOTest() { runBlocking { downloader.asyncFetchKeyFiles( - LocalDate.now(), listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 3 } From 7fd919b2222e28bbfbf29a1bb4737f67df6d0f97 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 00:33:35 +0200 Subject: [PATCH 21/29] Let legacy cache migration abort early, depending on whether the key dir exists. --- .../storage/legacy/LegacyKeyCacheMigration.kt | 13 ++++++++++- .../legacy/LegacyKeyCacheMigrationTest.kt | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt index 58a4e2ce37f..ca2cf95867b 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigration.kt @@ -2,6 +2,7 @@ package de.rki.coronawarnapp.diagnosiskeys.storage.legacy import android.content.Context import dagger.Lazy +import de.rki.coronawarnapp.risk.TimeVariables import de.rki.coronawarnapp.util.HashExtensions.hashToMD5 import de.rki.coronawarnapp.util.TimeStamper import kotlinx.coroutines.sync.Mutex @@ -30,6 +31,11 @@ class LegacyKeyCacheMigration @Inject constructor( if (isInit) return isInit = true + if (!cacheDir.exists()) { + Timber.tag(TAG).v("Legacy cache dir doesn't exist, we are done.") + return + } + try { legacyDao.get().clear() } catch (e: Exception) { @@ -42,7 +48,7 @@ class LegacyKeyCacheMigration @Inject constructor( val isExpired = Duration( Instant.ofEpochMilli(file.lastModified()), timeStamper.nowUTC - ).standardDays > 15 + ).standardDays > TimeVariables.getDefaultRetentionPeriodInDays() if (isExpired) { Timber.tag(TAG).d("Deleting expired file: %s", file) @@ -57,6 +63,11 @@ class LegacyKeyCacheMigration @Inject constructor( Timber.tag(TAG).e(e, "Reading legacy cached failed. Clearing.") cacheDir.deleteRecursively() } + + if (cacheDir.exists() && cacheDir.listFiles()?.isNullOrEmpty() == true) { + Timber.tag(TAG).v("Legacy cache dir is empty, deleting.") + cacheDir.delete() + } } suspend fun tryMigration(fileMD5: String?, targetPath: File): Boolean = workMutex.withLock { diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt index 3cf5cc0de0a..e1a33d27180 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/storage/legacy/LegacyKeyCacheMigrationTest.kt @@ -190,4 +190,26 @@ class LegacyKeyCacheMigrationTest : BaseIOTest() { legacyFile1.exists() shouldBe false migrationTarget.exists() shouldBe false } + + @Test + fun `init deletes empty cache dir`() { + legacyDir.mkdirs() + legacyDir.exists() shouldBe true + + runBlocking { + val tool = createTool() + tool.tryMigration("doesntmatter", File(testDir, "1")) + } + legacyDir.exists() shouldBe false + legacyDir.parentFile!!.exists() shouldBe true + + runBlocking { + val tool = createTool() + tool.tryMigration("doesntmatter", File(testDir, "1")) + } + legacyDir.exists() shouldBe false + legacyDir.parentFile!!.exists() shouldBe true + + coVerify(exactly = 1) { legacyDao.clear() } + } } From 4fa414a3e147b44a6efd12e77f41794a3faf1182 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 00:34:00 +0200 Subject: [PATCH 22/29] If we can't create the base directory for the key repo, throw an exception. --- .../coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt index c0a2379c073..4336c7ac87f 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/storage/KeyCacheRepository.kt @@ -29,6 +29,7 @@ import org.joda.time.LocalDate import org.joda.time.LocalTime import timber.log.Timber import java.io.File +import java.io.IOException import javax.inject.Inject import javax.inject.Singleton @@ -45,7 +46,7 @@ class KeyCacheRepository @Inject constructor( if (mkdirs()) { Timber.d("KeyCache directory created: %s", this) } else { - Timber.w("KeyCache directory creation failed: %s", this) + throw IOException("KeyCache directory creation failed: $this") } } } From 610aa6e241043c2a0f08dcf7cd4c5f79f5415044 Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 00:56:46 +0200 Subject: [PATCH 23/29] Delete cache entry if a download fails. --- .../coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index a7810ad12d0..4b0c66ea456 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -248,8 +248,8 @@ class KeyFileDownloader @Inject constructor( /** * Fetches files given by serverDates by respecting countries - * @param currentDate base for where only dates within 3 hours before will be fetched * @param availableCountries pair of dates per country code + * @param hourItemLimit how many hours to go back */ private suspend fun syncMissing3Hours( availableCountries: List, @@ -278,6 +278,7 @@ class KeyFileDownloader @Inject constructor( keyInfo to path } catch (e: Exception) { Timber.tag(TAG).e(e, "Download failed: %s", keyInfo) + keyCache.delete(listOf(keyInfo)) null } } From d694bac0ac4a433e12dc4a1cfa17bf1532e1058d Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 00:57:03 +0200 Subject: [PATCH 24/29] Fixed test regression due to refactoring. --- .../RetrieveDiagnosisKeysTransactionTest.kt | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt index f5d2817f6cc..c13d4f6f2c3 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransactionTest.kt @@ -63,8 +63,11 @@ class RetrieveDiagnosisKeysTransactionTest { @Test fun testTransactionNoFiles() { val requestedCountries = listOf("DE") - coEvery { RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"](any(), - requestedCountries) } returns listOf() + coEvery { + RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"]( + requestedCountries + ) + } returns listOf() runBlocking { RetrieveDiagnosisKeysTransaction.start(requestedCountries) @@ -72,7 +75,9 @@ class RetrieveDiagnosisKeysTransactionTest { coVerifyOrder { RetrieveDiagnosisKeysTransaction["executeSetup"]() RetrieveDiagnosisKeysTransaction["executeRetrieveRiskScoreParams"]() - RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"](any(), requestedCountries) + RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"]( + requestedCountries + ) RetrieveDiagnosisKeysTransaction["executeFetchDateUpdate"](any()) } } @@ -83,7 +88,11 @@ class RetrieveDiagnosisKeysTransactionTest { val file = Paths.get("src", "test", "resources", "keys.bin").toFile() val requestedCountries = listOf("DE") - coEvery { RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"](any(), requestedCountries) } returns listOf(file) + coEvery { + RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"]( + requestedCountries + ) + } returns listOf(file) runBlocking { RetrieveDiagnosisKeysTransaction.start(requestedCountries) @@ -92,7 +101,6 @@ class RetrieveDiagnosisKeysTransactionTest { RetrieveDiagnosisKeysTransaction["executeSetup"]() RetrieveDiagnosisKeysTransaction["executeRetrieveRiskScoreParams"]() RetrieveDiagnosisKeysTransaction["executeFetchKeyFilesFromServer"]( - any(), requestedCountries ) RetrieveDiagnosisKeysTransaction["executeAPISubmission"]( From e16e6b429330a7283d6f29f028048329d6592d2a Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 09:55:06 +0200 Subject: [PATCH 25/29] Consolidate staleness check into `getStale` --- .../download/KeyFileDownloader.kt | 79 +++++++++++-------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index 4b0c66ea456..dc3e5d1a6f4 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -111,20 +111,7 @@ class KeyFileDownloader @Inject constructor( val cachedDays = getCompletedKeyFiles(CachedKeyInfo.Type.COUNTRY_DAY) - // All cached files that are no longer on the server are considered stale - val staleDays = cachedDays.filter { cachedDay -> - val availableCountry = availableDays.singleOrNull { - it.country == cachedDay.location - } - if (availableCountry == null) { - Timber.tag(TAG).w("Unknown location %s, assuming stale day.", cachedDay.location) - return@filter true // It's stale - } - - availableCountry.dayData.none { date -> - cachedDay.day == date - } - } + val staleDays = getStale(cachedDays, availableDays) if (staleDays.isNotEmpty()) { Timber.tag(TAG).v("Deleting stale days: %s", staleDays) @@ -214,26 +201,7 @@ class KeyFileDownloader @Inject constructor( val cachedHours = getCompletedKeyFiles(CachedKeyInfo.Type.COUNTRY_HOUR) - // All cached files that are no longer on the server are considered stale - val staleHours = cachedHours.filter { cachedHour -> - val availCountry = availableHours.singleOrNull { - it.country == cachedHour.location && it.hourData.containsKey(cachedHour.day) - } - if (availCountry == null) { - Timber.w("Unknown location %s, assuming stale hour.", cachedHour.location) - return@filter true // It's stale - } - - val availableDay = availCountry.hourData[cachedHour.day] - if (availableDay == null) { - Timber.d("Unknown day %s, assuming stale hour.", cachedHour.location) - return@filter true // It's stale - } - - availableDay.none { time -> - cachedHour.hour == time - } - } + val staleHours = getStale(cachedHours, availableHours) if (staleHours.isNotEmpty()) { Timber.tag(TAG).v("Deleting stale hours: %s", staleHours) @@ -246,6 +214,49 @@ class KeyFileDownloader @Inject constructor( return availableHours.mapNotNull { it.toMissingHours(nonStaleHours) } } + // All cached files that are no longer on the server are considered stale + private fun getStale( + cachedKeys: List, + availableData: List + ): List = cachedKeys.filter { cachedKey -> + val availableCountry = availableData + .filter { it.country == cachedKey.location } + .singleOrNull { + when (cachedKey.type) { + CachedKeyInfo.Type.COUNTRY_DAY -> true + CachedKeyInfo.Type.COUNTRY_HOUR -> { + it as CountryHours + it.hourData.containsKey(cachedKey.day) + } + } + } + if (availableCountry == null) { + Timber.w("Unknown location %s, assuming stale hour.", cachedKey.location) + return@filter true // It's stale + } + + when (cachedKey.type) { + CachedKeyInfo.Type.COUNTRY_DAY -> { + availableCountry as CountryDays + availableCountry.dayData.none { date -> + cachedKey.day == date + } + } + CachedKeyInfo.Type.COUNTRY_HOUR -> { + availableCountry as CountryHours + val availableDay = availableCountry.hourData[cachedKey.day] + if (availableDay == null) { + Timber.d("Unknown day %s, assuming stale hour.", cachedKey.location) + return@filter true // It's stale + } + + availableDay.none { time -> + cachedKey.hour == time + } + } + } + } + /** * Fetches files given by serverDates by respecting countries * @param availableCountries pair of dates per country code From 8c4e31264890d78b446df12d94ff417813ccd84e Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 10:06:07 +0200 Subject: [PATCH 26/29] Consolidate clean up for failed downloads into the download method. Added tests to check that we delete the keycache entry if the download fails (which we didn't for hours :O!) --- .../download/KeyFileDownloader.kt | 27 ++++++++----------- .../download/KeyFileDownloaderTest.kt | 6 +++++ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt index dc3e5d1a6f4..f8148c33d8f 100644 --- a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt +++ b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloader.kt @@ -148,13 +148,7 @@ class KeyFileDownloader @Inject constructor( type = CachedKeyInfo.Type.COUNTRY_DAY ) - return@async try { - downloadKeyFile(keyInfo, path) - keyInfo to path - } catch (e: Exception) { - Timber.tag(TAG).e(e, "Download failed: %s", keyInfo) - null - } + return@async downloadKeyFile(keyInfo, path) } } @@ -284,14 +278,7 @@ class KeyFileDownloader @Inject constructor( type = CachedKeyInfo.Type.COUNTRY_HOUR ) - return@async try { - downloadKeyFile(keyInfo, path) - keyInfo to path - } catch (e: Exception) { - Timber.tag(TAG).e(e, "Download failed: %s", keyInfo) - keyCache.delete(listOf(keyInfo)) - null - } + return@async downloadKeyFile(keyInfo, path) } } } @@ -308,7 +295,10 @@ class KeyFileDownloader @Inject constructor( return@withContext } - private suspend fun downloadKeyFile(keyInfo: CachedKeyInfo, saveTo: File) { + private suspend fun downloadKeyFile( + keyInfo: CachedKeyInfo, + saveTo: File + ): Pair? = try { val validation = KeyFileHeaderHook { headers -> // tryMigration returns true when a file was migrated, meaning, no download necessary return@KeyFileHeaderHook !legacyKeyCache.tryMigration( @@ -331,6 +321,11 @@ class KeyFileDownloader @Inject constructor( Timber.tag(TAG).v("Hashed to MD5 in %dms: %s", duration, saveTo) keyCache.markKeyComplete(keyInfo, downloadedMD5) + keyInfo to saveTo + } catch (e: Exception) { + Timber.tag(TAG).e(e, "Download failed: %s", keyInfo) + keyCache.delete(listOf(keyInfo)) + null } companion object { diff --git a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt index 3cdf69b5819..d9df9734b50 100644 --- a/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt +++ b/Corona-Warn-App/src/test/java/de/rki/coronawarnapp/diagnosiskeys/download/KeyFileDownloaderTest.kt @@ -390,6 +390,9 @@ class KeyFileDownloaderTest : BaseIOTest() { listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 3 } + + // We delete the entry for the failed download + coVerify(exactly = 1) { keyCache.delete(any()) } } @Test @@ -618,6 +621,9 @@ class KeyFileDownloaderTest : BaseIOTest() { listOf(LocationCode("DE"), LocationCode("NL")) ).size shouldBe 5 } + + // We delete the entry for the failed download + coVerify(exactly = 1) { keyCache.delete(any()) } } @Test From 10407cf00e9c8cb998c1db0cfb35f3a03f04322b Mon Sep 17 00:00:00 2001 From: Matthias Urhahn Date: Fri, 11 Sep 2020 10:21:50 +0200 Subject: [PATCH 27/29] Because the hour-mode uses caching too, we add an explicit button to the test menu that clears the cache. --- .../de.rki.coronawarnapp/TestRiskLevelCalculation.kt | 6 ++++++ .../res/layout/fragment_test_for_a_p_i.xml | 7 ++++--- .../layout/fragment_test_risk_level_calculation.xml | 12 +++++++++++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt index a1cfbb2868e..945ddca4e51 100644 --- a/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt +++ b/Corona-Warn-App/src/deviceForTesters/java/de.rki.coronawarnapp/TestRiskLevelCalculation.kt @@ -130,6 +130,12 @@ class TestRiskLevelCalculation : Fragment() { } } + binding.buttonClearDiagnosisKeyCache.setOnClickListener { + lifecycleScope.launch { + AppInjector.component.keyCacheRepository.clear() + } + } + startObserving() } diff --git a/Corona-Warn-App/src/deviceForTesters/res/layout/fragment_test_for_a_p_i.xml b/Corona-Warn-App/src/deviceForTesters/res/layout/fragment_test_for_a_p_i.xml index 2452245478a..09ac7491525 100644 --- a/Corona-Warn-App/src/deviceForTesters/res/layout/fragment_test_for_a_p_i.xml +++ b/Corona-Warn-App/src/deviceForTesters/res/layout/fragment_test_for_a_p_i.xml @@ -1,5 +1,7 @@ - + @@ -241,8 +243,7 @@ android:id="@+id/input_country_codes_editText" android:layout_width="0dp" android:layout_height="wrap_content" - android:layout_weight="1" - /> + android:layout_weight="1" />