fix: Prevent OutOfMemoryException when uploading files(2)

projectNEWM · Nov 14, 2024 · 865f1c1 · 865f1c1
1 parent 3157cb2
commit 865f1c1
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 16 deletions.
diff --git a/...in/kotlin/io/newm/server/features/distribution/eveara/EvearaDistributionRepositoryImpl.kt b/...in/kotlin/io/newm/server/features/distribution/eveara/EvearaDistributionRepositoryImpl.kt
@@ -18,7 +18,6 @@ import io.ktor.client.request.patch
 import io.ktor.client.request.post
 import io.ktor.client.request.put
 import io.ktor.client.request.setBody
-import io.ktor.client.statement.bodyAsChannel
 import io.ktor.client.statement.bodyAsText
 import io.ktor.http.ContentType
 import io.ktor.http.Headers
@@ -27,11 +26,12 @@ import io.ktor.http.contentType
 import io.ktor.http.isSuccess
 import io.ktor.http.quote
 import io.ktor.server.application.ApplicationEnvironment
+import io.ktor.util.InternalAPI
+import io.ktor.util.cio.writeChannel
+import io.ktor.utils.io.copyAndClose
 import io.ktor.utils.io.core.isNotEmpty
 import io.ktor.utils.io.core.toByteArray
-import io.ktor.utils.io.core.use
 import io.ktor.utils.io.streams.asInput
-import io.ktor.utils.io.streams.writePacket
 import io.newm.chain.util.toB64String
 import io.newm.server.config.repo.ConfigRepository
 import io.newm.server.config.repo.ConfigRepository.Companion.CONFIG_KEY_EVEARA_CLIENT_ID
@@ -1961,6 +1961,7 @@ class EvearaDistributionRepositoryImpl(
     /**
      * Upload and add metadata to the distribution track
      */
+    @OptIn(InternalAPI::class)
     private suspend fun createDistributionTrack(
         user: User,
         song: Song
@@ -1993,17 +1994,12 @@ class EvearaDistributionRepositoryImpl(
 
             val trackFile = withContext(Dispatchers.IO) {
                 val tempTrackFile = File.createTempFile("newm_track_", url.getFileNameWithExtensionFromUrl())
-                tempTrackFile.outputStream().use { trackFileStream ->
-                    val channel = audioFileResponse.bodyAsChannel()
-                    while (!channel.isClosedForRead) {
-                        val packet = channel.readRemaining(DEFAULT_BUFFER_SIZE.toLong())
-                        if (packet.isNotEmpty) {
-                            trackFileStream.writePacket(packet)
-                        }
-                    }
-                }
+                log.info { "Save track to temp file: ${tempTrackFile.absolutePath}" }
+                // Use internal api .content to avoid using any potential interceptors on the http call.
+                audioFileResponse.content.copyAndClose(tempTrackFile.writeChannel())
                 tempTrackFile
             }
+
             log.info { "Downloaded track ${mutableSong.title} to ${trackFile.absolutePath} having size ${trackFile.length()}" }
             val response = addTrack(user, trackFile)
             log.info { "Created distribution track ${mutableSong.title} with track_id ${response.trackId}: ${response.message}" }

diff --git a/...st/kotlin/io/newm/server/features/distribution/eveara/EvearaDistributionRepositoryTest.kt b/...st/kotlin/io/newm/server/features/distribution/eveara/EvearaDistributionRepositoryTest.kt
@@ -3,7 +3,10 @@ package io.newm.server.features.distribution.eveara
 import ch.qos.logback.classic.Level
 import ch.qos.logback.classic.Logger
 import com.google.common.truth.Truth.assertThat
-import io.ktor.server.application.*
+import io.ktor.server.application.ApplicationEnvironment
+import io.ktor.util.cio.readChannel
+import io.ktor.util.cio.writeChannel
+import io.ktor.utils.io.copyAndClose
 import io.mockk.mockk
 import io.newm.server.BaseApplicationTests
 import io.newm.server.config.repo.ConfigRepository
@@ -29,16 +32,17 @@ import io.newm.server.ktx.toBucketAndKey
 import io.newm.server.typealiases.SongId
 import io.newm.shared.koin.inject
 import io.newm.shared.ktx.info
+import java.io.File
+import java.time.LocalDate
+import java.time.LocalDateTime
+import java.util.UUID
 import kotlinx.coroutines.runBlocking
 import org.jetbrains.exposed.sql.deleteAll
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.slf4j.LoggerFactory
-import java.time.LocalDate
-import java.time.LocalDateTime
-import java.util.*
 
 class EvearaDistributionRepositoryTest : BaseApplicationTests() {
     @BeforeEach
@@ -346,4 +350,17 @@ class EvearaDistributionRepositoryTest : BaseApplicationTests() {
         assertThat(key).isEqualTo("cabb57b8-89f0-476f-8e95-e5c7a7d992c6/Vibrate.flac")
         assertThat(s3Url.toAudioContentType()).isEqualTo("audio/x-flac")
     }
+
+    @Test
+    @Disabled
+    fun `test large file download`() =
+        runBlocking {
+            val inputFile = File("/home/westbam/Downloads/sparkman.wav")
+            val outputFile = File("/home/westbam/Downloads/sparkman2.wav")
+            val channel = inputFile.readChannel()
+
+            channel.copyAndClose(outputFile.writeChannel())
+
+            assertThat(outputFile.length()).isEqualTo(inputFile.length())
+        }
 }