Skip to content

Commit

Permalink
Async downloader/uploader progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@krystian-panek-vmltech
krystian-panek-vmltech committed May 12, 2021
1 parent c42d2fe commit 1e88ff4
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 50 deletions.
42 changes: 17 additions & 25 deletions src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileDownloader.kt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.cognifide.gradle.common.file.transfer

import com.cognifide.gradle.common.build.ProgressLogger
import com.cognifide.gradle.common.utils.Formats
import com.cognifide.gradle.common.CommonExtension
import java.io.File
Expand All @@ -11,14 +10,15 @@ class FileDownloader(private val common: CommonExtension) {

private var processedBytes: Long = 0

private var loggedKb: Long = 0

private var startTime: Long = -1

var size: Long = 0

var chunkSize: Int = common.prop.int("fileTransfer.downloader.chunkSize") ?: CHUNK_SIZE

fun download(input: InputStream, target: File) {
common.progressLogger {
common.progress {
updater { update(currentProgress(target)) }
input.use { inputStream ->
target.parentFile.mkdirs()
startTime = System.currentTimeMillis()
Expand All @@ -27,12 +27,12 @@ class FileDownloader(private val common: CommonExtension) {
var finished = false

try {
val buf = ByteArray(TRANSFER_CHUNK_100_KB)
val buf = ByteArray(chunkSize)
var read = inputStream.read(buf)

while (read >= 0) {
output.write(buf, 0, read)
logProgress("Downloading", read.toLong(), target)
processedBytes += read.toLong()
read = inputStream.read(buf)
}

Expand All @@ -48,36 +48,28 @@ class FileDownloader(private val common: CommonExtension) {
}
}

private fun ProgressLogger.logProgress(operation: String, readBytes: Long, file: File) {
processedBytes += readBytes

val processedKb = processedBytes / KILOBYTE
if (processedKb > loggedKb) {
val fileName = file.name.removeSuffix(FileTransferManager.TMP_SUFFIX)
val msg = if (size > 0) {
"$operation: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" +
" (${Formats.percent(processedBytes, size)}," +
" time left: ${Formats.duration(remainingTime())})"
} else {
"$operation: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}"
}

progress(msg)

loggedKb = processedKb
private fun currentProgress(file: File): String {
val fileName = file.name.removeSuffix(FileTransferManager.TMP_SUFFIX)
return if (size > 0) {
"Downloading: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" +
" (${Formats.percent(processedBytes, size)}," +
" time left: ${Formats.duration(remainingTime())})"
} else {
"Downloading: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}"
}
}

private fun remainingTime(): Long {
if (processedBytes == 0L) return 0
val elapsedTime = System.currentTimeMillis() - startTime
val allTime = (elapsedTime * size / processedBytes)

return (allTime - elapsedTime).coerceAtLeast(0L)
}

companion object {
const val TRANSFER_CHUNK_100_KB = 100 * 1024

const val KILOBYTE = 1024

const val CHUNK_SIZE = 512 * KILOBYTE
}
}
9 changes: 8 additions & 1 deletion src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileTransferManager.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import com.cognifide.gradle.common.file.transfer.http.HttpFileTransfer
import com.cognifide.gradle.common.file.transfer.resolve.ResolveFileTransfer
import com.cognifide.gradle.common.file.transfer.sftp.SftpFileTransfer
import com.cognifide.gradle.common.file.transfer.smb.SmbFileTransfer
import com.cognifide.gradle.common.utils.Formats
import com.cognifide.gradle.common.utils.using
import java.io.File

Expand Down Expand Up @@ -128,7 +129,7 @@ class FileTransferManager(private val common: CommonExtension) : FileTransfer {
*/
fun downloadUsing(transfer: FileTransfer, dirUrl: String, fileName: String, target: File) {
if (target.exists()) {
common.logger.info("Skipping downloading file from URL '$dirUrl/$fileName' to '$target' as of it already exists.")
logger.info("Skipping downloading file from URL '$dirUrl/$fileName' to '$target' as of it already exists.")
return
}

Expand All @@ -139,8 +140,11 @@ class FileTransferManager(private val common: CommonExtension) : FileTransfer {
tmp.delete()
}

val started = System.currentTimeMillis()
transfer.downloadFrom(dirUrl, fileName, tmp)
tmp.renameTo(target)

logger.info("Downloaded file from URL '$dirUrl/$fileName' to '$target' in ${Formats.durationWordsSince(started)}")
}

/**
Expand Down Expand Up @@ -171,7 +175,10 @@ class FileTransferManager(private val common: CommonExtension) : FileTransfer {
logger.debug("Cannot check status of uploaded file at URL '$fileUrl'", e)
}

val started = System.currentTimeMillis()
transfer.uploadTo(dirUrl, fileName, source)

logger.info("Uploaded file from '$source' to URL '$dirUrl/$fileName' in ${Formats.durationWordsSince(started)}")
}

/**
Expand Down
41 changes: 17 additions & 24 deletions src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileUploader.kt
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.cognifide.gradle.common.file.transfer

import com.cognifide.gradle.common.CommonExtension
import com.cognifide.gradle.common.build.ProgressLogger
import com.cognifide.gradle.common.utils.Formats
import java.io.File
import java.io.OutputStream
Expand All @@ -10,27 +9,29 @@ class FileUploader(private val common: CommonExtension) {

private var processedBytes: Long = 0

private var loggedKb: Long = 0

private var size: Long = 0

private var startTime: Long = -1

var chunkSize: Int = common.prop.int("fileTransfer.uploader.chunkSize") ?: CHUNK_SIZE

fun upload(file: File, output: OutputStream, cleanup: (File) -> Unit = {}) {
common.progressLogger {
common.progress {
updater { update(currentProgress(file)) }

file.inputStream().use { input ->
size = file.length()
startTime = System.currentTimeMillis()

var finished = false

try {
val buf = ByteArray(TRANSFER_CHUNK_100_KB)
val buf = ByteArray(chunkSize)
var read = input.read(buf)

while (read >= 0) {
output.write(buf, 0, read)
logProgress("Uploading", read.toLong(), file)
processedBytes += read.toLong()
read = input.read(buf)
}

Expand All @@ -46,35 +47,27 @@ class FileUploader(private val common: CommonExtension) {
}
}

private fun ProgressLogger.logProgress(operation: String, readBytes: Long, file: File) {
processedBytes += readBytes

val processedKb = processedBytes / KILOBYTE
if (processedKb > loggedKb) {
val msg = if (size > 0) {
"$operation: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" +
" (${Formats.percent(processedBytes, size)}," +
" time left: ${Formats.duration(remainingTime())})"
} else {
"$operation: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}"
}

progress(msg)

loggedKb = processedKb
private fun currentProgress(file: File): String {
return if (size > 0) {
"Uploading: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" +
" (${Formats.percent(processedBytes, size)}," +
" time left: ${Formats.duration(remainingTime())})"
} else {
"Uploading: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}"
}
}

private fun remainingTime(): Long {
if (processedBytes == 0L) return 0
val elapsedTime = System.currentTimeMillis() - startTime
val allTime = (elapsedTime * size / processedBytes)

return (allTime - elapsedTime).coerceAtLeast(0L)
}

companion object {
const val TRANSFER_CHUNK_100_KB = 100 * 1024

const val KILOBYTE = 1024

const val CHUNK_SIZE = 512 * KILOBYTE
}
}
4 changes: 4 additions & 0 deletions src/main/kotlin/com/cognifide/gradle/common/utils/Formats.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,12 @@ object Formats {

fun duration(millis: Long): String = DurationFormatUtils.formatDuration(millis.coerceAtLeast(0L), "mm:ss")

fun durationWords(millis: Long): String = DurationFormatUtils.formatDurationWords(millis.coerceAtLeast(0L), true, true)

fun durationSince(millis: Long) = duration(System.currentTimeMillis() - millis)

fun durationWordsSince(millis: Long) = durationWords(System.currentTimeMillis() - millis)

fun durationFit(thenMillis: Long, thenZoneId: ZoneId, durationMillis: Long): Boolean {
val nowTimestamp = LocalDateTime.now().atZone(ZoneId.systemDefault())
val thenTimestamp = localDateTimeAt(thenMillis, thenZoneId)
Expand Down

0 comments on commit 1e88ff4

Please sign in to comment.