Skip to content

Commit 3b60f9b

Browse files
bayangbayang
committed
fix: csv import tests and bugfixes
1 parent 4f7cf07 commit 3b60f9b

File tree

11 files changed

+160
-5
lines changed

11 files changed

+160
-5
lines changed

‎src/main/kotlin/io/github/bayang/jelu/controllers/ImportController.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package io.github.bayang.jelu.controllers
33
import io.github.bayang.jelu.config.JeluProperties
44
import io.github.bayang.jelu.dto.ImportConfigurationDto
55
import io.github.bayang.jelu.dto.JeluUser
6-
import io.github.bayang.jelu.service.import.CsvImportService
6+
import io.github.bayang.jelu.service.imports.CsvImportService
77
import io.swagger.v3.oas.annotations.Operation
88
import io.swagger.v3.oas.annotations.responses.ApiResponse
99
import mu.KotlinLogging

‎src/main/kotlin/io/github/bayang/jelu/dao/ImportRepository.kt

+11
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import io.github.bayang.jelu.utils.nowInstant
55
import mu.KotlinLogging
66
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
77
import org.jetbrains.exposed.sql.and
8+
import org.jetbrains.exposed.sql.deleteWhere
89
import org.jetbrains.exposed.sql.update
910
import org.springframework.stereotype.Repository
1011
import java.time.Instant
@@ -16,6 +17,16 @@ const val DEFAULT_BLOCK_SIZE: Int = 50
1617
@Repository
1718
class ImportRepository {
1819

20+
fun deleteByprocessingStatusAndUser(
21+
processingStatus: ProcessingStatus,
22+
userId: UUID
23+
): Int {
24+
// FIXME SQLite doesn't support LIMIT in DELETE clause., dialect: sqlite.
25+
return ImportEntityTable.deleteWhere {
26+
ImportEntityTable.processingStatus eq processingStatus and (ImportEntityTable.userId eq userId)
27+
}
28+
}
29+
1930
fun getByprocessingStatusAndUser(
2031
processingStatus: ProcessingStatus,
2132
userId: UUID

‎src/main/kotlin/io/github/bayang/jelu/service/BookService.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ class BookService(
193193
slugify(title),
194194
id, FilenameUtils.getExtension(dtoImage)
195195
)
196-
val currentFile = File(targetDir, dtoImage)
196+
val currentFile = File(targetDir, "$dtoImage.bak")
197197
val targetFile = File(currentFile.parent, targetFilename)
198198
val succeeded = currentFile.renameTo(targetFile)
199199
logger.debug { "renaming of metadata imported file $dtoImage was successful: $succeeded" }

‎src/main/kotlin/io/github/bayang/jelu/service/ImportService.kt

+6
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,10 @@ class ImportService(private val importRepository: ImportRepository) {
4242
processingStatus: ProcessingStatus,
4343
userId: UUID
4444
): Long = importRepository.countByprocessingStatusAndUser(processingStatus, userId)
45+
46+
@Transactional
47+
fun deleteByprocessingStatusAndUser(
48+
processingStatus: ProcessingStatus,
49+
userId: UUID
50+
): Int = importRepository.deleteByprocessingStatusAndUser(processingStatus, userId)
4551
}

‎src/main/kotlin/io/github/bayang/jelu/service/import/CsvImportService.kt ‎src/main/kotlin/io/github/bayang/jelu/service/imports/CsvImportService.kt

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package io.github.bayang.jelu.service.import
1+
package io.github.bayang.jelu.service.imports
22

33
import io.github.bayang.jelu.config.JeluProperties
44
import io.github.bayang.jelu.dao.ImportEntity
@@ -228,6 +228,7 @@ class CsvImportService(
228228
) {
229229
remainingToSave ++
230230
}
231+
// bookService.findUserBookById(savedUserBook.id)
231232
val nbAlreadyFinishedEvents = if (savedUserBook.readingEvents != null) savedUserBook.readingEvents.stream().filter { it.eventType == ReadingEventType.FINISHED }.count().toInt() else 0
232233
remainingToSave -= nbAlreadyFinishedEvents
233234
// we know the book has been read 3 times for example but don't have dates

‎src/test/kotlin/io/github/bayang/jelu/TestHelpers.kt

+6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package io.github.bayang.jelu
22

3+
import io.github.bayang.jelu.dao.ImportSource
34
import io.github.bayang.jelu.dao.ReadingEventType
45
import io.github.bayang.jelu.dto.AuthorDto
56
import io.github.bayang.jelu.dto.BookCreateDto
67
import io.github.bayang.jelu.dto.CreateUserBookDto
8+
import io.github.bayang.jelu.dto.ImportConfigurationDto
79
import io.github.bayang.jelu.dto.TagDto
810
import java.time.Instant
911

@@ -56,3 +58,7 @@ fun tags(): List<TagDto> {
5658
fun tagDto(name: String = "science fiction"): TagDto {
5759
return TagDto(id = null, creationDate = null, modificationDate = null, name)
5860
}
61+
62+
fun importConfigurationDto(): ImportConfigurationDto {
63+
return ImportConfigurationDto(shouldFetchMetadata = false, shouldFetchCovers = false, ImportSource.GOODREADS)
64+
}

‎src/test/kotlin/io/github/bayang/jelu/service/BookServiceTest.kt

-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ class BookServiceTest(
4949
fun setupUser() {
5050
userService.save(CreateUserDto(login = "testuser", password = "1234", isAdmin = true))
5151
jeluProperties.files.images = tempDir.absolutePath
52-
println(jeluProperties.files.images)
5352
}
5453

5554
@AfterAll

‎src/test/kotlin/io/github/bayang/jelu/service/ReadingEventServiceTest.kt

-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ class ReadingEventServiceTest(
4646
fun setupUser() {
4747
userService.save(CreateUserDto(login = "testuser", password = "1234", isAdmin = true))
4848
jeluProperties.files.images = tempDir.absolutePath
49-
println(jeluProperties.files.images)
5049
}
5150

5251
@AfterAll

‎src/test/kotlin/io/github/bayang/jelu/service/imports/CsvImportServiceTest.kt

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
package io.github.bayang.jelu.service.imports
2+
3+
import io.github.bayang.jelu.config.JeluProperties
4+
import io.github.bayang.jelu.dao.ProcessingStatus
5+
import io.github.bayang.jelu.dao.User
6+
import io.github.bayang.jelu.dto.CreateUserDto
7+
import io.github.bayang.jelu.dto.JeluUser
8+
import io.github.bayang.jelu.dto.UserBookWithoutEventsAndUserDto
9+
import io.github.bayang.jelu.importConfigurationDto
10+
import io.github.bayang.jelu.service.BookService
11+
import io.github.bayang.jelu.service.ImportService
12+
import io.github.bayang.jelu.service.ReadingEventService
13+
import io.github.bayang.jelu.service.UserService
14+
import org.junit.jupiter.api.AfterAll
15+
import org.junit.jupiter.api.AfterEach
16+
import org.junit.jupiter.api.Assertions
17+
import org.junit.jupiter.api.BeforeAll
18+
import org.junit.jupiter.api.Test
19+
import org.junit.jupiter.api.TestInstance
20+
import org.junit.jupiter.api.io.TempDir
21+
import org.springframework.beans.factory.annotation.Autowired
22+
import org.springframework.boot.test.context.SpringBootTest
23+
import org.springframework.data.domain.Page
24+
import org.springframework.data.domain.Pageable
25+
import java.io.File
26+
27+
@SpringBootTest
28+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
29+
class CsvImportServiceTest(
30+
@Autowired private val bookService: BookService,
31+
@Autowired private val userService: UserService,
32+
@Autowired private val jeluProperties: JeluProperties,
33+
@Autowired private val readingEventService: ReadingEventService,
34+
@Autowired private val csvImportService: CsvImportService,
35+
@Autowired private val importService: ImportService
36+
) {
37+
38+
companion object {
39+
@TempDir
40+
lateinit var tempDir: File
41+
}
42+
43+
@BeforeAll
44+
fun setupUser() {
45+
userService.save(CreateUserDto(login = "testuser", password = "1234", isAdmin = true))
46+
jeluProperties.files.imports = tempDir.absolutePath
47+
}
48+
49+
@AfterAll
50+
fun teardDown() {
51+
userService.findAll(null).forEach { userService.deleteUser(it.id!!) }
52+
}
53+
54+
@AfterEach
55+
fun cleanTest() {
56+
tempDir.listFiles().forEach {
57+
it.deleteRecursively()
58+
}
59+
importService.deleteByprocessingStatusAndUser(ProcessingStatus.SAVED, user().id.value)
60+
readingEventService.findAll(null, null, Pageable.ofSize(30)).content.forEach {
61+
readingEventService.deleteReadingEventById(it.id!!)
62+
}
63+
bookService.findUserBookByCriteria(user().id.value, null, null, Pageable.ofSize(30))
64+
.forEach { bookService.deleteUserBookById(it.id!!) }
65+
bookService.findAllAuthors(null, Pageable.ofSize(30)).forEach {
66+
bookService.deleteAuthorById(it.id!!)
67+
}
68+
}
69+
70+
@Test
71+
fun testParse() {
72+
val userId = user().id.value
73+
val csv = File(this::class.java.getResource("/csv-import/goodreads1.csv").file)
74+
csvImportService.parse(csv, userId, importConfigurationDto())
75+
val nb = importService.countByprocessingStatusAndUser(ProcessingStatus.SAVED, userId)
76+
Assertions.assertEquals(10, nb)
77+
val dtos = importService.getByprocessingStatusAndUser(ProcessingStatus.SAVED, userId)
78+
dtos.forEach {
79+
if (it.title.equals("La somme de nos folies")) {
80+
Assertions.assertEquals("9782843048302", it.isbn13)
81+
Assertions.assertEquals("Éditions Zulma", it.publisher)
82+
val shelves = it.tags?.split(",")
83+
Assertions.assertEquals(3, shelves?.size)
84+
}
85+
}
86+
}
87+
88+
@Test
89+
fun testNoDuplicates() {
90+
val userId = user().id.value
91+
val csv = File(this::class.java.getResource("/csv-import/goodreads-duplicate-events.csv").file)
92+
csvImportService.parse(csv, userId, importConfigurationDto())
93+
val nb = importService.countByprocessingStatusAndUser(ProcessingStatus.SAVED, userId)
94+
Assertions.assertEquals(5, nb)
95+
val dtos = importService.getByprocessingStatusAndUser(ProcessingStatus.SAVED, userId)
96+
97+
val (success, failures) = csvImportService.importFromDb(userId, importConfigurationDto())
98+
Assertions.assertEquals(5, success)
99+
Assertions.assertEquals(0, failures)
100+
val userbooksPage: Page<UserBookWithoutEventsAndUserDto> = bookService.findUserBookByCriteria(userId, null, null, Pageable.ofSize(30))
101+
userbooksPage.content.forEach {
102+
val userbook = bookService.findUserBookById(it.id!!)
103+
if (userbook.book.title == "The Gulag Archipelago 1918–1956 (Abridged)") {
104+
Assertions.assertEquals(2, userbook.readingEvents?.size)
105+
} else {
106+
Assertions.assertEquals(1, userbook.readingEvents?.size)
107+
}
108+
}
109+
}
110+
111+
fun user(): User {
112+
val userDetail = userService.loadUserByUsername("testuser")
113+
return (userDetail as JeluUser).user
114+
}
115+
}

‎src/test/resources/csv-import/goodreads-duplicate-events.csv

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID
2+
70561,The Gulag Archipelago 1918–1956 (Abridged),Aleksandr Solzhenitsyn,"Solzhenitsyn, Aleksandr",Edward E. Ericson Jr.,"=""0060007761""","=""9780060007768""",0,4.31,HarperCollins,Paperback,472,2002,1973,,2019/01/29,currently-reading,currently-reading (#1),currently-reading,,,,1,,,0,,,,,
3+
30659,Meditations,Marcus Aurelius,"Aurelius, Marcus","Martin Hammond, Albert Wittstock, عادل مصطفى, Simone Mooij-Valk, Diskin Clay","=""0140449337""","=""9780140449334""",5,4.24,Penguin Books,Paperback,303,2006,180,2020/11/15,2020/06/25,,,read,,,,1,,,0,,,,,
4+
51893,Thus Spoke Zarathustra,Friedrich Nietzsche,"Nietzsche, Friedrich",Walter Kaufmann,"=""""","=""""",0,4.06,Penguin Books,Paperback,327,1978,1883,,2020/02/25,to-read,to-read (#42),to-read,,,,0,,,0,,,,,
5+
511228,Cowl,Neal Asher,"Asher, Neal",,"=""0765352796""","=""9780765352798""",3,3.78,Tor Science Fiction,Paperback,432,2006,2004,2020/02/23,2019/11/29,,,read,,,,1,,,0,,,,,
6+
28335698,"Tiamat's Wrath (The Expanse, #8)",James S.A. Corey,"Corey, James S.A.",,"=""0316332879""","=""9780316332873""",4,4.57,Orbit Books,Hardcover,534,2019,2019,,2019/02/13,,,read,,,,1,,,0,,,,,
7+
35606041,"A Little Hatred (The Age of Madness, #1)",Joe Abercrombie,"Abercrombie, Joe",,"=""031618716X""","=""9780316187169""",4,4.45,Orbit,Hardcover,480,2019,2019,2020/02/09,2020/01/31,,,read,,,,1,,,0,,,,,

‎src/test/resources/csv-import/goodreads1.csv

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID
2+
40725766,La somme de nos folies,Shih-Li Kow,"Kow, Shih-Li",,"=""""","=""9782843048302""",0,3.89,Éditions Zulma,Paperback,384,2018,2014,,2021/04/02,"to-read, test-aad, avec-espace","to-read (#80), test-aad (#1), avec-espace (#1)",to-read,,,,0,,,0,,,,,
3+
33178099,La vie des arbres,Francis Hallé,"Hallé, Francis",,"=""2227483148""","=""9782227483149""",0,4.12,Bayard Culture,Paperback,70,2011,,2020/11/11,2021/01/10,,,read,,,,1,,,0,,,,,21
4+
55819582,La Fabrique des lendemains,Rich Larson,"Larson, Rich",Pierre-Paul Duranstanti,"=""2843449731""","=""9782843449734""",0,4.16,Le Bélial',Paperback,612,2020,2018,,2020/11/09,to-read,to-read (#74),to-read,,,,0,,,0,,,,,
5+
17876040,"Le Crystal des Elfes bleus (Elfes, #1)",Jean-Luc Istin,"Istin, Jean-Luc","Kyko Duarte, Diogo Saito","=""2302027191""","=""9782302027190""",0,3.86,Soleil,Hardcover,54,2013,2013,2021/04/25,2021/03/15,,,read,,,,1,,,0,,,,,
6+
2189436,Quartier lointain,Jirō Taniguchi,"Taniguchi, Jirō",谷口 ジロー,"=""220339644X""","=""9782203396449""",0,4.38,Casterman,Hardcover,405,2006,1998,,2019/03/21,waiting,waiting (#49),waiting,,,,0,,,0,,,,,
7+
10576969,Un Automne à Hanoï,Clément Baloup,"Baloup, Clément",,"=""2849530085""","=""9782849530085""",0,3.00,La Boîte à bulles,Hardcover,48,2004,2004,2021/04/25,2021/04/25,,,read,,,,1,,,0,,,,,
8+
2320102,Le Secret Du Chant Des Baleines,Christopher Moore,"Moore, Christopher",,"=""2070316572""","=""9782070316571""",0,3.75,Gallimard,,414,2006,2003,2021/04/21,2021/04/24,,,read,,,,1,,,0,,,,,
9+
4268810,Un Sale Boulot,Christopher Moore,"Moore, Christopher",Luc Baranger,"=""270213839X""","=""9782702138397""",0,4.01,Calmann-Lévy,Paperback,405,2007,2006,,2021/04/24,currently-reading,currently-reading (#2),currently-reading,,,,1,,,0,,,,,
10+
19246484,"Clothes, Clothes, Clothes. Music, Music, Music. Boys, Boys, Boys.",Viv Albertine,"Albertine, Viv",,"=""0571297773""","=""9780571297771""",0,4.27,Faber & Faber,ebook,,2014,2014,2021/03/21,2021/03/13,,,read,,,,1,,,0,,,,,
11+
17181203,Tout Corum,Michael Moorcock,"Moorcock, Michael",,"=""284172090X""","=""9782841720903""",5,4.03,L'Atalante,Paperback,862,1998,1998,,2018/10/24,,,read,,,,1,,,0,,,,,

0 commit comments

Comments
 (0)