Skip to content

Commit

Permalink
First attempt/experiment for #4 with some refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
@dacr
dacr committed Aug 4, 2024
1 parent 078ce68 commit a7c2dea
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 117 deletions.
37 changes: 19 additions & 18 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,18 @@ val versions = new {
}

lazy val deepJavaLearningLibs = Seq(
"ai.djl" % "api" % versions.djl,
"ai.djl" % "basicdataset" % versions.djl,
"ai.djl" % "model-zoo" % versions.djl,
"ai.djl.huggingface" % "tokenizers" % versions.djl,
"ai.djl.mxnet" % "mxnet-engine" % versions.djl,
"ai.djl.mxnet" % "mxnet-model-zoo" % versions.djl,
"ai.djl.pytorch" % "pytorch-engine" % versions.djl,
"ai.djl.pytorch" % "pytorch-model-zoo" % versions.djl,
"ai.djl.tensorflow" % "tensorflow-engine" % versions.djl,
"ai.djl.tensorflow" % "tensorflow-model-zoo" % versions.djl,
"ai.djl.onnxruntime" % "onnxruntime-engine" % versions.djl,
"net.java.dev.jna" % "jna" % "5.14.0"
"ai.djl" % "api" % versions.djl,
"ai.djl" % "basicdataset" % versions.djl,
"ai.djl" % "model-zoo" % versions.djl,
"ai.djl.huggingface" % "tokenizers" % versions.djl,
"ai.djl.mxnet" % "mxnet-engine" % versions.djl,
"ai.djl.mxnet" % "mxnet-model-zoo" % versions.djl,
"ai.djl.pytorch" % "pytorch-engine" % versions.djl,
"ai.djl.pytorch" % "pytorch-model-zoo" % versions.djl,
"ai.djl.tensorflow" % "tensorflow-engine" % versions.djl,
"ai.djl.tensorflow" % "tensorflow-model-zoo" % versions.djl,
"ai.djl.onnxruntime" % "onnxruntime-engine" % versions.djl,
"net.java.dev.jna" % "jna" % "5.14.0"
)

lazy val lmdbJavaOptions = Seq(
Expand Down Expand Up @@ -153,13 +153,14 @@ lazy val userInterfacesCLI =
fork := true,
javaOptions ++= lmdbJavaOptions,
libraryDependencies ++= Seq(
"dev.zio" %% "zio" % versions.zio,
"dev.zio" %% "zio-config" % versions.zioconfig,
"dev.zio" %% "zio-config-typesafe" % versions.zioconfig,
"dev.zio" %% "zio-config-magnolia" % versions.zioconfig,
"dev.zio" %% "zio-logging" % versions.ziologging, // Temporary
"dev.zio" %% "zio-logging-slf4j2-bridge" % versions.ziologging // Temporary
"dev.zio" %% "zio" % versions.zio,
"dev.zio" %% "zio-config" % versions.zioconfig,
"dev.zio" %% "zio-config-typesafe" % versions.zioconfig,
"dev.zio" %% "zio-config-magnolia" % versions.zioconfig,
"dev.zio" %% "zio-logging" % versions.ziologging, // Temporary
"dev.zio" %% "zio-logging-slf4j2-bridge" % versions.ziologging, // Temporary
// "ch.qos.logback" % "logback-classic" % "1.4.11" // Temporary
"com.github.haifengl" % "smile-core" % "3.1.1" // Temporary for quick&dirty evaluation of the DBSCAN clustering algo
)
)

Expand Down
2 changes: 2 additions & 0 deletions modules/core/src/main/scala/fr/janalyse/sotohp/store/PhotoStoreService.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ trait PhotoStoreService {
def photoFacesDelete(photoId: PhotoId): IO[PhotoStoreIssue, Unit]

// photo face features collection
def photoFaceFeaturesStream(): Stream[PhotoStoreIssue, (FaceId, FaceFeatures)]
def photoFaceFeaturesGet(faceId: FaceId): IO[PhotoStoreIssue, Option[FaceFeatures]]
def photoFaceFeaturesContains(faceId: FaceId): IO[PhotoStoreIssue, Boolean]
def photoFaceFeaturesUpsert(faceId: FaceId, faceFeatures: FaceFeatures): IO[PhotoStoreIssue, Unit]
Expand Down Expand Up @@ -156,6 +157,7 @@ object PhotoStoreService {
def photoFacesUpsert(photoId: PhotoId, metaData: PhotoFaces): ZIO[PhotoStoreService, PhotoStoreIssue, Unit] = serviceWithZIO(_.photoFacesUpsert(photoId, metaData))
def photoFacesDelete(photoId: PhotoId): ZIO[PhotoStoreService, PhotoStoreIssue, Unit] = serviceWithZIO(_.photoFacesDelete(photoId))

def photoFaceFeaturesStream(): ZStream[PhotoStoreService, PhotoStoreIssue, (FaceId, FaceFeatures)] = ZStream.serviceWithStream(_.photoFaceFeaturesStream())
def photoFaceFeaturesGet(faceId: FaceId): ZIO[PhotoStoreService, PhotoStoreIssue, Option[FaceFeatures]] = serviceWithZIO(_.photoFaceFeaturesGet(faceId))
def photoFaceFeaturesContains(faceId: FaceId): ZIO[PhotoStoreService, PhotoStoreIssue, Boolean] = serviceWithZIO(_.photoFaceFeaturesContains(faceId))
def photoFaceFeaturesUpsert(faceId: FaceId, faceFeatures: FaceFeatures): ZIO[PhotoStoreService, PhotoStoreIssue, Unit] = serviceWithZIO(_.photoFaceFeaturesUpsert(faceId, faceFeatures))
Expand Down
185 changes: 86 additions & 99 deletions modules/core/src/main/scala/fr/janalyse/sotohp/store/PhotoStoreServiceLive.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,19 +167,17 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, result => result.map((key, daoState) => daoStateToState(daoState)))

// ===================================================================================================================
def daoSourceToSource(from: Option[DaoPhotoSource]): Option[PhotoSource] = {
from.map(daoSource =>
PhotoSource(
photoId = PhotoId(ULID(daoSource.photoId)),
original = Original(
ownerId = PhotoOwnerId(ULID(daoSource.originalOwnerId)),
baseDirectory = Path.of(daoSource.originalBaseDirectory),
path = Path.of(daoSource.originalPath)
),
fileSize = daoSource.fileSize,
fileHash = PhotoHash(daoSource.fileHash),
fileLastModified = daoSource.fileLastModified
)
def daoSourceToSource(daoSource: DaoPhotoSource): PhotoSource = {
PhotoSource(
photoId = PhotoId(ULID(daoSource.photoId)),
original = Original(
ownerId = PhotoOwnerId(ULID(daoSource.originalOwnerId)),
baseDirectory = Path.of(daoSource.originalBaseDirectory),
path = Path.of(daoSource.originalPath)
),
fileSize = daoSource.fileSize,
fileHash = PhotoHash(daoSource.fileHash),
fileLastModified = daoSource.fileLastModified
)
}

Expand All @@ -198,7 +196,7 @@ class PhotoStoreServiceLive private (
override def photoSourceGet(originalId: OriginalId): IO[PhotoStoreIssue, Option[PhotoSource]] =
sourcesCollection
.fetch(originalIdToCollectionKey(originalId))
.mapBoth(convertFailures, daoSourceToSource)
.mapBoth(convertFailures, _.map(daoSourceToSource))

override def photoSourceContains(originalId: OriginalId): IO[PhotoStoreIssue, Boolean] =
sourcesCollection
Expand All @@ -216,15 +214,13 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, _ => ())

// ===================================================================================================================
def daoMetaDataToMetaData(from: Option[DaoPhotoMetaData]): Option[PhotoMetaData] = {
from.map(daoMetaData =>
PhotoMetaData(
dimension = daoMetaData.dimension.map(daoDim => Dimension2D(width = daoDim.width, height = daoDim.height)),
shootDateTime = daoMetaData.shootDateTime,
orientation = PhotoOrientation.values.find(orientation => daoMetaData.orientation.isDefined && orientation.code == daoMetaData.orientation.get), // TODO can be enhanced
cameraName = daoMetaData.cameraName,
tags = daoMetaData.tags
)
def daoMetaDataToMetaData(daoMetaData: DaoPhotoMetaData): PhotoMetaData = {
PhotoMetaData(
dimension = daoMetaData.dimension.map(daoDim => Dimension2D(width = daoDim.width, height = daoDim.height)),
shootDateTime = daoMetaData.shootDateTime,
orientation = PhotoOrientation.values.find(orientation => daoMetaData.orientation.isDefined && orientation.code == daoMetaData.orientation.get), // TODO can be enhanced
cameraName = daoMetaData.cameraName,
tags = daoMetaData.tags
)
}

Expand All @@ -241,7 +237,7 @@ class PhotoStoreServiceLive private (
override def photoMetaDataGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoMetaData]] =
metaDataCollection
.fetch(photoIdToCollectionKey(photoId))
.mapBoth(convertFailures, daoMetaDataToMetaData)
.mapBoth(convertFailures, _.map(daoMetaDataToMetaData))

override def photoMetaDataContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean] =
metaDataCollection
Expand All @@ -259,14 +255,12 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, _ => ())

// ===================================================================================================================
def daoPlaceToPlace(from: Option[DaoPhotoPlace]): Option[PhotoPlace] = {
from.map(daoPlace =>
PhotoPlace(
latitude = LatitudeDecimalDegrees(daoPlace.latitude),
longitude = LongitudeDecimalDegrees(daoPlace.longitude),
altitude = daoPlace.altitude,
deducted = daoPlace.deducted
)
def daoPlaceToPlace(daoPlace: DaoPhotoPlace): PhotoPlace = {
PhotoPlace(
latitude = LatitudeDecimalDegrees(daoPlace.latitude),
longitude = LongitudeDecimalDegrees(daoPlace.longitude),
altitude = daoPlace.altitude,
deducted = daoPlace.deducted
)
}

Expand All @@ -281,7 +275,7 @@ class PhotoStoreServiceLive private (
override def photoPlaceGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoPlace]] =
placesCollection
.fetch(photoIdToCollectionKey(photoId))
.mapBoth(convertFailures, daoPlaceToPlace)
.mapBoth(convertFailures, _.map(daoPlaceToPlace))

override def photoPlaceContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean] =
placesCollection
Expand All @@ -299,11 +293,9 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, _ => ())

// ===================================================================================================================
def daoMiniaturesToMiniatures(from: Option[DaoMiniatures]): Option[Miniatures] = {
from.map(daoMiniatures =>
Miniatures(
sources = daoMiniatures.sources.map(s => MiniatureSource(size = s.size, dimension = Dimension2D(width = s.dimension.width, height = s.dimension.height)))
)
def daoMiniaturesToMiniatures(daoMiniatures: DaoMiniatures): Miniatures = {
Miniatures(
sources = daoMiniatures.sources.map(s => MiniatureSource(size = s.size, dimension = Dimension2D(width = s.dimension.width, height = s.dimension.height)))
)
}

Expand All @@ -315,7 +307,7 @@ class PhotoStoreServiceLive private (
override def photoMiniaturesGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[Miniatures]] =
miniaturesCollection
.fetch(photoIdToCollectionKey(photoId))
.mapBoth(convertFailures, daoMiniaturesToMiniatures)
.mapBoth(convertFailures, _.map(daoMiniaturesToMiniatures))

override def photoMiniaturesContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean] =
miniaturesCollection
Expand All @@ -333,14 +325,12 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, _ => ())

// =================================================================================================================== // ===================================================================================================================
def daoNormalizedToNormalized(from: Option[DaoNormalizedPhoto]): Option[NormalizedPhoto] = {
from.map(daoNormalized =>
NormalizedPhoto(
size = daoNormalized.size,
dimension = Dimension2D(
width = daoNormalized.dimension.width,
height = daoNormalized.dimension.height
)
def daoNormalizedToNormalized(daoNormalized: DaoNormalizedPhoto): NormalizedPhoto = {
NormalizedPhoto(
size = daoNormalized.size,
dimension = Dimension2D(
width = daoNormalized.dimension.width,
height = daoNormalized.dimension.height
)
)
}
Expand All @@ -357,7 +347,7 @@ class PhotoStoreServiceLive private (
override def photoNormalizedGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[NormalizedPhoto]] =
normalizedCollection
.fetch(photoIdToCollectionKey(photoId))
.mapBoth(convertFailures, daoNormalizedToNormalized)
.mapBoth(convertFailures, _.map(daoNormalizedToNormalized))

override def photoNormalizedContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean] =
normalizedCollection
Expand All @@ -375,11 +365,9 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, _ => ())

// ===================================================================================================================
def daoClassificationsToClassifications(from: Option[DaoPhotoClassifications]): Option[PhotoClassifications] = {
from.map(daoClassifications =>
PhotoClassifications(
classifications = daoClassifications.classifications.map(that => DetectedClassification(that.name))
)
def daoClassificationsToClassifications(daoClassifications: DaoPhotoClassifications): PhotoClassifications = {
PhotoClassifications(
classifications = daoClassifications.classifications.map(that => DetectedClassification(that.name))
)
}

Expand All @@ -392,7 +380,7 @@ class PhotoStoreServiceLive private (
override def photoClassificationsGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoClassifications]] =
classificationsCollection
.fetch(photoIdToCollectionKey(photoId))
.mapBoth(convertFailures, daoClassificationsToClassifications)
.mapBoth(convertFailures, _.map(daoClassificationsToClassifications))

override def photoClassificationsContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean] =
classificationsCollection
Expand All @@ -410,18 +398,16 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, _ => ())

// ===================================================================================================================
def daoObjectsToObjects(from: Option[DaoPhotoObjects]): Option[PhotoObjects] = {
from.map(daoObjects =>
PhotoObjects(
objects = daoObjects.objects.map(that =>
DetectedObject(
name = that.name,
box = BoundingBox(
x = that.box.x,
y = that.box.y,
width = that.box.width,
height = that.box.height
)
def daoObjectsToObjects(daoObjects: DaoPhotoObjects): PhotoObjects = {
PhotoObjects(
objects = daoObjects.objects.map(that =>
DetectedObject(
name = that.name,
box = BoundingBox(
x = that.box.x,
y = that.box.y,
width = that.box.width,
height = that.box.height
)
)
)
Expand All @@ -447,7 +433,7 @@ class PhotoStoreServiceLive private (
override def photoObjectsGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoObjects]] =
objectsCollection
.fetch(photoIdToCollectionKey(photoId))
.mapBoth(convertFailures, daoObjectsToObjects)
.mapBoth(convertFailures, _.map(daoObjectsToObjects))

override def photoObjectsContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean] =
objectsCollection
Expand All @@ -465,21 +451,19 @@ class PhotoStoreServiceLive private (
.mapBoth(convertFailures, _ => ())

// ===================================================================================================================
def daoFacesToFaces(from: Option[DaoPhotoFaces]): Option[PhotoFaces] = {
from.map(daoFaces =>
PhotoFaces(
count = daoFaces.count,
faces = daoFaces.faces.map(that =>
DetectedFace(
someoneId = that.someoneId.map(id => SomeoneId(ULID.fromString(id))),
box = BoundingBox(
x = that.box.x,
y = that.box.y,
width = that.box.width,
height = that.box.height
),
faceId = FaceId(ULID.fromString(that.faceId))
)
def daoFacesToFaces(daoFaces: DaoPhotoFaces): PhotoFaces = {
PhotoFaces(
count = daoFaces.count,
faces = daoFaces.faces.map(that =>
DetectedFace(
someoneId = that.someoneId.map(id => SomeoneId(ULID.fromString(id))),
box = BoundingBox(
x = that.box.x,
y = that.box.y,
width = that.box.width,
height = that.box.height
),
faceId = FaceId(ULID.fromString(that.faceId))
)
)
)
Expand All @@ -506,7 +490,7 @@ class PhotoStoreServiceLive private (
override def photoFacesGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoFaces]] =
facesCollection
.fetch(photoIdToCollectionKey(photoId))
.mapBoth(convertFailures, daoFacesToFaces)
.mapBoth(convertFailures, _.map(daoFacesToFaces))

override def photoFacesContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean] =
facesCollection
Expand All @@ -525,23 +509,21 @@ class PhotoStoreServiceLive private (

// ===================================================================================================================

def daoFaceFeaturesToFaces(from: Option[DaoFaceFeatures]): Option[FaceFeatures] = {
from.map(daoFaceFeatures =>
FaceFeatures(
photoId = PhotoId(ULID.fromString(daoFaceFeatures.photoId)),
someoneId = daoFaceFeatures.someoneId.map(id => SomeoneId(ULID.fromString(id))),
box = BoundingBox(
x = daoFaceFeatures.box.x,
y = daoFaceFeatures.box.y,
width = daoFaceFeatures.box.width,
height = daoFaceFeatures.box.height
),
features = daoFaceFeatures.features
)
def daoFaceFeaturesToFaceFeatures(daoFaceFeatures: DaoFaceFeatures): FaceFeatures = {
FaceFeatures(
photoId = PhotoId(ULID.fromString(daoFaceFeatures.photoId)),
someoneId = daoFaceFeatures.someoneId.map(id => SomeoneId(ULID.fromString(id))),
box = BoundingBox(
x = daoFaceFeatures.box.x,
y = daoFaceFeatures.box.y,
width = daoFaceFeatures.box.width,
height = daoFaceFeatures.box.height
),
features = daoFaceFeatures.features
)
}

def faceFeaturesToDaoFaces(from: FaceFeatures): DaoFaceFeatures = {
def faceFeaturesToDaoFaceFeatures(from: FaceFeatures): DaoFaceFeatures = {
DaoFaceFeatures(
photoId = from.photoId.id.toString(),
someoneId = from.someoneId.map(_.toString),
Expand All @@ -555,10 +537,15 @@ class PhotoStoreServiceLive private (
)
}

override def photoFaceFeaturesStream(): ZStream[Any, PhotoStoreIssue, (FaceId, FaceFeatures)] =
faceFeaturesCollection
.streamWithKeys()
.mapBoth(convertFailures, (key, value) => (FaceId(ULID.fromString(key)), daoFaceFeaturesToFaceFeatures(value)))

def photoFaceFeaturesGet(faceId: FaceId): IO[PhotoStoreIssue, Option[FaceFeatures]] =
faceFeaturesCollection
.fetch(faceIdToCollectionKey(faceId))
.mapBoth(convertFailures, daoFaceFeaturesToFaces)
.mapBoth(convertFailures, _.map(daoFaceFeaturesToFaceFeatures))

def photoFaceFeaturesContains(faceId: FaceId): IO[PhotoStoreIssue, Boolean] =
faceFeaturesCollection
Expand All @@ -567,7 +554,7 @@ class PhotoStoreServiceLive private (

def photoFaceFeaturesUpsert(faceId: FaceId, faceFeatures: FaceFeatures): IO[PhotoStoreIssue, Unit] =
faceFeaturesCollection
.upsertOverwrite(faceIdToCollectionKey(faceId), faceFeaturesToDaoFaces(faceFeatures))
.upsertOverwrite(faceIdToCollectionKey(faceId), faceFeaturesToDaoFaceFeatures(faceFeatures))
.mapBoth(convertFailures, _ => ())

def photoFaceFeaturesDelete(faceId: FaceId): IO[PhotoStoreIssue, Unit] =
Expand Down
Loading

0 comments on commit a7c2dea

Please sign in to comment.