diff --git a/cli-commands.txt b/cli-commands.txt index 82e4eb6..ee232bd 100644 --- a/cli-commands.txt +++ b/cli-commands.txt @@ -39,8 +39,6 @@ backup cleanup-ended-videos now -scrub-ended-properties - download-video video-id^* prefer-single-format @@ -114,6 +112,10 @@ remove-videos source force +scrub-deposition-properties + +scrub-ended-properties + serve port$* stderr$ diff --git a/cli/scrub_deposition_properties.go b/cli/scrub_deposition_properties.go new file mode 100644 index 0000000..7b833ed --- /dev/null +++ b/cli/scrub_deposition_properties.go @@ -0,0 +1,82 @@ +package cli + +import ( + "github.com/boggydigital/kevlar" + "github.com/boggydigital/nod" + "github.com/boggydigital/yet/data" + "net/url" + "slices" +) + +func ScrubDepositionPropertiesHandler(u *url.URL) error { + return ScrubDepositionProperties(nil) +} + +// ScrubDepositionProperties will remove all accumulated property depositions: +// - search results +// - older channel and playlist videos properties +// To do that we start by identifying all critical videos: +// - part of current channel, playlist data +// - downloaded, not-ended videos +// Then we iterate over all non-preserved properties and remove data for all non-critical videos +func ScrubDepositionProperties(rdx kevlar.WriteableRedux) error { + sdpa := nod.NewProgress("scrubbing deposition properties...") + defer sdpa.End() + + var err error + rdx, err = validateWritableRedux(rdx, data.AllProperties()...) + if err != nil { + return sdpa.EndWithError(err) + } + + currentVideoIds := make(map[string]any) + + for _, videoId := range rdx.Keys(data.VideoDownloadCompletedProperty) { + if rdx.HasKey(data.VideoEndedDateProperty, videoId) { + continue + } + currentVideoIds[videoId] = nil + } + + for _, channelId := range rdx.Keys(data.ChannelAutoRefreshProperty) { + if videos, ok := rdx.GetAllValues(data.ChannelVideosProperty, channelId); ok { + for _, videoId := range videos { + currentVideoIds[videoId] = nil + } + } + } + + for _, playlistId := range rdx.Keys(data.PlaylistAutoRefreshProperty) { + if videos, ok := rdx.GetAllValues(data.PlaylistVideosProperty, playlistId); ok { + for _, videoId := range videos { + currentVideoIds[videoId] = nil + } + } + } + + properties := data.VideoProperties() + + sdpa.TotalInt(len(properties)) + + for _, vp := range properties { + if slices.Contains(preserveVideoProperties, vp) { + sdpa.Increment() + continue + } + videos := rdx.Keys(vp) + for _, videoId := range videos { + if _, ok := currentVideoIds[videoId]; !ok { + if rdx.HasKey(vp, videoId) { + if err := rdx.CutKeys(vp, videoId); err != nil { + return sdpa.EndWithError(err) + } + } + } + } + sdpa.Increment() + } + + sdpa.EndWithResult("done") + + return nil +} diff --git a/cli/scrub_ended_properties.go b/cli/scrub_ended_properties.go index 6cc1de9..1253363 100644 --- a/cli/scrub_ended_properties.go +++ b/cli/scrub_ended_properties.go @@ -18,12 +18,13 @@ var preserveVideoProperties = []string{ data.VideoDownloadCleanedUpProperty, // required for cleanup } -func ScrubEndedPropertiesHandler(u *url.URL) error { +func ScrubEndedPropertiesHandler(_ *url.URL) error { return ScrubEndedProperties(nil) } +// ScrubEndedProperties will remove all non-preserved properties for ended videos. +// Preserved properties are required for core functionality - history, cleanup, etc. func ScrubEndedProperties(rdx kevlar.WriteableRedux) error { - sevpa := nod.NewProgress("scrubbing ended videos properties...") defer sevpa.End() diff --git a/cli/sync.go b/cli/sync.go index 03e74bc..5301a92 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -56,6 +56,9 @@ func Sync(rdx kevlar.WriteableRedux, opt *VideoOptions) error { if err := ScrubEndedProperties(rdx); err != nil { return sa.EndWithError(err) } + if err := ScrubDepositionProperties(rdx); err != nil { + return sa.EndWithError(err) + } if err := CleanupEndedVideos(false, rdx); err != nil { return sa.EndWithError(err) } diff --git a/main.go b/main.go index 39a284d..308d228 100644 --- a/main.go +++ b/main.go @@ -47,32 +47,33 @@ func main() { } clo.HandleFuncs(map[string]clo.Handler{ - "add-channel": cli.AddChannelHandler, - "add-playlist": cli.AddPlaylistHandler, - "add-video": cli.AddVideoHandler, - "backup": cli.BackupHandler, - "cleanup-ended-videos": cli.CleanupEndedVideosHandler, - "download-video": cli.DownloadVideoHandler, - "get-captions": cli.GetCaptionsHandler, - "get-channels-metadata": cli.GetChannelsMetadataHandler, - "get-playlists-metadata": cli.GetPlaylistsMetadataHandler, - "get-poster": cli.GetPosterHandler, - "get-rutube-video": cli.GetRuTubeVideoHandler, - "get-video-metadata": cli.GetVideoMetadataHandler, - "process-queue": cli.ProcessQueueHandler, - "migrate": cli.MigrateHandler, - "queue-channels-downloads": cli.QueueChannelsDownloadsHandler, - "queue-playlists-downloads": cli.QueuePlaylistsDownloadsHandler, - "refresh-channels-metadata": cli.RefreshChannelsMetadataHandler, - "refresh-playlists-metadata": cli.RefreshPlaylistsMetadataHandler, - "remove-channel": cli.RemoveChannelHandler, - "remove-playlist": cli.RemovePlaylistHandler, - "remove-videos": cli.RemoveVideosHandler, - "scrub-ended-properties": cli.ScrubEndedPropertiesHandler, - "serve": cli.ServeHandler, - "sync": cli.SyncHandler, - "test-dependencies": cli.TestDependenciesHandler, - "version": cli.VersionHandler, + "add-channel": cli.AddChannelHandler, + "add-playlist": cli.AddPlaylistHandler, + "add-video": cli.AddVideoHandler, + "backup": cli.BackupHandler, + "cleanup-ended-videos": cli.CleanupEndedVideosHandler, + "download-video": cli.DownloadVideoHandler, + "get-captions": cli.GetCaptionsHandler, + "get-channels-metadata": cli.GetChannelsMetadataHandler, + "get-playlists-metadata": cli.GetPlaylistsMetadataHandler, + "get-poster": cli.GetPosterHandler, + "get-rutube-video": cli.GetRuTubeVideoHandler, + "get-video-metadata": cli.GetVideoMetadataHandler, + "process-queue": cli.ProcessQueueHandler, + "migrate": cli.MigrateHandler, + "queue-channels-downloads": cli.QueueChannelsDownloadsHandler, + "queue-playlists-downloads": cli.QueuePlaylistsDownloadsHandler, + "refresh-channels-metadata": cli.RefreshChannelsMetadataHandler, + "refresh-playlists-metadata": cli.RefreshPlaylistsMetadataHandler, + "remove-channel": cli.RemoveChannelHandler, + "remove-playlist": cli.RemovePlaylistHandler, + "remove-videos": cli.RemoveVideosHandler, + "scrub-deposition-properties": cli.ScrubDepositionPropertiesHandler, + "scrub-ended-properties": cli.ScrubEndedPropertiesHandler, + "serve": cli.ServeHandler, + "sync": cli.SyncHandler, + "test-dependencies": cli.TestDependenciesHandler, + "version": cli.VersionHandler, }) if err := defs.AssertCommandsHaveHandlers(); err != nil {