Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set upgrade during postprocessing #3336

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Set upgrade during postprocessing #3336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
489c6cb
Basic soulseek support
el133 Feb 27, 2024
81f6d9e
Added general maximum bitrate setting
el133 Feb 28, 2024
be6c8c5
Lossy will be set to Wanted Lossless during processing
el133 Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 37 additions & 2 deletions data/interfaces/default/config.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,33 @@ <h1 class="clearfix"><i class="fa fa-gear"></i> Settings</h1>
<label>Prefer</label>
<input type="radio" name="prefer_torrents" id="prefer_torrents_0" value="0" ${config['prefer_torrents_0']}>NZBs
<input type="radio" name="prefer_torrents" id="prefer_torrents_1" value="1" ${config['prefer_torrents_1']}>Torrents
<input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>No Preference
<input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>Soulseek
<input type="radio" name="prefer_torrents" id="prefer_torrents_3" value="3" ${config['prefer_torrents_3']}>No Preference
</div>
</fieldset>
</td>
<td>
<fieldset>
<legend>Soulseek</legend>
<div class="row">
<label>Soulseek API URL</label>
<input type="text" name="soulseek_api_url" value="${config['soulseek_api_url']}" size="50">
</div>
<div class="row">
<label>Soulseek API KEY</label>
<input type="text" name="soulseek_api_key" value="${config['soulseek_api_key']}" size="20">
</div>
<div class="row">
<label title="Path to folder where Headphones can find the downloads.">
Soulseek Download Dir:
</label>
<input type="text" name="soulseek_download_dir" value="${config['soulseek_download_dir']}" size="50">
</div>
<div class="row">
<label title="Path to folder where Headphones can find the downloads.">
Soulseek Incomplete Download Dir:
</label>
<input type="text" name="soulseek_incomplete_download_dir" value="${config['soulseek_incomplete_download_dir']}" size="50">
</div>
</fieldset>
</td>
Expand Down Expand Up @@ -589,14 +615,18 @@ <h1 class="clearfix"><i class="fa fa-gear"></i> Settings</h1>
</div>
</div>
</fieldset>

<fieldset>
<legend>Other</legend>
<fieldset>
<div class="row checkbox left">
<input id="use_bandcamp" type="checkbox" class="bigcheck" name="use_bandcamp" value="1" ${config['use_bandcamp']} /><label for="use_bandcamp"><span class="option">Bandcamp</span></label>
</div>
</fieldset>
<fieldset>
<div class="row checkbox left">
<input id="use_soulseek" type="checkbox" class="bigcheck" name="use_soulseek" value="1" ${config['use_soulseek']} /><label for="use_soulseek"><span class="option">Soulseek</span></label>
</div>
</fieldset>
</fieldset>
</td>
<td>
Expand Down Expand Up @@ -801,6 +831,11 @@ <h1 class="clearfix"><i class="fa fa-gear"></i> Settings</h1>
<td>
<legend>Quality</legend>
<fieldset>
<div class="row">
<label>Absolute maximum bitrate</label>
<input type="text" name="maximum_bitrate" value="${config['maximum_bitrate']}" size="20">
<small>0 for unlimited</small>
</div>
<div class="row checkbox left clearfix nopad">
<label title="Snatch the highest quality available, excluding lossless.">
<input type="radio" name="preferred_quality" id="preferred_quality0" value="0" ${config['pref_qual_0']}>
Expand Down
6 changes: 6 additions & 0 deletions headphones/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ def __repr__(self):
'LOSSLESS_BITRATE_FROM': (int, 'General', 0),
'LOSSLESS_BITRATE_TO': (int, 'General', 0),
'LOSSLESS_DESTINATION_DIR': (path, 'General', ''),
'MAXIMUM_BITRATE': (int, 'General', 0),
'MB_IGNORE_AGE': (int, 'General', 365),
'MB_IGNORE_AGE_MISSING': (int, 'General', 0),
'MIRROR': (str, 'General', 'musicbrainz.org'),
Expand Down Expand Up @@ -269,6 +270,11 @@ def __repr__(self):
'SONGKICK_ENABLED': (int, 'Songkick', 1),
'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
'SONGKICK_LOCATION': (str, 'Songkick', ''),
'SOULSEEK_API_URL': (str, 'Soulseek', ''),
'SOULSEEK_API_KEY': (str, 'Soulseek', ''),
'SOULSEEK_DOWNLOAD_DIR': (str, 'Soulseek', ''),
'SOULSEEK_INCOMPLETE_DOWNLOAD_DIR': (str, 'Soulseek', ''),
'SOULSEEK': (int, 'Soulseek', 0),
'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
'SUBSONIC_HOST': (str, 'Subsonic', ''),
'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),
Expand Down
57 changes: 47 additions & 10 deletions headphones/postprocessor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from beets import logging as beetslogging
from mediafile import MediaFile, FileTypeError, UnreadableFileError
from beetsplug import lyrics as beetslyrics
from headphones import notifiers, utorrent, transmission, deluge, qbittorrent
from headphones import notifiers, utorrent, transmission, deluge, qbittorrent, soulseek
from headphones import db, albumart, librarysync
from headphones import logger, helpers, mb, music_encoder
from headphones import metadata
Expand All @@ -36,20 +36,44 @@


def checkFolder():
logger.debug("Checking download folder for completed downloads (only snatched ones).")
logger.info("Checking download folder for completed downloads (only snatched ones).")

with postprocessor_lock:
myDB = db.DBConnection()
snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched"')

# If soulseek is used, this part will get the status from the soulseek api and return completed and errored albums
completed_albums, errored_albums = set(), set()
if any(album['Kind'] == 'soulseek' for album in snatched):
completed_albums, errored_albums = soulseek.download_completed()

for album in snatched:
if album['FolderName']:
folder_name = album['FolderName']
single = False
if album['Kind'] == 'nzb':
download_dir = headphones.CONFIG.DOWNLOAD_DIR
if album['Kind'] == 'soulseek':
if folder_name in errored_albums:
# If the album had any tracks with errors in it, the whole download is considered faulty. Status will be reset to wanted.
logger.info(f"Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.")
myDB.action('UPDATE albums SET Status="Wanted" WHERE AlbumID=? AND Status="Snatched"', (album['AlbumID'],))

# Folder will be removed from configured complete and Incomplete directory
complete_path = os.path.join(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR, folder_name)
incomplete_path = os.path.join(headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR, folder_name)
for path in [complete_path, incomplete_path]:
try:
shutil.rmtree(path)
except Exception as e:
pass
continue
elif folder_name in completed_albums:
download_dir = headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR
else:
continue
elif album['Kind'] == 'nzb':
download_dir = headphones.CONFIG.DOWNLOAD_DIR
elif album['Kind'] == 'bandcamp':
download_dir = headphones.CONFIG.BANDCAMP_DIR
download_dir = headphones.CONFIG.BANDCAMP_DIR
else:
if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3:
download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY
Expand Down Expand Up @@ -381,12 +405,15 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
# Need to update the downloaded track list with the new location.
# Could probably just throw in the "headphones-modified" folder,
# but this is good to make sure we're not counting files that may have failed to move

if new_folder:
downloaded_track_list = []
for r, d, f in os.walk(albumpath):
for files in f:
if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
downloaded_track_list.append(os.path.join(r, files))

all_files_lossless = all(file.lower().endswith(tuple('.' + format.lower() for format in headphones.LOSSLESS_MEDIA_FORMATS)) for file in downloaded_track_list)

builder = metadata.AlbumMetadataBuilder()
# Check if files are valid media files and are writable, before the steps
Expand Down Expand Up @@ -480,11 +507,19 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
if headphones.CONFIG.FILE_PERMISSIONS_ENABLED:
updateFilePermissions(albumpaths)

myDB = db.DBConnection()
myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid])
myDB.action(
'UPDATE snatched SET status = "Processed" WHERE Status NOT LIKE "Seed%" and AlbumID=?',
[albumid])
# If the downloaded tracks are lossy, but lossless is wanted, set status to wanted lossless
if not all_files_lossless and headphones.CONFIG.PREFERRED_QUALITY == 1:
myDB = db.DBConnection()
myDB.action('UPDATE albums SET status = "Wanted Lossless" WHERE AlbumID=?', [albumid])
myDB.action(
'UPDATE snatched SET status = "Processed" WHERE Status NOT LIKE "Seed%" and AlbumID=?',
[albumid])
else:
myDB = db.DBConnection()
myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid])
myDB.action(
'UPDATE snatched SET status = "Processed" WHERE Status NOT LIKE "Seed%" and AlbumID=?',
[albumid])

# Check if torrent has finished seeding
if headphones.CONFIG.TORRENT_DOWNLOADER != 0:
Expand Down Expand Up @@ -1172,6 +1207,8 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
download_dirs.append(dir)
if headphones.CONFIG.DOWNLOAD_DIR and not dir:
download_dirs.append(headphones.CONFIG.DOWNLOAD_DIR)
if headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR and not dir:
download_dirs.append(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR)
if headphones.CONFIG.DOWNLOAD_TORRENT_DIR and not dir:
download_dirs.append(
headphones.CONFIG.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace'))
Expand Down
62 changes: 57 additions & 5 deletions headphones/searcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@
from headphones.common import USER_AGENT
from headphones.types import Result
from headphones import logger, db, helpers, classes, sab, nzbget, request
from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent, bandcamp
from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent, bandcamp, soulseek


# Magnet to torrent services, for Black hole. Stolen from CouchPotato.
TORRENT_TO_MAGNET_SERVICES = [
Expand Down Expand Up @@ -260,6 +261,8 @@ def strptime_musicbrainz(date_str):


def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):


NZB_PROVIDERS = (headphones.CONFIG.HEADPHONES_INDEXER or
headphones.CONFIG.NEWZNAB or
headphones.CONFIG.NZBSORG or
Expand Down Expand Up @@ -300,7 +303,11 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
results = searchNZB(album, new, losslessOnly, albumlength)

if not results and headphones.CONFIG.BANDCAMP:
results = searchBandcamp(album, new, albumlength)
results = searchBandcamp(album, new, albumlength)

elif headphones.CONFIG.PREFER_TORRENTS == 2 and not choose_specific_download:
results = searchSoulseek(album, new, losslessOnly, albumlength)

else:

nzb_results = None
Expand Down Expand Up @@ -344,6 +351,7 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
(data, result) = preprocess(sorted_search_results)

if data and result:
#print(f'going to send stuff to downloader. data: {data}, album: {album}')
send_to_downloader(data, result, album)


Expand Down Expand Up @@ -427,7 +435,12 @@ def sort_search_results(resultlist, album, new, albumlength):
logger.info(
'No more results found for: %s - %s' % (album['ArtistName'], album['AlbumTitle']))
return None

print(headphones.CONFIG.MAXIMUM_BITRATE)
if headphones.CONFIG.MAXIMUM_BITRATE > 0:
resultlist = [
result for result in resultlist
if ((result.size * 8) / albumlength) <= headphones.CONFIG.MAXIMUM_BITRATE
]
# Add a priority if it has any of the preferred words
results_with_priority = []
preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS)
Expand Down Expand Up @@ -849,11 +862,15 @@ def send_to_downloader(data, result, album):
except Exception as e:
logger.error('Couldn\'t write NZB file: %s', e)
return

elif kind == 'bandcamp':
folder_name = bandcamp.download(album, result)
logger.info("Setting folder_name to: {}".format(folder_name))


elif kind == 'soulseek':
soulseek.download(user=result.user, filelist=result.files)
folder_name = result.folder

else:
folder_name = '%s - %s [%s]' % (
unidecode(album['ArtistName']).replace('/', '_'),
Expand Down Expand Up @@ -1918,14 +1935,49 @@ def set_proxy(proxy_url):
return results


def searchSoulseek(album, new=False, losslessOnly=False, albumlength=None):
# Not using some of the input stuff for now or ever
replacements = {
'...': '',
' & ': ' ',
' = ': ' ',
'?': '',
'$': '',
' + ': ' ',
'"': '',
',': '',
'*': '',
'.': '',
':': ''
}

num_tracks = get_album_track_count(album['AlbumID'])
year = get_year_from_release_date(album['ReleaseDate'])
cleanalbum = unidecode(helpers.replace_all(album['AlbumTitle'], replacements)).strip()
cleanartist = unidecode(helpers.replace_all(album['ArtistName'], replacements)).strip()

results = soulseek.search(artist=cleanartist, album=cleanalbum, year=year, losslessOnly=losslessOnly, num_tracks=num_tracks)

return results


def get_album_track_count(album_id):
# Not sure if this should be considered a helper function.
myDB = db.DBConnection()
track_count = myDB.select('SELECT COUNT(*) as count FROM tracks WHERE AlbumID=?', [album_id])[0]['count']
return track_count


# THIS IS KIND OF A MESS AND PROBABLY NEEDS TO BE CLEANED UP


def preprocess(resultlist):
for result in resultlist:

headers = {'User-Agent': USER_AGENT}

if result.kind == 'soulseek':
return True, result

if result.kind == 'torrent':

# rutracker always needs the torrent data
Expand Down
Loading