Merge branch 'development' of https://github.com/morpheus65535/bazarr …

…into development

README.md

@@ -62,6 +62,7 @@ If you need something that is not already part of Bazarr, feel free to create a
 - Karagarga.in
 - Ktuvit (Get `hashed_password` using method described [here](https://github.com/XBMCil/service.subtitles.ktuvit))
 - LegendasDivx
+- Legendas.net
 - Napiprojekt
 - Napisy24
 - Nekur

bazarr/api/system/settings.py

@@ -73,6 +73,7 @@ def post(self):
                             mustNotContain=str(item['mustNotContain']),
                             originalFormat=int(item['originalFormat']) if item['originalFormat'] not in None_Keys else
                             None,
+                            tag=item['tag'] if 'tag' in item else None,
                         )
                         .where(TableLanguagesProfiles.profileId == item['profileId']))
                     existing.remove(item['profileId'])
@@ -89,6 +90,7 @@ def post(self):
                             mustNotContain=str(item['mustNotContain']),
                             originalFormat=int(item['originalFormat']) if item['originalFormat'] not in None_Keys else
                             None,
+                            tag=item['tag'] if 'tag' in item else None,
                         ))
             for profileId in existing:
                 # Remove deleted profiles

bazarr/api/system/status.py

@@ -6,10 +6,12 @@
 
 from flask_restx import Resource, Namespace
 from tzlocal import get_localzone_name
+from alembic.migration import MigrationContext
 
 from radarr.info import get_radarr_info
 from sonarr.info import get_sonarr_info
 from app.get_args import args
+from app.database import engine, database, select
 from init import startTime
 
 from ..utils import authenticate
@@ -34,13 +36,25 @@ def get(self):
             timezone = "Exception while getting time zone name."
             logging.exception("BAZARR is unable to get configured time zone name.")
 
+        try:
+            database_version = ".".join([str(x) for x in engine.dialect.server_version_info])
+        except Exception:
+            database_version = ""
+
+        try:
+            database_migration = MigrationContext.configure(engine.connect()).get_current_revision()
+        except Exception:
+            database_migration = "unknown"
+
         system_status = {}
         system_status.update({'bazarr_version': os.environ["BAZARR_VERSION"]})
         system_status.update({'package_version': package_version})
         system_status.update({'sonarr_version': get_sonarr_info.version()})
         system_status.update({'radarr_version': get_radarr_info.version()})
         system_status.update({'operating_system': platform.platform()})
         system_status.update({'python_version': platform.python_version()})
+        system_status.update({'database_engine': f'{engine.dialect.name.capitalize()} {database_version}'})
+        system_status.update({'database_migration': database_migration})
         system_status.update({'bazarr_directory': os.path.dirname(os.path.dirname(os.path.dirname(
             os.path.dirname(__file__))))})
         system_status.update({'bazarr_config_directory': args.config_dir})

bazarr/app/config.py

@@ -31,12 +31,20 @@ def base_url_slash_cleaner(uri):
 
 
 def validate_ip_address(ip_string):
+    if ip_string == '*':
+        return True
     try:
         ip_address(ip_string)
         return True
     except ValueError:
         return False
 
+def validate_tags(tags):
+    if not tags:
+        return True
+
+    return all(re.match( r'^[a-z0-9_-]+$', item) for item in tags)
+
 
 ONE_HUNDRED_YEARS_IN_MINUTES = 52560000
 ONE_HUNDRED_YEARS_IN_HOURS = 876000
@@ -67,7 +75,7 @@ def check_parser_binary(value):
     # general section
     Validator('general.flask_secret_key', must_exist=True, default=hexlify(os.urandom(16)).decode(),
               is_type_of=str),
-    Validator('general.ip', must_exist=True, default='0.0.0.0', is_type_of=str, condition=validate_ip_address),
+    Validator('general.ip', must_exist=True, default='*', is_type_of=str, condition=validate_ip_address),
     Validator('general.port', must_exist=True, default=6767, is_type_of=int, gte=1, lte=65535),
     Validator('general.base_url', must_exist=True, default='', is_type_of=str),
     Validator('general.path_mappings', must_exist=True, default=[], is_type_of=list),
@@ -88,6 +96,9 @@ def check_parser_binary(value):
     Validator('general.use_sonarr', must_exist=True, default=False, is_type_of=bool),
     Validator('general.use_radarr', must_exist=True, default=False, is_type_of=bool),
     Validator('general.path_mappings_movie', must_exist=True, default=[], is_type_of=list),
+    Validator('general.serie_tag_enabled', must_exist=True, default=False, is_type_of=bool),
+    Validator('general.movie_tag_enabled', must_exist=True, default=False, is_type_of=bool),
+    Validator('general.remove_profile_tags', must_exist=True, default=[], is_type_of=list, condition=validate_tags),
     Validator('general.serie_default_enabled', must_exist=True, default=False, is_type_of=bool),
     Validator('general.serie_default_profile', must_exist=True, default='', is_type_of=(int, str)),
     Validator('general.movie_default_enabled', must_exist=True, default=False, is_type_of=bool),
@@ -176,7 +187,7 @@ def check_parser_binary(value):
     Validator('sonarr.only_monitored', must_exist=True, default=False, is_type_of=bool),
     Validator('sonarr.series_sync', must_exist=True, default=60, is_type_of=int,
               is_in=[15, 60, 180, 360, 720, 1440, 10080, ONE_HUNDRED_YEARS_IN_MINUTES]),
-    Validator('sonarr.excluded_tags', must_exist=True, default=[], is_type_of=list),
+    Validator('sonarr.excluded_tags', must_exist=True, default=[], is_type_of=list, condition=validate_tags),
     Validator('sonarr.excluded_series_types', must_exist=True, default=[], is_type_of=list),
     Validator('sonarr.use_ffprobe_cache', must_exist=True, default=True, is_type_of=bool),
     Validator('sonarr.exclude_season_zero', must_exist=True, default=False, is_type_of=bool),
@@ -199,7 +210,7 @@ def check_parser_binary(value):
     Validator('radarr.only_monitored', must_exist=True, default=False, is_type_of=bool),
     Validator('radarr.movies_sync', must_exist=True, default=60, is_type_of=int,
               is_in=[15, 60, 180, 360, 720, 1440, 10080, ONE_HUNDRED_YEARS_IN_MINUTES]),
-    Validator('radarr.excluded_tags', must_exist=True, default=[], is_type_of=list),
+    Validator('radarr.excluded_tags', must_exist=True, default=[], is_type_of=list, condition=validate_tags),
     Validator('radarr.use_ffprobe_cache', must_exist=True, default=True, is_type_of=bool),
     Validator('radarr.defer_search_signalr', must_exist=True, default=False, is_type_of=bool),
     Validator('radarr.sync_only_monitored_movies', must_exist=True, default=False, is_type_of=bool),
@@ -271,6 +282,10 @@ def check_parser_binary(value):
     Validator('legendasdivx.password', must_exist=True, default='', is_type_of=str, cast=str),
     Validator('legendasdivx.skip_wrong_fps', must_exist=True, default=False, is_type_of=bool),
 
+    # legendasnet section
+    Validator('legendasnet.username', must_exist=True, default='', is_type_of=str, cast=str),
+    Validator('legendasnet.password', must_exist=True, default='', is_type_of=str, cast=str),
+
     # ktuvit section
     Validator('ktuvit.email', must_exist=True, default='', is_type_of=str),
     Validator('ktuvit.hashed_password', must_exist=True, default='', is_type_of=str, cast=str),
@@ -298,6 +313,12 @@ def check_parser_binary(value):
 
     # analytics section
     Validator('analytics.enabled', must_exist=True, default=True, is_type_of=bool),
+
+    # jimaku section
+    Validator('jimaku.api_key', must_exist=True, default='', is_type_of=str),
+    Validator('jimaku.enable_name_search_fallback', must_exist=True, default=True, is_type_of=bool),
+    Validator('jimaku.enable_archives_download', must_exist=True, default=False, is_type_of=bool),
+    Validator('jimaku.enable_ai_subs', must_exist=True, default=False, is_type_of=bool),
 
     # titlovi section
     Validator('titlovi.username', must_exist=True, default='', is_type_of=str, cast=str),
@@ -454,6 +475,7 @@ def write_config():
               'enabled_integrations',
               'path_mappings',
               'path_mappings_movie',
+              'remove_profile_tags',
               'language_equals',
               'blacklisted_languages',
               'blacklisted_providers']

bazarr/app/database.py

@@ -379,14 +379,16 @@ def update_profile_id_list():
         'mustContain': ast.literal_eval(x.mustContain) if x.mustContain else [],
         'mustNotContain': ast.literal_eval(x.mustNotContain) if x.mustNotContain else [],
         'originalFormat': x.originalFormat,
+        'tag': x.tag,
     } for x in database.execute(
         select(TableLanguagesProfiles.profileId,
                TableLanguagesProfiles.name,
                TableLanguagesProfiles.cutoff,
                TableLanguagesProfiles.items,
                TableLanguagesProfiles.mustContain,
                TableLanguagesProfiles.mustNotContain,
-               TableLanguagesProfiles.originalFormat))
+               TableLanguagesProfiles.originalFormat,
+               TableLanguagesProfiles.tag))
         .all()
     ]
 
@@ -421,7 +423,7 @@ def get_profile_cutoff(profile_id):
     if profile_id and profile_id != 'null':
         cutoff_language = []
         for profile in profile_id_list:
-            profileId, name, cutoff, items, mustContain, mustNotContain, originalFormat = profile.values()
+            profileId, name, cutoff, items, mustContain, mustNotContain, originalFormat, tag = profile.values()
             if cutoff:
                 if profileId == int(profile_id):
                     for item in items:
@@ -511,7 +513,8 @@ def upgrade_languages_profile_hi_values():
                 TableLanguagesProfiles.items,
                 TableLanguagesProfiles.mustContain,
                 TableLanguagesProfiles.mustNotContain,
-                TableLanguagesProfiles.originalFormat)
+                TableLanguagesProfiles.originalFormat,
+                TableLanguagesProfiles.tag)
             ))\
             .all():
         items = json.loads(languages_profile.items)
@@ -525,3 +528,32 @@ def upgrade_languages_profile_hi_values():
             .values({"items": json.dumps(items)})
             .where(TableLanguagesProfiles.profileId == languages_profile.profileId)
         )
+
+
+def fix_languages_profiles_with_duplicate_ids():
+    languages_profiles = database.execute(
+        select(TableLanguagesProfiles.profileId, TableLanguagesProfiles.items, TableLanguagesProfiles.cutoff)).all()
+    for languages_profile in languages_profiles:
+        if languages_profile.cutoff:
+            # ignore profiles that have a cutoff set
+            continue
+        languages_profile_ids = []
+        languages_profile_has_duplicate = False
+        languages_profile_items = json.loads(languages_profile.items)
+        for items in languages_profile_items:
+            if items['id'] in languages_profile_ids:
+                languages_profile_has_duplicate = True
+                break
+            else:
+                languages_profile_ids.append(items['id'])
+
+        if languages_profile_has_duplicate:
+            item_id = 0
+            for items in languages_profile_items:
+                item_id += 1
+                items['id'] = item_id
+            database.execute(
+                update(TableLanguagesProfiles)
+                .values({"items": json.dumps(languages_profile_items)})
+                .where(TableLanguagesProfiles.profileId == languages_profile.profileId)
+            )

bazarr/app/get_providers.py

@@ -264,6 +264,10 @@ def get_providers_auth():
             'password': settings.legendasdivx.password,
             'skip_wrong_fps': settings.legendasdivx.skip_wrong_fps,
         },
+        'legendasnet': {
+            'username': settings.legendasnet.username,
+            'password': settings.legendasnet.password,
+        },
         'xsubs': {
             'username': settings.xsubs.username,
             'password': settings.xsubs.password,
@@ -285,6 +289,12 @@ def get_providers_auth():
             'username': settings.titlovi.username,
             'password': settings.titlovi.password,
         },
+        'jimaku': {
+            'api_key': settings.jimaku.api_key,
+            'enable_name_search_fallback': settings.jimaku.enable_name_search_fallback,
+            'enable_archives_download': settings.jimaku.enable_archives_download,
+            'enable_ai_subs': settings.jimaku.enable_ai_subs,
+        },
         'ktuvit': {
             'email': settings.ktuvit.email,
             'hashed_password': settings.ktuvit.hashed_password,

bazarr/app/logger.py

@@ -58,10 +58,13 @@ def formatException(self, record):
 
 class UnwantedWaitressMessageFilter(logging.Filter):
     def filter(self, record):
-        if settings.general.debug:
-            # no filtering in debug mode
+        if settings.general.debug or "BAZARR" in record.msg:
+            # no filtering in debug mode or if originating from us
             return True
 
+        if record.levelno < logging.ERROR:
+            return False
+
         unwantedMessages = [
             "Exception while serving /api/socket.io/",
             ['Session is disconnected', 'Session not found'],
@@ -161,17 +164,22 @@ def configure_logging(debug=False):
         logging.getLogger("websocket").setLevel(logging.CRITICAL)
         logging.getLogger("ga4mp.ga4mp").setLevel(logging.ERROR)
 
-    logging.getLogger("waitress").setLevel(logging.ERROR)
+    logging.getLogger("waitress").setLevel(logging.INFO)
     logging.getLogger("waitress").addFilter(UnwantedWaitressMessageFilter())
     logging.getLogger("knowit").setLevel(logging.CRITICAL)
     logging.getLogger("enzyme").setLevel(logging.CRITICAL)
     logging.getLogger("guessit").setLevel(logging.WARNING)
     logging.getLogger("rebulk").setLevel(logging.WARNING)
     logging.getLogger("stevedore.extension").setLevel(logging.CRITICAL)
 
+def empty_file(filename):
+    # Open the log file in write mode to clear its contents
+    with open(filename, 'w'):
+        pass  # Just opening and closing the file will clear it
 
 def empty_log():
     fh.doRollover()
+    empty_file(get_log_file_path())
     logging.info('BAZARR Log file emptied')
 
 

bazarr/app/server.py

@@ -50,7 +50,7 @@ def configure_server(self):
             self.connected = True
         except OSError as error:
             if error.errno == errno.EADDRNOTAVAIL:
-                logging.exception("BAZARR cannot bind to specified IP, trying with default (0.0.0.0)")
+                logging.exception("BAZARR cannot bind to specified IP, trying with 0.0.0.0")
                 self.address = '0.0.0.0'
                 self.connected = False
                 super(Server, self).__init__()
@@ -76,8 +76,7 @@ def interrupt_handler(self, signum, frame):
             self.shutdown(EXIT_INTERRUPT)
 
     def start(self):
-        logging.info(f'BAZARR is started and waiting for request on http://{self.server.effective_host}:'
-                     f'{self.server.effective_port}')
+        self.server.print_listen("BAZARR is started and waiting for requests on: http://{}:{}")
         signal.signal(signal.SIGINT, self.interrupt_handler)
         try:
             self.server.run()

bazarr/languages/custom_lang.py

@@ -5,7 +5,8 @@
 
 from subzero.language import Language
 
-from app.database import database, insert
+from app.database import database, insert, update
+from sqlalchemy.exc import IntegrityError
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +19,7 @@ class CustomLanguage:
     language = "pt-BR"
     official_alpha2 = "pt"
     official_alpha3 = "por"
-    name = "Brazilian Portuguese"
+    name = "Portuguese (Brazil)"
     iso = "BR"
     _scripts = []
     _possible_matches = ("pt-br", "pob", "pb", "brazilian", "brasil", "brazil")
@@ -50,13 +51,19 @@ def register(cls, table):
         """Register the custom language subclasses in the database."""
 
         for sub in cls.__subclasses__():
-            database.execute(
-                insert(table)
-                .values(code3=sub.alpha3,
-                        code2=sub.alpha2,
-                        name=sub.name,
-                        enabled=0)
-                .on_conflict_do_nothing())
+            try:
+                database.execute(
+                    insert(table)
+                    .values(code3=sub.alpha3,
+                            code2=sub.alpha2,
+                            name=sub.name,
+                            enabled=0))
+            except IntegrityError:
+                database.execute(
+                    update(table)
+                    .values(code2=sub.alpha2,
+                            name=sub.name)
+                    .where(table.code3 == sub.alpha3))
 
     @classmethod
     def found_external(cls, subtitle, subtitle_path):
@@ -212,7 +219,7 @@ class LatinAmericanSpanish(CustomLanguage):
     language = "es-MX"
     official_alpha2 = "es"
     official_alpha3 = "spa"
-    name = "Latin American Spanish"
+    name = "Spanish (Latino)"
     iso = "MX"  # Not fair, but ok
     _scripts = ("419",)
     _possible_matches = (

bazarr/languages/get_languages.py

@@ -44,6 +44,12 @@ def create_languages_dict():
         .values(name='Chinese Simplified')
         .where(TableSettingsLanguages.code3 == 'zho'))
 
+    # replace Modern Greek by Greek to match Sonarr and Radarr languages
+    database.execute(
+        update(TableSettingsLanguages)
+        .values(name='Greek')
+        .where(TableSettingsLanguages.code3 == 'ell'))
+
     languages_dict = [{
         'code3': x.code3,
         'code2': x.code2,
@@ -55,6 +61,19 @@ def create_languages_dict():
         .all()]
 
 
+def audio_language_from_name(lang):
+    lang_map = {
+        'Chinese': 'zh',
+    }
+
+    alpha2_code = lang_map.get(lang, None)
+
+    if alpha2_code is None:
+        return lang
+
+    return language_from_alpha2(alpha2_code)
+
+
 def language_from_alpha2(lang):
     return next((item['name'] for item in languages_dict if item['code2'] == lang[:2]), None)